Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
98b9f2cf73 | ||
|
|
6fa65ebb9f | ||
|
|
add163a092 | ||
|
|
7b84d9b10a | ||
|
|
e9a1dbca4b | ||
|
|
145a2654d2 | ||
|
|
9f14cdb1d7 | ||
|
|
5cefa2e995 | ||
|
|
13cbe0fd42 | ||
|
|
c8bb0c0bc8 | ||
|
|
4cc941fbec | ||
|
|
b010192f8e | ||
|
|
2b1af8eea2 | ||
|
|
6ea1f6f78c | ||
|
|
1215470cc1 | ||
|
|
94195b7018 | ||
|
|
3c9c178bdc | ||
|
|
cc64547920 | ||
|
|
a23f2ca956 | ||
|
|
4dacc52ecb | ||
|
|
ccaeecfd46 | ||
|
|
3e45449e8c | ||
|
|
f352a3c597 | ||
|
|
8cdbc847d1 | ||
|
|
54dbf66cb7 | ||
|
|
177d384126 | ||
|
|
561fe757f3 | ||
|
|
c670ab638f | ||
|
|
eea5cbbafc | ||
|
|
e4c61a4a13 | ||
|
|
0f0afadc94 | ||
|
|
659563f27a | ||
|
|
cbd8a68d0f | ||
|
|
54229282a4 | ||
|
|
514575d0f1 | ||
|
|
756427560c | ||
|
|
900eeb177a | ||
|
|
e67fa3cadc | ||
|
|
c7c32fd4dc | ||
|
|
9471748f4c | ||
|
|
920b1868bb | ||
|
|
eed7c54aaf | ||
|
|
36e58a33ad | ||
|
|
93e8ff749c | ||
|
|
6cb22444ac | ||
|
|
78df101473 | ||
|
|
840508b8ba | ||
|
|
2761254790 | ||
|
|
f47d174f0b | ||
|
|
c50fbc7935 | ||
|
|
75f23a1723 | ||
|
|
f0aa31f9e8 | ||
|
|
aaf59218da | ||
|
|
0203ef66ea | ||
|
|
9c66108dc5 | ||
|
|
558da0a027 | ||
|
|
1d39ec0317 | ||
|
|
281b6468a0 | ||
|
|
9ba6da1f4d | ||
|
|
bde7a7fed6 | ||
|
|
7e021746e4 | ||
|
|
a61bb5f528 | ||
|
|
6a5289b2a2 | ||
|
|
081a5df82c | ||
|
|
3e77ec8008 | ||
|
|
b0b222afa8 | ||
|
|
d85d6f627d | ||
|
|
9505f93da6 | ||
|
|
1671034330 | ||
|
|
3f0ca563e4 | ||
|
|
ffa2a5e8fd | ||
|
|
c3633895d0 | ||
|
|
c15af0a63a | ||
|
|
56e2d77039 | ||
|
|
2cf6fa14a1 | ||
|
|
c374725255 | ||
|
|
babfb6b8d6 | ||
|
|
2ffa8ec7d0 | ||
|
|
b516119224 | ||
|
|
c0e6bb97d8 | ||
|
|
cc30223ff1 | ||
|
|
53c8748942 | ||
|
|
b3a8966e71 | ||
|
|
74800a0767 | ||
|
|
cadd1f993d | ||
|
|
d3807cb86c | ||
|
|
735e9195b5 | ||
|
|
a6619a628e | ||
|
|
456572f87c | ||
|
|
f024b96934 | ||
|
|
25c71bcd98 | ||
|
|
319e7ecc61 | ||
|
|
d9ece5e96a | ||
|
|
4bfe4450b1 | ||
|
|
f9f12ffb61 | ||
|
|
63f81d1033 | ||
|
|
8895d21abf | ||
|
|
2ac1ebeaa0 | ||
|
|
25437602d7 | ||
|
|
8159c71950 | ||
|
|
aa66af7ba6 | ||
|
|
be3633622c | ||
|
|
31655a641c | ||
|
|
b235f6fc6b | ||
|
|
76b297b173 | ||
|
|
1e5cff1d8b | ||
|
|
4e2296d5f6 | ||
|
|
18310632eb | ||
|
|
80ed8d9869 | ||
|
|
2a2658d440 | ||
|
|
710e549c03 | ||
|
|
47fae977dd | ||
|
|
0111919975 | ||
|
|
80837cb040 | ||
|
|
46153a8991 | ||
|
|
a9fccfd4ce | ||
|
|
dfb57e9438 | ||
|
|
eda2731702 | ||
|
|
f47eac5935 | ||
|
|
3bb4dece5a | ||
|
|
0504e231a1 | ||
|
|
6097e14617 | ||
|
|
0b149ce5f1 | ||
|
|
79384e7bf8 | ||
|
|
01eb79cac2 | ||
|
|
128b32c62c | ||
|
|
c1233bd26b | ||
|
|
e23f67def6 | ||
|
|
95f4b29be5 | ||
|
|
5727c708cf | ||
|
|
e49f0f99a1 | ||
|
|
84d340382b | ||
|
|
5bfb067d0f | ||
|
|
d8cb62a92c | ||
|
|
aab8cc40f9 | ||
|
|
9c82689f65 | ||
|
|
ce2755467d | ||
|
|
02b19402ae | ||
|
|
80af66e852 | ||
|
|
15cfa45646 | ||
|
|
ac63849b49 | ||
|
|
02142e352b | ||
|
|
3d92413372 | ||
|
|
5022ed2acc | ||
|
|
31b9d22649 | ||
|
|
7a54894015 | ||
|
|
7100345a09 | ||
|
|
929db15214 | ||
|
|
d9455926c5 | ||
|
|
ba5cd741e1 | ||
|
|
8c94b6243e | ||
|
|
f6473e3383 | ||
|
|
3cd9d6a604 | ||
|
|
142281777e | ||
|
|
8e7b8af180 | ||
|
|
c046cd02d5 | ||
|
|
f5508fcdf1 | ||
|
|
adc2eab6da | ||
|
|
702698d8aa | ||
|
|
73dbe1de37 | ||
|
|
c5706fa28b | ||
|
|
c906a9571e | ||
|
|
038712f431 | ||
|
|
ed2d2c6f86 | ||
|
|
e7158e5a4b | ||
|
|
c66df259a6 | ||
|
|
0b8437fa12 | ||
|
|
a32b6ebf82 | ||
|
|
edbda1696e | ||
|
|
ece218785c | ||
|
|
cc1b3609a1 | ||
|
|
0841aad07a | ||
|
|
30b04ed0c2 | ||
|
|
90b28b3dbd | ||
|
|
0f7ca8f24e | ||
|
|
726d85ebd0 | ||
|
|
c524de9198 | ||
|
|
2fbf03d8ca | ||
|
|
68b8112af0 | ||
|
|
975e83a015 | ||
|
|
1ddb02dbb7 | ||
|
|
1159204ed8 | ||
|
|
a5212e7741 | ||
|
|
1932fd72e3 | ||
|
|
c061fe091b | ||
|
|
3621983a4d | ||
|
|
b6656690ca | ||
|
|
0c9e82ee6b | ||
|
|
b8ebb1faf1 | ||
|
|
690575193f | ||
|
|
9e9cf63c90 | ||
|
|
f5dc213e53 | ||
|
|
138e9c1486 | ||
|
|
7053c0dc93 | ||
|
|
ee8a653b13 | ||
|
|
71fad7e97f | ||
|
|
04740eb928 | ||
|
|
8c63d58c96 | ||
|
|
991672d423 | ||
|
|
55ce38cc87 | ||
|
|
96f3390407 | ||
|
|
aa2426d4e4 | ||
|
|
d0e3e13852 | ||
|
|
a2a1e30d68 | ||
|
|
a6f08b9e0c | ||
|
|
e70842a38c | ||
|
|
2af98d7d42 | ||
|
|
f28eee34e8 | ||
|
|
208610b645 | ||
|
|
e38f8076bc | ||
|
|
6180e1a48b | ||
|
|
e8fe1cd14d | ||
|
|
75d1fa585c | ||
|
|
072ff4b02e | ||
|
|
d8c4e2a72e | ||
|
|
66396de6a3 | ||
|
|
1ca9f575ef | ||
|
|
272e074a15 | ||
|
|
9a738c3783 | ||
|
|
6b4519e959 | ||
|
|
9b1d5213a6 | ||
|
|
f9ed9a4505 | ||
|
|
3ed7ce5630 | ||
|
|
63cd8343c4 | ||
|
|
2d5f8aed24 | ||
|
|
b8d23d0965 | ||
|
|
f4ef92e653 | ||
|
|
c7430f4ae9 | ||
|
|
623fda3087 | ||
|
|
80f27f2b29 | ||
|
|
defbd73b4f | ||
|
|
5285982515 | ||
|
|
f83daa5a22 | ||
|
|
0a07c8f131 | ||
|
|
35481a92f5 | ||
|
|
0995d864e8 | ||
|
|
1a42e43573 | ||
|
|
132aa569fe | ||
|
|
e93a787bd6 | ||
|
|
d6f902653f | ||
|
|
d172d43ddf | ||
|
|
67909724e7 | ||
|
|
1c88cbc5f6 | ||
|
|
3fb4831804 | ||
|
|
b65570fb16 | ||
|
|
871e6e6f64 | ||
|
|
0cd92a96c0 | ||
|
|
ff31ddd2d5 | ||
|
|
e9f5b43785 | ||
|
|
36b41081ab | ||
|
|
1092313892 | ||
|
|
ce605cb8f3 | ||
|
|
0733d885fb | ||
|
|
ef6fe3d1d9 | ||
|
|
82a4fea81a | ||
|
|
14a0252819 | ||
|
|
849aeb65f0 | ||
|
|
a99dc6e93d | ||
|
|
1a5fcc1348 | ||
|
|
c9d88f5628 | ||
|
|
0910711aca | ||
|
|
6a1a9ef6bc | ||
|
|
ef1c059079 | ||
|
|
a05ab37373 | ||
|
|
7ea88cfd22 | ||
|
|
8d14fe6a90 | ||
|
|
49a05b42f9 | ||
|
|
b6da1d7769 | ||
|
|
7e2ad58edd | ||
|
|
7ad0d98404 | ||
|
|
b7052bba95 | ||
|
|
4a5618bcc4 | ||
|
|
c1850fe604 | ||
|
|
fccdf32c9f | ||
|
|
c474411e1b | ||
|
|
9033eac5b1 | ||
|
|
0ba0426591 | ||
|
|
9a9b540701 | ||
|
|
b8ddbb9a26 | ||
|
|
bc3c2b96ed | ||
|
|
622dc87d84 | ||
|
|
18edbdf404 | ||
|
|
02a3eab693 | ||
|
|
dded868a80 | ||
|
|
6fe878bdb9 | ||
|
|
505ece9e36 | ||
|
|
72acfa1e0c | ||
|
|
02cd8b9a44 | ||
|
|
f32250f431 | ||
|
|
1523cad044 | ||
|
|
ed55353700 | ||
|
|
8d3bd45102 | ||
|
|
f69753f6a0 | ||
|
|
b7e2e87ac1 | ||
|
|
3b98508986 | ||
|
|
36ebfc42d9 | ||
|
|
ae5f55ff65 | ||
|
|
01131da5b7 | ||
|
|
66ae865de0 | ||
|
|
8c50f08fec | ||
|
|
e69de47daf | ||
|
|
acef10e848 | ||
|
|
edb9d50d9e | ||
|
|
b989d4f6be | ||
|
|
84dd5c0eca | ||
|
|
db8daf20b7 | ||
|
|
cb985b5a03 | ||
|
|
28c4e38272 | ||
|
|
b29377dc2b | ||
|
|
ca09592a72 | ||
|
|
b0e14fae43 | ||
|
|
dfa68f126b | ||
|
|
4570f68727 | ||
|
|
4705acba55 | ||
|
|
1dea20a358 | ||
|
|
191f661b35 | ||
|
|
f3ab38ba8f | ||
|
|
2d0a892e70 | ||
|
|
00eefe87d7 | ||
|
|
f68fd96555 | ||
|
|
fc50a3242f | ||
|
|
64e9ce2bfa | ||
|
|
e8d04b1921 | ||
|
|
9fedf14e55 | ||
|
|
7d535ef7d1 | ||
|
|
85d610bcdf | ||
|
|
3af8b7bd0c | ||
|
|
6215d85581 | ||
|
|
308d9f5abd | ||
|
|
bcfabba038 | ||
|
|
69139b1987 | ||
|
|
3f8f9d9d73 | ||
|
|
2283579109 | ||
|
|
1e0af94ba2 | ||
|
|
dd631328e0 | ||
|
|
cca2a57c69 | ||
|
|
bbfbcacc6a | ||
|
|
3f95c1a9d5 | ||
|
|
7bd05e9b16 | ||
|
|
10742d7c51 | ||
|
|
bbf7940e45 | ||
|
|
f5af71f4e6 | ||
|
|
1ffcd67c9d | ||
|
|
cf8592ba12 | ||
|
|
1e1d036590 | ||
|
|
64afb6d305 | ||
|
|
20a9e018f7 | ||
|
|
a574c33ac6 | ||
|
|
d21e458fe1 | ||
|
|
bb3ba53c6e | ||
|
|
ba2e4df76b | ||
|
|
33e7e4bee3 | ||
|
|
b83dce2cb3 | ||
|
|
6821676853 | ||
|
|
52024986ee | ||
|
|
2e3342d20d | ||
|
|
419aa1d78f | ||
|
|
de4ddd578f | ||
|
|
5dd68a7ed4 | ||
|
|
881168dc7a | ||
|
|
295e427abe | ||
|
|
67c5d15f51 | ||
|
|
39a7f37f35 | ||
|
|
15d276b0bc | ||
|
|
4387025c48 | ||
|
|
52c20f02c7 | ||
|
|
413dab8094 | ||
|
|
6b1d11a04d | ||
|
|
5441379f38 | ||
|
|
52d954c64e | ||
|
|
415495fc16 | ||
|
|
4223298634 | ||
|
|
e058c71019 | ||
|
|
12b2bbfa26 | ||
|
|
7d22bfffa5 | ||
|
|
4ade80c8a9 | ||
|
|
9674381fc2 | ||
|
|
f6d91280d7 | ||
|
|
e4ebbd8f69 | ||
|
|
7f5560de6d | ||
|
|
0925171e07 | ||
|
|
900d3ad01d | ||
|
|
6a1efedc1f | ||
|
|
0f602e0e62 | ||
|
|
3a425fc47d | ||
|
|
36722cbbd8 | ||
|
|
9572336e96 | ||
|
|
4895cde474 | ||
|
|
21ca1a73cf | ||
|
|
457c1c6ae6 | ||
|
|
c1ff127101 | ||
|
|
5e815a5832 | ||
|
|
4604e9cd54 | ||
|
|
ef580a7680 | ||
|
|
2102776be8 | ||
|
|
f3af7a6b77 | ||
|
|
5abeceac1f | ||
|
|
22f73a0817 | ||
|
|
29b5aef73a | ||
|
|
3e602bce52 | ||
|
|
fe39901885 | ||
|
|
ee9e8b365c | ||
|
|
bd6fa35d4a | ||
|
|
5e1fdf66b0 | ||
|
|
d86e709b90 | ||
|
|
f82cffe882 | ||
|
|
3d88eb92c1 | ||
|
|
0811ebb52d | ||
|
|
491d431ea5 | ||
|
|
ceeebebcad | ||
|
|
04d766aba7 | ||
|
|
3b431e180e | ||
|
|
a2eab1577d | ||
|
|
f8dd3d05dc | ||
|
|
93993e62d6 | ||
|
|
483c88b958 | ||
|
|
4a1e846958 | ||
|
|
927b9afe7c | ||
|
|
9e8827f015 | ||
|
|
ca45f461ae | ||
|
|
7e5af11c2d | ||
|
|
102436274f | ||
|
|
49c26a76b4 | ||
|
|
d469ef7bf2 | ||
|
|
a5374cb876 | ||
|
|
a53e911a0c | ||
|
|
c4a78a149b | ||
|
|
3814912adf | ||
|
|
75ef652cb3 | ||
|
|
1eae3fe5dc | ||
|
|
d0bc383a79 | ||
|
|
ea5d31b530 | ||
|
|
e56ad19e81 | ||
|
|
ed393eb75c | ||
|
|
2e2d309d38 | ||
|
|
6bfb33adb2 | ||
|
|
05138e1db1 | ||
|
|
b65402db41 | ||
|
|
fbef187400 | ||
|
|
25d1b8b9e5 | ||
|
|
ec91594083 | ||
|
|
e67f55bd74 | ||
|
|
276a7f9d2a | ||
|
|
4888d7a567 | ||
|
|
767f78315b | ||
|
|
4311b218fb | ||
|
|
89892042dd | ||
|
|
a22dfb3864 | ||
|
|
f40c42dcf0 | ||
|
|
1d31aa9da3 | ||
|
|
f999073fb4 | ||
|
|
5eb7c64ec3 | ||
|
|
7e79818555 | ||
|
|
a51576b250 | ||
|
|
0bee2689d1 | ||
|
|
c232743963 | ||
|
|
c2d5f092b3 | ||
|
|
8e8e186de9 | ||
|
|
55f47a2b1b | ||
|
|
5c27b9eb90 | ||
|
|
1d018533d1 | ||
|
|
34b8841f5c | ||
|
|
a54166a27c | ||
|
|
4adec39dce | ||
|
|
e63ae08638 | ||
|
|
f1b4c6a1ec | ||
|
|
78b4b3ffa8 | ||
|
|
f51fd48cdb | ||
|
|
9127a816e3 | ||
|
|
1a5cd9b062 | ||
|
|
a86b4c4e58 | ||
|
|
a384960a7b | ||
|
|
5817b80c94 | ||
|
|
97efd63c99 | ||
|
|
5ebcc01bcc | ||
|
|
ddcc008ea6 | ||
|
|
8ec10e2304 | ||
|
|
172f275bc7 | ||
|
|
0dac5943bb | ||
|
|
2dcd2b13be | ||
|
|
8743decd17 | ||
|
|
a15d7ca543 | ||
|
|
a90029ec50 | ||
|
|
563896b126 | ||
|
|
5aecd02668 | ||
|
|
eef6312a17 | ||
|
|
1e48f5c8f4 | ||
|
|
7908fa2814 | ||
|
|
7ffb3ea042 | ||
|
|
842dd90780 | ||
|
|
ee71a03f96 | ||
|
|
5221631b40 | ||
|
|
ade2b4ac0d | ||
|
|
643b95c0e4 | ||
|
|
ed0f9d4fb5 | ||
|
|
6f353c31cb | ||
|
|
404412a950 | ||
|
|
0e1778d8da | ||
|
|
dfc0ca9525 | ||
|
|
53bf1fe4fd | ||
|
|
5c05daa007 | ||
|
|
f74f075b4e | ||
|
|
1b53e8e98c | ||
|
|
4caee284bc | ||
|
|
5368c91399 | ||
|
|
00b1881f20 | ||
|
|
b727bd20c8 | ||
|
|
5a703c116f | ||
|
|
57f728f40e | ||
|
|
74bbde0c34 | ||
|
|
c58fc0781c | ||
|
|
7a266d49c1 | ||
|
|
344eeeb8c0 | ||
|
|
08d50ddfc5 | ||
|
|
f83c892b13 | ||
|
|
7dbec9d395 | ||
|
|
5171c801d9 | ||
|
|
72b4a778a6 | ||
|
|
0254df63f9 | ||
|
|
03c02ff74f | ||
|
|
9277c3c674 | ||
|
|
34b1259dc6 | ||
|
|
257e38de12 | ||
|
|
1984132eae | ||
|
|
8b1d75ea98 | ||
|
|
a79af11371 | ||
|
|
087a9b70b4 | ||
|
|
8b37952128 | ||
|
|
8336930091 | ||
|
|
d875325cee | ||
|
|
f1c5d1cb23 | ||
|
|
b55752e715 | ||
|
|
36a99e6ec8 | ||
|
|
df3c5b9a77 | ||
|
|
4c2ab23c1f | ||
|
|
0a3f81e364 | ||
|
|
69c5631b25 | ||
|
|
cca4331ac4 | ||
|
|
5972dbb6a7 | ||
|
|
794f2d8576 | ||
|
|
e35dce530a | ||
|
|
6034bd3cb2 | ||
|
|
42c77bbb67 | ||
|
|
d504c8f4a3 | ||
|
|
976c8b3ce0 | ||
|
|
905e7e41a6 | ||
|
|
16f48d0400 | ||
|
|
6d7608d851 | ||
|
|
7cd30efbe9 | ||
|
|
8c3652e1ac | ||
|
|
7b9e3aace9 | ||
|
|
568737e123 | ||
|
|
808e3ea6e3 | ||
|
|
e4f14d871f | ||
|
|
1cedd8ee5b | ||
|
|
b80fe24179 | ||
|
|
3cea41a854 | ||
|
|
62b80c13cc | ||
|
|
08b612575c | ||
|
|
b3b804bf2d | ||
|
|
09d8a86e30 | ||
|
|
cc5d917c72 | ||
|
|
7cc3e9d5ef | ||
|
|
87aa9fc6c4 | ||
|
|
8d5c315817 | ||
|
|
f89486f5fe | ||
|
|
cf4d78a24c | ||
|
|
ee88970075 | ||
|
|
f5fa40feb7 | ||
|
|
da13192cfe | ||
|
|
8ff65a65db | ||
|
|
2d882a11a5 | ||
|
|
79adc677a1 | ||
|
|
a8e2dad6d0 | ||
|
|
16954c5626 | ||
|
|
56305c4ede | ||
|
|
30713cc21d | ||
|
|
2bac0a25e0 | ||
|
|
c3aa5b0515 | ||
|
|
30afa43ed4 | ||
|
|
f2e66abe67 | ||
|
|
cdb311a501 | ||
|
|
4c2b3e157c | ||
|
|
52625b710b | ||
|
|
b411bb4e85 | ||
|
|
f16111369f | ||
|
|
628d052749 | ||
|
|
2e26bb0e57 | ||
|
|
652c7d72f6 | ||
|
|
0b046ea8cf | ||
|
|
7133f05fea | ||
|
|
da1c23d7dd | ||
|
|
a329dcd844 | ||
|
|
b57212d5f7 | ||
|
|
12c9e16f37 | ||
|
|
422381ac01 | ||
|
|
4b817cef65 | ||
|
|
4255a9916d | ||
|
|
b1566e904b | ||
|
|
8629076814 | ||
|
|
761e4f2718 | ||
|
|
3441dd4d2d | ||
|
|
9a8139a8b2 | ||
|
|
a0f823910c | ||
|
|
755338ded0 | ||
|
|
ecd85a53a1 | ||
|
|
8faa0d9e01 | ||
|
|
20ca2eac37 | ||
|
|
85b56247a3 | ||
|
|
febc66fb02 | ||
|
|
3f929aee3c | ||
|
|
8c46dd194d | ||
|
|
a9f99d05b9 | ||
|
|
e224c33817 | ||
|
|
4e6afbc8ff | ||
|
|
80b912ea14 | ||
|
|
85a571521b | ||
|
|
98efa9f466 | ||
|
|
a30a9ad763 | ||
|
|
36787eeeeb | ||
|
|
104cafedb0 | ||
|
|
284d307c0f | ||
|
|
f50c98e10d | ||
|
|
a4a5d63da7 | ||
|
|
9f0f9e0928 | ||
|
|
ef62fd7e34 | ||
|
|
0151dd8875 | ||
|
|
2026024c8a | ||
|
|
3d56baddd7 | ||
|
|
c10e7b2b41 | ||
|
|
ba605e985a | ||
|
|
4cd1cdf495 | ||
|
|
f9c644b964 | ||
|
|
da68bcf952 | ||
|
|
2c3b625b6b | ||
|
|
16ef259ae1 | ||
|
|
c56b3e75ea | ||
|
|
772866389f | ||
|
|
41f8b5aee2 | ||
|
|
75052fc773 | ||
|
|
e7a580fd3a | ||
|
|
c967230969 | ||
|
|
f3e6b2ab57 | ||
|
|
7385cb0c89 | ||
|
|
d1be9e7dc8 | ||
|
|
a22052e85b | ||
|
|
ab401541f2 | ||
|
|
dab4427c18 | ||
|
|
ad7d508a3c | ||
|
|
cb30514cbc | ||
|
|
53af9db285 | ||
|
|
b557a14134 | ||
|
|
8922fd6dde | ||
|
|
6b39e549f7 | ||
|
|
eb0dd827b1 | ||
|
|
a48902d7d3 | ||
|
|
7fca1f865d | ||
|
|
c5dc06962d | ||
|
|
45ecdafd52 | ||
|
|
f9e8bf92bd | ||
|
|
de614b293c | ||
|
|
8382075147 | ||
|
|
b605630a9c | ||
|
|
0a89fb1928 | ||
|
|
4c81906b50 | ||
|
|
b6220e985a | ||
|
|
7a97a28e00 | ||
|
|
dda8f281a1 | ||
|
|
28b561a810 | ||
|
|
44aba93d37 | ||
|
|
efb5157027 | ||
|
|
f48c1562f3 | ||
|
|
098b5b28a8 | ||
|
|
998e8e748c | ||
|
|
be96c77467 | ||
|
|
b5e8aec548 | ||
|
|
278c7dc836 | ||
|
|
1425ad87ab | ||
|
|
f985a2a8ad | ||
|
|
20f423a2fc | ||
|
|
210c1215d6 | ||
|
|
1c507393a9 | ||
|
|
b04cfdc091 | ||
|
|
42a3006f6e | ||
|
|
7d5c618246 | ||
|
|
74250982d1 | ||
|
|
8bd6fb5b4e | ||
|
|
bb2fb92b0e | ||
|
|
10b016312d | ||
|
|
b4500a2914 | ||
|
|
da9b1b3f58 | ||
|
|
da31520465 | ||
|
|
f3d3a0e57f | ||
|
|
9a655988dd | ||
|
|
43b541bb7d | ||
|
|
e32853744a | ||
|
|
b017b63745 | ||
|
|
29d6d824fc | ||
|
|
d6983a25b0 | ||
|
|
4cab6726b0 | ||
|
|
509b8f3a29 | ||
|
|
461c2f9055 | ||
|
|
c1d0f9cf1b | ||
|
|
019de08c64 | ||
|
|
30f88f99b7 | ||
|
|
04ee9894c9 | ||
|
|
46368f52f5 | ||
|
|
226e9084c9 | ||
|
|
caf5d2f788 | ||
|
|
591634f175 | ||
|
|
dfaf51a432 | ||
|
|
c2ec066c85 | ||
|
|
3e7c09bb44 | ||
|
|
28f66e30dd | ||
|
|
bac7f097a9 | ||
|
|
fecd29a967 | ||
|
|
c22eed9109 | ||
|
|
2c1ba623a4 | ||
|
|
561cdba7c9 | ||
|
|
e5e25012ff | ||
|
|
c5d653a51a | ||
|
|
c4620a0b0c | ||
|
|
dc969f80f3 | ||
|
|
a82f85f058 | ||
|
|
3ef85e711d | ||
|
|
29f7307f9d | ||
|
|
24eed71959 | ||
|
|
9341ccb643 | ||
|
|
4385b90cca | ||
|
|
9c10ee12fd | ||
|
|
7a622986d6 | ||
|
|
99b3cdd45d | ||
|
|
7b9d84f528 | ||
|
|
5f49827bd2 | ||
|
|
e1b4fe02d6 | ||
|
|
cdab127b98 | ||
|
|
bac7ea5dc0 | ||
|
|
b3b5581f2d | ||
|
|
9a26d869fe | ||
|
|
57138d9629 | ||
|
|
47d897b48d | ||
|
|
6b9f1d519e | ||
|
|
8b966c61d8 | ||
|
|
87e09efff5 | ||
|
|
65a0583c85 | ||
|
|
2000a1b334 | ||
|
|
cdd44af043 | ||
|
|
df99d06d11 | ||
|
|
b32d096be2 | ||
|
|
b9d5d1041c | ||
|
|
994dc9ff33 | ||
|
|
2ac97736ce | ||
|
|
925e106c43 | ||
|
|
59002413f8 | ||
|
|
32293ee3bb | ||
|
|
e04f35af4f | ||
|
|
c50f83400a | ||
|
|
31fa974379 | ||
|
|
aa8c362cb8 | ||
|
|
e48cc40610 | ||
|
|
a931f59dda | ||
|
|
4140fc4d28 | ||
|
|
f0d39e6081 | ||
|
|
c11872a229 | ||
|
|
f4fe730dc9 | ||
|
|
a4c83a08c7 | ||
|
|
c3277d24c3 | ||
|
|
28a23590ef | ||
|
|
162275992b | ||
|
|
08e25f4622 | ||
|
|
9a300d773f | ||
|
|
a5362f3170 | ||
|
|
7b653b90a8 | ||
|
|
f464b3ae7c | ||
|
|
d4b7064d2f | ||
|
|
2540b9969f | ||
|
|
4355b799cb | ||
|
|
a7fb434f9f | ||
|
|
b3465f3bbf | ||
|
|
4dfc3ff94c | ||
|
|
2670279b0d | ||
|
|
f4fd4245be | ||
|
|
200d239e0b | ||
|
|
41a0590182 | ||
|
|
63b2c49354 | ||
|
|
c99d7ce89d | ||
|
|
79e382e064 | ||
|
|
fa1d4dae0f | ||
|
|
35f84b4414 | ||
|
|
a882bdb08c | ||
|
|
b5c2e1c95b | ||
|
|
cfd6ea5a34 | ||
|
|
b3cf5e8698 | ||
|
|
2d07223995 | ||
|
|
32bb9fd4ec | ||
|
|
37b5f22c1a | ||
|
|
e0b00c09ce | ||
|
|
71c4dba34d | ||
|
|
9a3187e36c | ||
|
|
c714a3f91c | ||
|
|
5d4846cf60 | ||
|
|
543b327d09 | ||
|
|
1ccd2a3cc9 | ||
|
|
94398071eb | ||
|
|
143476c76c | ||
|
|
e85683e40f | ||
|
|
141162e5ae | ||
|
|
13acbfb64f | ||
|
|
a55e3d8578 | ||
|
|
2486ae6d3f | ||
|
|
ba991b47ea | ||
|
|
a096d0db37 | ||
|
|
e4a30c052e | ||
|
|
ccfe80ecfa | ||
|
|
df5b889f7d | ||
|
|
55f25c2613 | ||
|
|
34eaa822a0 | ||
|
|
5072f503c4 | ||
|
|
8e6b5a5e55 | ||
|
|
246647641a | ||
|
|
06a1bdbcba | ||
|
|
a0b75d7ada | ||
|
|
d36036a9fe | ||
|
|
a68d891770 | ||
|
|
f272a9ead2 | ||
|
|
8db2e10696 | ||
|
|
bea4f7da98 | ||
|
|
3cd74c06d6 | ||
|
|
053681e4a5 | ||
|
|
e3225f7123 | ||
|
|
ef62b0e746 | ||
|
|
7756d12993 | ||
|
|
3b92d12a44 | ||
|
|
4449bf3c75 | ||
|
|
0200ed1341 | ||
|
|
02bde900d6 | ||
|
|
36e299b4cf | ||
|
|
4e34f861c9 | ||
|
|
42df2fe242 | ||
|
|
7f11da8650 | ||
|
|
dff0fcb1fd | ||
|
|
c31d22f996 | ||
|
|
14ab2b52e9 | ||
|
|
e91f34f09b | ||
|
|
6c733f84e9 | ||
|
|
71c2cd5065 | ||
|
|
45f9a10bfb | ||
|
|
6ad203f767 | ||
|
|
5bf895eefe | ||
|
|
5a8f5b14c1 | ||
|
|
27ccb37646 | ||
|
|
b4c528f6b5 | ||
|
|
517a255d5a | ||
|
|
d76feeb320 | ||
|
|
e4f23b3fce | ||
|
|
02d8dc22e9 | ||
|
|
2fc3ec7224 | ||
|
|
a243ac7a01 | ||
|
|
6f1ee78623 | ||
|
|
6633370a34 | ||
|
|
a22e29532c | ||
|
|
96c8d1f68b | ||
|
|
4aa3bd56c4 | ||
|
|
7d1cf7975a | ||
|
|
59d08a92d0 | ||
|
|
33c1a002da | ||
|
|
8f04d3418a | ||
|
|
f5b556c6cb | ||
|
|
a18f3bf08a | ||
|
|
41a0a0c64c | ||
|
|
b4c8a317d0 | ||
|
|
30ac075555 | ||
|
|
fcb317ca40 | ||
|
|
9991f79326 | ||
|
|
7cf3f8c7cd | ||
|
|
eeef183cc6 | ||
|
|
96bc02c8cb | ||
|
|
af276a24ed | ||
|
|
a92dbae459 | ||
|
|
b0bfb5e443 | ||
|
|
671e6da104 | ||
|
|
485be533c9 | ||
|
|
53ff4cd651 | ||
|
|
a98c760d93 | ||
|
|
9b0c95e37d | ||
|
|
b419b8f642 | ||
|
|
c32f8d0998 | ||
|
|
c6d253cc25 | ||
|
|
c05cb62cdb | ||
|
|
ddd213dda3 | ||
|
|
8aab844a48 | ||
|
|
8fc1cbfc54 | ||
|
|
82ea330dda | ||
|
|
c63aa3033a | ||
|
|
7837738c17 | ||
|
|
bdb0ad4836 | ||
|
|
0dbf810cfe | ||
|
|
58aa4887ad | ||
|
|
3ff0916f49 | ||
|
|
cc9fb3ff41 | ||
|
|
2c51928b9d | ||
|
|
4451978af6 | ||
|
|
d4193c55ae | ||
|
|
ec3e365b3c | ||
|
|
c245c69389 | ||
|
|
9a9a7c9d7a | ||
|
|
20b516b5fa | ||
|
|
4dcfeca092 | ||
|
|
8ca8c73868 | ||
|
|
1a1c101b9a | ||
|
|
a0a0e2d324 | ||
|
|
d544199eff | ||
|
|
407ebeb699 | ||
|
|
9ba12b3e3f | ||
|
|
97976cbee9 | ||
|
|
cb5c772233 | ||
|
|
2bfbb824bf | ||
|
|
05189651ee | ||
|
|
0c80134523 | ||
|
|
6c686520c4 | ||
|
|
49ae029572 | ||
|
|
10e43203ed | ||
|
|
e25be5a522 | ||
|
|
47ff64a9e9 | ||
|
|
35f3887d79 | ||
|
|
dacfc8b00a | ||
|
|
0840ac7d06 | ||
|
|
f34b96f565 | ||
|
|
808fbed4f3 | ||
|
|
b48800049b | ||
|
|
7e5d4166d8 | ||
|
|
6b724ba8db | ||
|
|
25e1427a84 | ||
|
|
82944bc103 | ||
|
|
1e73b2437d | ||
|
|
574cd304e3 | ||
|
|
2edcd1f5b2 | ||
|
|
e144b26619 | ||
|
|
9ef9220689 | ||
|
|
23b6a8c495 | ||
|
|
4b9552fdf5 | ||
|
|
a249097170 | ||
|
|
6f4b2d7967 | ||
|
|
888d77c96c | ||
|
|
e2ae1ca866 | ||
|
|
7ef9a0f0bf | ||
|
|
885a0882f2 | ||
|
|
2ddeca2976 | ||
|
|
2085012874 | ||
|
|
ebb495ab8b | ||
|
|
905274d6d4 | ||
|
|
120d7edf12 | ||
|
|
12cd06cae5 | ||
|
|
42c26102d8 | ||
|
|
e906564f70 | ||
|
|
b8b2dc3b6d | ||
|
|
d69f76e68a | ||
|
|
ec1ba02cca | ||
|
|
49869d071a | ||
|
|
39275682d9 | ||
|
|
1dbc3a049b | ||
|
|
0a7b2683ff | ||
|
|
9714985855 | ||
|
|
d809b8eee0 | ||
|
|
52e2ed2aa7 | ||
|
|
ab3cc3d2b6 | ||
|
|
0a33324321 | ||
|
|
a948335cb9 | ||
|
|
6d2e5af6bc | ||
|
|
9a7c71394e | ||
|
|
1ebf184e9f | ||
|
|
48b013fa85 | ||
|
|
e96dee478d | ||
|
|
25d2a12a92 | ||
|
|
b0af31a71c | ||
|
|
9fdd766429 | ||
|
|
28ce23350d | ||
|
|
1abe890067 | ||
|
|
75fbedeeac | ||
|
|
a1861a2f84 | ||
|
|
0eecfb52ed | ||
|
|
ce1c28ffb9 | ||
|
|
9667b81c8d | ||
|
|
afac596a7b | ||
|
|
636692ba6a | ||
|
|
203f7e7ea6 | ||
|
|
8aee19b0ef | ||
|
|
49ed31c895 | ||
|
|
526075cd5e | ||
|
|
71340805ec | ||
|
|
654585aa68 | ||
|
|
441d3c1220 | ||
|
|
fc2d7258fb | ||
|
|
8195cedcf4 | ||
|
|
3e98477707 | ||
|
|
7b9479eca9 | ||
|
|
f15b67d386 | ||
|
|
f3141727d7 | ||
|
|
c7e4638736 | ||
|
|
d04981969f | ||
|
|
a148d4f446 | ||
|
|
a9e105f9eb | ||
|
|
c8afce2a3a | ||
|
|
ebf9d11f4d | ||
|
|
7a3bdf93e9 | ||
|
|
35d5c585ec | ||
|
|
45eb4f7201 | ||
|
|
e4c2585e23 | ||
|
|
a05f2fe3fb | ||
|
|
94cdd6829c | ||
|
|
34aa51e98a | ||
|
|
6e93c68a77 | ||
|
|
f16eec2ad2 | ||
|
|
657c1a4626 | ||
|
|
6bf6e943cc | ||
|
|
6252f903ee | ||
|
|
a1e827fb3b | ||
|
|
40218e02e1 | ||
|
|
0891b0d294 | ||
|
|
756c43c087 | ||
|
|
6488e66088 | ||
|
|
31ee3fb12b | ||
|
|
b80d45ad45 | ||
|
|
ea2a3c3b70 | ||
|
|
c0312159eb | ||
|
|
e124f582aa | ||
|
|
54150a8f2a | ||
|
|
385c221ef8 | ||
|
|
7df6e99e28 | ||
|
|
76a5f2f516 | ||
|
|
936c1f2ada | ||
|
|
5787802077 | ||
|
|
c902d25524 | ||
|
|
d152336f42 | ||
|
|
7ab0ae18be | ||
|
|
6189eddecf | ||
|
|
cc7471a9af | ||
|
|
eabaca361c | ||
|
|
141ca2498c | ||
|
|
964db2a779 | ||
|
|
f2867542b1 | ||
|
|
8f6e886821 | ||
|
|
dfca7bc2c4 | ||
|
|
f94b04bb19 | ||
|
|
be1e97c616 | ||
|
|
9790e9f843 | ||
|
|
e339d3857f | ||
|
|
0e4f21842c | ||
|
|
d4ea9a1ba3 | ||
|
|
5300fa929b | ||
|
|
8a5a66a06a | ||
|
|
43ed4427b4 | ||
|
|
22b0d92e6b | ||
|
|
7eb528232e | ||
|
|
33b6b85e72 | ||
|
|
8177119671 | ||
|
|
4b407a9dd2 | ||
|
|
9b904a7b4a | ||
|
|
e0424a1e38 | ||
|
|
00c08e05f5 | ||
|
|
0836d9c5ee | ||
|
|
ff5b6c4eab | ||
|
|
bb5cd49327 | ||
|
|
6f40ef6015 | ||
|
|
c7758506ac | ||
|
|
192ac6959a | ||
|
|
465a4643a8 | ||
|
|
1bf0065d9d | ||
|
|
c14efc843c | ||
|
|
0506dfba10 | ||
|
|
7e4f36e8b4 | ||
|
|
55dc482a61 | ||
|
|
05b9f9cae6 | ||
|
|
c79868de3e | ||
|
|
7eeac060ab | ||
|
|
c8b2b7b279 | ||
|
|
f4b3187494 |
No files matched your search
+144
@@ -0,0 +1,144 @@
|
||||
version: 1.0.{build}
|
||||
|
||||
environment:
|
||||
key_pass:
|
||||
secure: u6DydPcdrUJlxGL9uc7yQRYG8+5rY6aAEE9nfCSzFyNzZlX9NniOp8Uh5ZKQqX7bGEngLI6ipbLfiJvn0XFnhbn2iTkOuMqOXVJVOehvwlQ=
|
||||
pfx_key:
|
||||
secure: 1mwqyRy7hDqDjDK+TIAoaXyXzpNgwruFNA6TPkinUcVM7A+NLD33RQLnfnwVy+R5ovD2pUfhQ6+N0Fqebv6tZh436LIEsock+6IOdpgFwrg=
|
||||
|
||||
|
||||
notifications:
|
||||
- provider: Slack
|
||||
incoming_webhook:
|
||||
secure: LuxwG5OZnnA//gmSXzCKu8/FRqYjCgGfVFqajSsGHeQ1HQNp7rYNhQpsO8/3PK63xKJj3wzt86DJekf9q9Q5OcHa9AHXUQbEveX0psd7elw=
|
||||
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
- appveyor
|
||||
- windows-setup
|
||||
|
||||
clone_folder: c:\projects\lbry
|
||||
|
||||
init:
|
||||
- ps: >-
|
||||
$env:Path += ";C:\MinGW\bin\"
|
||||
|
||||
$env:Path += ";C:\Program Files (x86)\Windows Kits\10\bin\x86\"
|
||||
|
||||
gcc --version
|
||||
|
||||
mingw32-make --version
|
||||
|
||||
mkdir C:\temp
|
||||
|
||||
Invoke-WebRequest "https://pypi.python.org/packages/55/90/e987e28ed29b571f315afea7d317b6bf4a551e37386b344190cffec60e72/miniupnpc-1.9.tar.gz" -OutFile "C:\temp\miniupnpc-1.9.tar.gz"
|
||||
|
||||
cd C:\temp
|
||||
|
||||
7z e miniupnpc-1.9.tar.gz
|
||||
|
||||
7z x miniupnpc-1.9.tar
|
||||
|
||||
cd C:\temp\miniupnpc-1.9
|
||||
|
||||
mingw32-make.exe -f Makefile.mingw
|
||||
|
||||
C:\Python27\python.exe C:\temp\miniupnpc-1.9\setupmingw32.py build --compiler=mingw32
|
||||
|
||||
C:\Python27\python.exe C:\temp\miniupnpc-1.9\setupmingw32.py install
|
||||
|
||||
Invoke-WebRequest "https://github.com/lbryio/lbry/raw/master/packaging/windows/libs/gmpy-1.17-cp27-none-win32.whl" -OutFile "C:\temp\gmpy-1.17-cp27-none-win32.whl"
|
||||
|
||||
C:\Python27\Scripts\pip.exe install "C:\temp\gmpy-1.17-cp27-none-win32.whl"
|
||||
|
||||
C:\Python27\Scripts\pip.exe install pypiwin32==219
|
||||
|
||||
C:\Python27\Scripts\pip.exe install six==1.9.0
|
||||
|
||||
C:\Python27\Scripts\pip.exe install requests==2.9.1
|
||||
|
||||
C:\Python27\Scripts\pip.exe install zope.interface==4.1.3
|
||||
|
||||
C:\Python27\Scripts\pip.exe install cx-freeze==4.3.3
|
||||
|
||||
C:\Python27\Scripts\pip.exe install cython==0.24.1
|
||||
|
||||
C:\Python27\Scripts\pip.exe install Twisted==16.0.0
|
||||
|
||||
C:\Python27\Scripts\pip.exe install Yapsy==1.11.223
|
||||
|
||||
C:\Python27\Scripts\pip.exe install appdirs==1.4.0
|
||||
|
||||
C:\Python27\Scripts\pip.exe install argparse==1.2.1
|
||||
|
||||
C:\Python27\Scripts\pip.exe install colorama==0.3.7
|
||||
|
||||
C:\Python27\Scripts\pip.exe install dnspython==1.12.0
|
||||
|
||||
C:\Python27\Scripts\pip.exe install ecdsa==0.13
|
||||
|
||||
C:\Python27\Scripts\pip.exe install jsonrpc==1.2
|
||||
|
||||
C:\Python27\Scripts\pip.exe install jsonrpclib==0.1.7
|
||||
|
||||
C:\Python27\Scripts\pip.exe install loggly-python-handler==1.0.0
|
||||
|
||||
C:\Python27\Scripts\pip.exe install pbkdf2==1.3
|
||||
|
||||
C:\Python27\Scripts\pip.exe install protobuf==3.0.0
|
||||
|
||||
C:\Python27\Scripts\pip.exe install pycrypto==2.6.1
|
||||
|
||||
C:\Python27\Scripts\pip.exe install python-bitcoinrpc==0.1
|
||||
|
||||
C:\Python27\Scripts\pip.exe install qrcode==5.2.2
|
||||
|
||||
C:\Python27\Scripts\pip.exe install requests_futures==0.9.7
|
||||
|
||||
C:\Python27\Scripts\pip.exe install seccure==0.3.1.3
|
||||
|
||||
C:\Python27\Scripts\pip.exe install simplejson==3.8.2
|
||||
|
||||
C:\Python27\Scripts\pip.exe install slowaes==0.1a1
|
||||
|
||||
C:\Python27\Scripts\pip.exe install txJSON-RPC==0.3.1
|
||||
|
||||
C:\Python27\Scripts\pip.exe install unqlite==0.5.3
|
||||
|
||||
C:\Python27\Scripts\pip.exe install wsgiref==0.1.2
|
||||
|
||||
C:\Python27\Scripts\pip.exe install base58==0.2.2
|
||||
|
||||
C:\Python27\Scripts\pip.exe install googlefinance==0.7
|
||||
|
||||
C:\Python27\Scripts\pip.exe install git+https://github.com/lbryio/lbryum.git
|
||||
|
||||
cd C:\projects\lbry
|
||||
|
||||
install:
|
||||
- nuget install secure-file -ExcludeVersion
|
||||
- secure-file\tools\secure-file -decrypt packaging\windows\certs\lbry2.pfx.enc -secret %pfx_key%
|
||||
- cmd: C:\Python27\python.exe setup.py install
|
||||
|
||||
|
||||
build_script:
|
||||
- ps: C:\Python27\python.exe setup.py build bdist_msi
|
||||
- signtool.exe sign /f packaging\windows\certs\lbry2.pfx /p %key_pass% /tr http://tsa.starfieldtech.com /td SHA256 /fd SHA256 dist\*.msi
|
||||
|
||||
|
||||
test_script:
|
||||
- ps: >-
|
||||
C:\Python27\Scripts\pip.exe install mock
|
||||
|
||||
C:\Python27\Scripts\pip.exe install pylint
|
||||
|
||||
C:\Python27\python.exe C:\Python27\Scripts\trial.py C:\projects\lbry\tests
|
||||
|
||||
artifacts:
|
||||
- path: dist/*.msi
|
||||
name: msi
|
||||
- path: build/exe.win32-2.7/
|
||||
name: lbry-portable
|
||||
- path: packaging/windows/lbry-win32-app/LBRY-URI.reg
|
||||
name: LBRY-URI
|
||||
@@ -0,0 +1,9 @@
|
||||
[bumpversion]
|
||||
current_version = 0.4.7
|
||||
commit = True
|
||||
tag = True
|
||||
|
||||
[bumpversion:file:lbrynet/__init__.py]
|
||||
|
||||
[bumpversion:file:packaging/ubuntu/lbry.desktop]
|
||||
|
||||
+31
@@ -1 +1,32 @@
|
||||
*.pyc
|
||||
|
||||
*.xml
|
||||
|
||||
.idea/.name
|
||||
|
||||
*.iml
|
||||
|
||||
*.log
|
||||
|
||||
*.txt
|
||||
|
||||
lbrynet.egg-info/PKG-INFO
|
||||
|
||||
*.egg
|
||||
|
||||
/build
|
||||
|
||||
/dist
|
||||
|
||||
*.so
|
||||
|
||||
*.pem
|
||||
|
||||
*.decTest
|
||||
|
||||
.coverage
|
||||
|
||||
# temporary files from the twisted.trial test runner
|
||||
_trial_temp/
|
||||
|
||||
.DS_Store
|
||||
Whitespace-only changes.
@@ -0,0 +1,379 @@
|
||||
[MASTER]
|
||||
|
||||
# Specify a configuration file.
|
||||
#rcfile=
|
||||
|
||||
# Python code to execute, usually for sys.path manipulation such as
|
||||
# pygtk.require().
|
||||
#init-hook=
|
||||
|
||||
# Add files or directories to the blacklist. They should be base names, not
|
||||
# paths.
|
||||
ignore=CVS
|
||||
|
||||
# Pickle collected data for later comparisons.
|
||||
persistent=yes
|
||||
|
||||
# List of plugins (as comma separated values of python modules names) to load,
|
||||
# usually to register additional checkers.
|
||||
load-plugins=
|
||||
|
||||
# Use multiple processes to speed up Pylint.
|
||||
jobs=1
|
||||
|
||||
# Allow loading of arbitrary C extensions. Extensions are imported into the
|
||||
# active Python interpreter and may run arbitrary code.
|
||||
unsafe-load-any-extension=no
|
||||
|
||||
# A comma-separated list of package or module names from where C extensions may
|
||||
# be loaded. Extensions are loading into the active Python interpreter and may
|
||||
# run arbitrary code
|
||||
extension-pkg-whitelist=
|
||||
|
||||
# Allow optimization of some AST trees. This will activate a peephole AST
|
||||
# optimizer, which will apply various small optimizations. For instance, it can
|
||||
# be used to obtain the result of joining multiple strings with the addition
|
||||
# operator. Joining a lot of strings can lead to a maximum recursion error in
|
||||
# Pylint and this flag can prevent that. It has one side effect, the resulting
|
||||
# AST will be different than the one from reality.
|
||||
optimize-ast=no
|
||||
|
||||
|
||||
[MESSAGES CONTROL]
|
||||
|
||||
# Only show warnings with the listed confidence levels. Leave empty to show
|
||||
# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
|
||||
confidence=
|
||||
|
||||
# Enable the message, report, category or checker with the given id(s). You can
|
||||
# either give multiple identifier separated by comma (,) or put this option
|
||||
# multiple time (only on the command line, not in the configuration file where
|
||||
# it should appear only once). See also the "--disable" option for examples.
|
||||
#enable=
|
||||
|
||||
# Disable the message, report, category or checker with the given id(s). You
|
||||
# can either give multiple identifiers separated by comma (,) or put this
|
||||
# option multiple times (only on the command line, not in the configuration
|
||||
# file where it should appear only once).You can also use "--disable=all" to
|
||||
# disable everything first and then reenable specific checks. For example, if
|
||||
# you want to run only the similarities checker, you can use "--disable=all
|
||||
# --enable=similarities". If you want to run only the classes checker, but have
|
||||
# no Warning level messages displayed, use"--disable=all --enable=classes
|
||||
# --disable=W"
|
||||
disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating
|
||||
|
||||
|
||||
[REPORTS]
|
||||
|
||||
# Set the output format. Available formats are text, parseable, colorized, msvs
|
||||
# (visual studio) and html. You can also give a reporter class, eg
|
||||
# mypackage.mymodule.MyReporterClass.
|
||||
output-format=text
|
||||
|
||||
# Put messages in a separate file for each module / package specified on the
|
||||
# command line instead of printing them on stdout. Reports (if any) will be
|
||||
# written in a file name "pylint_global.[txt|html]".
|
||||
files-output=no
|
||||
|
||||
# Tells whether to display a full report or only the messages
|
||||
reports=yes
|
||||
|
||||
# Python expression which should return a note less than 10 (10 is the highest
|
||||
# note). You have access to the variables errors warning, statement which
|
||||
# respectively contain the number of errors / warnings messages and the total
|
||||
# number of statements analyzed. This is used by the global evaluation report
|
||||
# (RP0004).
|
||||
evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
|
||||
|
||||
# Template used to display messages. This is a python new-style format string
|
||||
# used to format the message information. See doc for all details
|
||||
#msg-template=
|
||||
|
||||
|
||||
[VARIABLES]
|
||||
|
||||
# Tells whether we should check for unused import in __init__ files.
|
||||
init-import=no
|
||||
|
||||
# A regular expression matching the name of dummy variables (i.e. expectedly
|
||||
# not used).
|
||||
dummy-variables-rgx=_$|dummy
|
||||
|
||||
# List of additional names supposed to be defined in builtins. Remember that
|
||||
# you should avoid to define new builtins when possible.
|
||||
additional-builtins=
|
||||
|
||||
# List of strings which can identify a callback function by name. A callback
|
||||
# name must start or end with one of those strings.
|
||||
callbacks=cb_,_cb
|
||||
|
||||
|
||||
[LOGGING]
|
||||
|
||||
# Logging modules to check that the string format arguments are in logging
|
||||
# function parameter format
|
||||
logging-modules=logging
|
||||
|
||||
|
||||
[BASIC]
|
||||
|
||||
# List of builtins function names that should not be used, separated by a comma
|
||||
bad-functions=map,filter,input
|
||||
|
||||
# Good variable names which should always be accepted, separated by a comma
|
||||
good-names=i,j,k,ex,Run,_
|
||||
|
||||
# Bad variable names which should always be refused, separated by a comma
|
||||
bad-names=foo,bar,baz,toto,tutu,tata
|
||||
|
||||
# Colon-delimited sets of names that determine each other's naming style when
|
||||
# the name regexes allow several styles.
|
||||
name-group=
|
||||
|
||||
# Include a hint for the correct naming format with invalid-name
|
||||
include-naming-hint=no
|
||||
|
||||
# Regular expression matching correct function names
|
||||
function-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for function names
|
||||
function-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression matching correct variable names
|
||||
variable-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for variable names
|
||||
variable-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression matching correct constant names
|
||||
const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
|
||||
|
||||
# Naming hint for constant names
|
||||
const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
|
||||
|
||||
# Regular expression matching correct attribute names
|
||||
attr-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for attribute names
|
||||
attr-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression matching correct argument names
|
||||
argument-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for argument names
|
||||
argument-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression matching correct class attribute names
|
||||
class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
|
||||
|
||||
# Naming hint for class attribute names
|
||||
class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
|
||||
|
||||
# Regular expression matching correct inline iteration names
|
||||
inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
|
||||
|
||||
# Naming hint for inline iteration names
|
||||
inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
|
||||
|
||||
# Regular expression matching correct class names
|
||||
class-rgx=[A-Z_][a-zA-Z0-9]+$
|
||||
|
||||
# Naming hint for class names
|
||||
class-name-hint=[A-Z_][a-zA-Z0-9]+$
|
||||
|
||||
# Regular expression matching correct module names
|
||||
module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
|
||||
|
||||
# Naming hint for module names
|
||||
module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
|
||||
|
||||
# Regular expression matching correct method names
|
||||
method-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for method names
|
||||
method-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression which should only match function or class names that do
|
||||
# not require a docstring.
|
||||
no-docstring-rgx=^_
|
||||
|
||||
# Minimum line length for functions/classes that require docstrings, shorter
|
||||
# ones are exempt.
|
||||
docstring-min-length=-1
|
||||
|
||||
|
||||
[ELIF]
|
||||
|
||||
# Maximum number of nested blocks for function / method body
|
||||
max-nested-blocks=5
|
||||
|
||||
|
||||
[SPELLING]
|
||||
|
||||
# Spelling dictionary name. Available dictionaries: none. To make it working
|
||||
# install python-enchant package.
|
||||
spelling-dict=
|
||||
|
||||
# List of comma separated words that should not be checked.
|
||||
spelling-ignore-words=
|
||||
|
||||
# A path to a file that contains private dictionary; one word per line.
|
||||
spelling-private-dict-file=
|
||||
|
||||
# Tells whether to store unknown words to indicated private dictionary in
|
||||
# --spelling-private-dict-file option instead of raising a message.
|
||||
spelling-store-unknown-words=no
|
||||
|
||||
|
||||
[FORMAT]
|
||||
|
||||
# Maximum number of characters on a single line.
|
||||
max-line-length=100
|
||||
|
||||
# Regexp for a line that is allowed to be longer than the limit.
|
||||
ignore-long-lines=^\s*(# )?<?https?://\S+>?$
|
||||
|
||||
# Allow the body of an if to be on the same line as the test if there is no
|
||||
# else.
|
||||
single-line-if-stmt=no
|
||||
|
||||
# List of optional constructs for which whitespace checking is disabled. `dict-
|
||||
# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}.
|
||||
# `trailing-comma` allows a space between comma and closing bracket: (a, ).
|
||||
# `empty-line` allows space-only lines.
|
||||
no-space-check=trailing-comma,dict-separator
|
||||
|
||||
# Maximum number of lines in a module
|
||||
max-module-lines=1000
|
||||
|
||||
# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
|
||||
# tab).
|
||||
indent-string=' '
|
||||
|
||||
# Number of spaces of indent required inside a hanging or continued line.
|
||||
indent-after-paren=4
|
||||
|
||||
# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
|
||||
expected-line-ending-format=
|
||||
|
||||
|
||||
[MISCELLANEOUS]
|
||||
|
||||
# List of note tags to take in consideration, separated by a comma.
|
||||
notes=FIXME,XXX,TODO
|
||||
|
||||
|
||||
[SIMILARITIES]
|
||||
|
||||
# Minimum lines number of a similarity.
|
||||
min-similarity-lines=4
|
||||
|
||||
# Ignore comments when computing similarities.
|
||||
ignore-comments=yes
|
||||
|
||||
# Ignore docstrings when computing similarities.
|
||||
ignore-docstrings=yes
|
||||
|
||||
# Ignore imports when computing similarities.
|
||||
ignore-imports=no
|
||||
|
||||
|
||||
[TYPECHECK]
|
||||
|
||||
# Tells whether missing members accessed in mixin class should be ignored. A
|
||||
# mixin class is detected if its name ends with "mixin" (case insensitive).
|
||||
ignore-mixin-members=yes
|
||||
|
||||
# List of module names for which member attributes should not be checked
|
||||
# (useful for modules/projects where namespaces are manipulated during runtime
|
||||
# and thus existing member attributes cannot be deduced by static analysis. It
|
||||
# supports qualified module names, as well as Unix pattern matching.
|
||||
ignored-modules=twisted.internet.reactor,leveldb
|
||||
|
||||
# List of classes names for which member attributes should not be checked
|
||||
# (useful for classes with attributes dynamically set). This supports can work
|
||||
# with qualified names.
|
||||
ignored-classes=twisted.internet,RequestMessage
|
||||
|
||||
# List of members which are set dynamically and missed by pylint inference
|
||||
# system, and so shouldn't trigger E1101 when accessed. Python regular
|
||||
# expressions are accepted.
|
||||
generated-members=lbrynet.lbrynet_daemon.LBRYDaemon.Parameters
|
||||
|
||||
|
||||
[IMPORTS]
|
||||
|
||||
# Deprecated modules which should not be used, separated by a comma
|
||||
deprecated-modules=regsub,TERMIOS,Bastion,rexec
|
||||
|
||||
# Create a graph of every (i.e. internal and external) dependencies in the
|
||||
# given file (report RP0402 must not be disabled)
|
||||
import-graph=
|
||||
|
||||
# Create a graph of external dependencies in the given file (report RP0402 must
|
||||
# not be disabled)
|
||||
ext-import-graph=
|
||||
|
||||
# Create a graph of internal dependencies in the given file (report RP0402 must
|
||||
# not be disabled)
|
||||
int-import-graph=
|
||||
|
||||
|
||||
[DESIGN]
|
||||
|
||||
# Maximum number of arguments for function / method
|
||||
max-args=5
|
||||
|
||||
# Argument names that match this expression will be ignored. Default to name
|
||||
# with leading underscore
|
||||
ignored-argument-names=_.*
|
||||
|
||||
# Maximum number of locals for function / method body
|
||||
max-locals=15
|
||||
|
||||
# Maximum number of return / yield for function / method body
|
||||
max-returns=6
|
||||
|
||||
# Maximum number of branch for function / method body
|
||||
max-branches=12
|
||||
|
||||
# Maximum number of statements in function / method body
|
||||
max-statements=50
|
||||
|
||||
# Maximum number of parents for a class (see R0901).
|
||||
max-parents=7
|
||||
|
||||
# Maximum number of attributes for a class (see R0902).
|
||||
max-attributes=7
|
||||
|
||||
# Minimum number of public methods for a class (see R0903).
|
||||
min-public-methods=2
|
||||
|
||||
# Maximum number of public methods for a class (see R0904).
|
||||
max-public-methods=20
|
||||
|
||||
# Maximum number of boolean expressions in a if statement
|
||||
max-bool-expr=5
|
||||
|
||||
|
||||
[CLASSES]
|
||||
|
||||
# List of method names used to declare (i.e. assign) instance attributes.
|
||||
defining-attr-methods=__init__,__new__,setUp
|
||||
|
||||
# List of valid names for the first argument in a class method.
|
||||
valid-classmethod-first-arg=cls
|
||||
|
||||
# List of valid names for the first argument in a metaclass class method.
|
||||
valid-metaclass-classmethod-first-arg=mcs
|
||||
|
||||
# List of member names, which should be excluded from the protected access
|
||||
# warning.
|
||||
exclude-protected=_asdict,_fields,_replace,_source,_make
|
||||
|
||||
|
||||
[EXCEPTIONS]
|
||||
|
||||
# Exceptions that will emit a warning when being caught. Defaults to
|
||||
# "Exception"
|
||||
overgeneral-exceptions=Exception
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
matrix:
|
||||
include:
|
||||
- os: linux
|
||||
sudo: required
|
||||
dist: trusty
|
||||
# dh-virtualenv requires that we use the same python interpreter
|
||||
# that comes with the system, so we don't want to use anything that
|
||||
# travis would try to set-up for us in python
|
||||
language: generic
|
||||
- os: osx
|
||||
# Use generic language for osx
|
||||
# python 2.7 is broken on osx on travis, so follow we have to specify the installation ourselves
|
||||
# https://github.com/travis-ci/travis-ci/issues/2312#issuecomment-195620855
|
||||
language: generic
|
||||
osx_image: xcode7.3
|
||||
|
||||
notifications:
|
||||
slack:
|
||||
secure: "Am13HPtpgCMljh0MDVuoFHvQXB8yhf4Kvf/qAeSp5N0vsHGL70CSF9Ahccw8dVPE6mbuak1OGtSUb6/UaErLHkpz3ztaRLkDa9x7CmBB3Kynnh8oO2VbB7b/2ROULqkhF4VZmAnNfwrQrbC3gs8Sybp261Nyc7y4ww15xDYBrk2fyq4ds2DCaJdRxfJUJFonrZ6KXr3fVaXosO6cjuyS8eRodcmrqsT4cCtinjNTD1hGWoH107E4ObSmpVelxQO193KhNJMRiLlEcVkvYUOqIWBtwdGHbNE/6Yeuq1TXgKJ0KeJWAmW3wTfUYNngGXNAsyCnrhul5TKNevNzfIAQZHvRsczYiWPJV6LtohHT0CcUiCXJtvEPOyahEBfwK3etY/xxFqny7N9OEmpdW2sgsEPNPX2LJynJti2rQA9SuAD1ogR3ZpDy/NXoaAZf8PTdPcuNUMULV9PGG7tVrLBecO/W1qO6hdFxwlLdgqGLxAENZgGp++v/DhPk/WvtmHj7iTbRq0nxaTWyX5uKOn2ADH+k/yfutjv6BsQU9xNyPeZEEtuEpc6X6waiYn/8G9vl9PecvKC5H0MgsZ6asAxmg7mZ3VSMFG7mo8ENeOhSZ0Oz6ZTBILL3wFccZA9uJIq7NWmqC9dRiGiuKXBB62No7sINoHg3114e2xYa9qvNmGg="
|
||||
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/.cache/pip
|
||||
- $HOME/Library/Caches/pip
|
||||
- $TRAVIS_BUILD_DIR/cache/wheel
|
||||
|
||||
before_install:
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ./packaging/travis/setup_osx.sh; fi
|
||||
|
||||
install:
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./packaging/travis/install_dependencies_and_run_tests.sh; fi
|
||||
|
||||
before_script:
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then openssl aes-256-cbc -k "$ENCRYPTION_SECRET" -in packaging/osx/certs/dist.cer.enc -d -a -out packaging/osx/certs/dist.cer; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then openssl aes-256-cbc -k "$ENCRYPTION_SECRET" -in packaging/osx/certs/dist.p12.enc -d -a -out packaging/osx/certs/dist.p12; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ./packaging/osx/add-key.sh; fi
|
||||
|
||||
script:
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then bash packaging/ubuntu/ubuntu_package_setup.sh -t; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then cd packaging/osx/lbry-osx-app && ./setup_app.sh && cd $TRAVIS_BUILD_DIR; fi
|
||||
# fail the build if this is a build for a tag and we don't have the versions matching; allow tags that start with 'test' to pass
|
||||
- if [[ -n "${TRAVIS_TAG}" ]]; then if [[ "${TRAVIS_TAG}" == test* ]] || [[ "v`python setup.py -V`" = "${TRAVIS_TAG}" ]]; then true; else false; fi; fi
|
||||
|
||||
deploy:
|
||||
- provider: releases
|
||||
file: "${TRAVIS_BUILD_DIR}/`python setup.py --name`_`python setup.py -V`_amd64.deb"
|
||||
skip_cleanup: true
|
||||
prerelease: true
|
||||
on:
|
||||
tags: true
|
||||
condition: "$TRAVIS_OS_NAME = linux"
|
||||
# this is the oauth token for the lbry-ci user
|
||||
api_key:
|
||||
secure: nKdWGROnLNodx9k9nWvq2wezkPvSVL8zF63qjPXjhdOe9kCUbcLp7WnFDYpn9EJj4Pofq/ejeCHwjA+5x7JUP3Szk7SlJV61B4c/5hl64rl7oSKOoKskjdE2jaOG3CJuOUrh0yQ59U3vMMABcsnw/wJaCIuu/erdPIm8g8R+stu1YHOGtl5Y9WiW+zLJn2vc3GooV1TWtki9EnrmFfw0Vrqc4RMVMFB1ojE7ggrK1LIwcmGSbLIYzker1ZRz8SCy+84sGk4//V+2i2NNiz5AkPuG7BBGrU2twE9nD23IlruJAdVdi71P3ytAmi0kKyvxIU4VeNaqyTk9zeL5IB9J5IIgvekHgKcsKhFUZ6QcXT1Xfxl4ELftvWCTHWiewnXFdqLcG9GZiUaE6+7wdalwDAP3tqS2emiibetlBZERHR+RMR00ej+1MBYWGMlTse/0Tglndv0a2qqgAJYLKPRT02hTRYGxZ1MrJe+WGnChRmzwgLVTIgZuiDciFOahN0TYGSORk6OpnZBsxvpzSqDw5UDJx0BmbJ1xMNDFbOs8ubZ9yIpB9yNMGw66FPacOF61XNYnmA68ILC28UtOFKuuHLrUPbM5JmQkDVhtTfFbBnyHefyCLAL4MHvJJKGi1oaOXjYaJ/J095h636/kQ0cHHuVMgoWUQZOQ44xRAz7tMuc=
|
||||
- provider: releases
|
||||
file: "${TRAVIS_BUILD_DIR}/packaging/osx/lbry-osx-app/`python setup.py --name`.`python setup.py -V`.dmg"
|
||||
skip_cleanup: true
|
||||
prerelease: true
|
||||
on:
|
||||
tags: true
|
||||
condition: "$TRAVIS_OS_NAME = osx"
|
||||
# this is the oauth token for the lbry-ci user
|
||||
api_key:
|
||||
secure: nKdWGROnLNodx9k9nWvq2wezkPvSVL8zF63qjPXjhdOe9kCUbcLp7WnFDYpn9EJj4Pofq/ejeCHwjA+5x7JUP3Szk7SlJV61B4c/5hl64rl7oSKOoKskjdE2jaOG3CJuOUrh0yQ59U3vMMABcsnw/wJaCIuu/erdPIm8g8R+stu1YHOGtl5Y9WiW+zLJn2vc3GooV1TWtki9EnrmFfw0Vrqc4RMVMFB1ojE7ggrK1LIwcmGSbLIYzker1ZRz8SCy+84sGk4//V+2i2NNiz5AkPuG7BBGrU2twE9nD23IlruJAdVdi71P3ytAmi0kKyvxIU4VeNaqyTk9zeL5IB9J5IIgvekHgKcsKhFUZ6QcXT1Xfxl4ELftvWCTHWiewnXFdqLcG9GZiUaE6+7wdalwDAP3tqS2emiibetlBZERHR+RMR00ej+1MBYWGMlTse/0Tglndv0a2qqgAJYLKPRT02hTRYGxZ1MrJe+WGnChRmzwgLVTIgZuiDciFOahN0TYGSORk6OpnZBsxvpzSqDw5UDJx0BmbJ1xMNDFbOs8ubZ9yIpB9yNMGw66FPacOF61XNYnmA68ILC28UtOFKuuHLrUPbM5JmQkDVhtTfFbBnyHefyCLAL4MHvJJKGi1oaOXjYaJ/J095h636/kQ0cHHuVMgoWUQZOQ44xRAz7tMuc=
|
||||
|
||||
env:
|
||||
global:
|
||||
# needed to unlock the identity file when its on the keychain
|
||||
- secure: "aqgVNHfeh6JUqTTWG5+W+tTsIJkePS6HyLkcZlq6ODYrfdGKa90EeV4q4wdGfP5IkrBbf+WBFACDGVp574E7vfMLMNKUDuUtgnGVPwqOIhjG82ud8Xa7qF7lsw50QOnRYYVd2GLlCIzk8sffT5ncjSPN2ClNVM5iTwCkC8TNSAVnEJxu6bG6hcaT4uCWWjs0m+39O1xJwxJ4vTjpE/gy+j2FTSaUR5cNavOVCyJqpeKlga9aoBVsQHWvxlYurdWbwRLwVIV7bDE+sYPnwn1nMFQpx5RZ1AX1Z2UxNFKcYzLJgcWe85OjxLyT4udX+XZ9SLsdOjm1n201OLEKsmTxmHS8yqpbu6+pKQ1rLMEVpgPGfS8DdtqZyb3z0u6q4jztpm+uBe8hnFgwQFGXO63nOQsI0n1PMR3evAnVx7jYt7y/UALs2A9yjosMwDqgql1FMhyd0OJGo8Ky8YpDsr2J5zGVvIqt2/N+lP+SOe2D1J/5EhGlY0o4tqIAFskh3q8/GR6UGm70KT2l4LJqrU1WGrGxPJ+HoOEmvG6eqLesk03fAX3v5+DgXZWErnzXMIOGDPaFVpmx+G9VZTIlmf+3Wbu9TnZE5PRwFTdP1rqjGpjUhHeF0VTc2qgNq1OfL98CBO1wLgA195+em58cELalnwDMWUTmY1Jt1kUuAtCc11U="
|
||||
# used to decypt the aes decrypt the certificate and identity files
|
||||
- secure: "FzftiEBFMngQIci5KZ+tpKs/BubalhWXJ9f8yogGNVa3XplLheUXYgcsNU8sYT2MJaKAHN9fSjHWb67UtKT7yXddXxeOPnW6wJ3ua/FXnpynsmeGKTfh3stvgjpzdXous67uHmCWMMklfb6z7UuDohjUMAe5n5HKw5tq1RzTKpc2kJacOC6qUT5laUOvULyCaO9E9HmbHeR5ZeXAC6pnzX2ccsSrcXvPozHzBIZ9RyothKs+CZw8PEuJo07RRL8meboegqYOUrOYuz6A2gS2mZJoy59ivZKOFxS5shEuv2Jt80RyfyxBoUpKFq8OG3Am5nAEzDiTIGzmIoKDEGKeTagk/sEtSZXiMDkzDT4GX/j1rUNLCBU87bXEFS2zfRsrfg8c1XZPIzDYBT1PY2QtLBdddF5zzDoKdPLJ3sjN+fZFE5RlnwfwnMHriVRZZlzjcdk0Z06gKTBCxUg5BZamnOOK+K8qunMJXVS+Vmi5u4RoTZZiCosUlYKnSKJ8suO9C0+znxoViusPqP4ONprNHgDoZw+UKio84QW3PrZv9h4zH/D+msDgJRZ0ceqDD+6Wz1J8Mm5ptW1GOLh/IU12TPXjxteqh0Um2vv5eIPmjK9uEK666kK7PqtPDkYhAfWvF+nmOOyPMJfbP4MW/i9WHNF4ghsIMbPKfqNhgSmfrYw="
|
||||
@@ -0,0 +1,85 @@
|
||||
#### Getting LBRY for development
|
||||
|
||||
Q: How do I get lbry for command line?
|
||||
|
||||
A: In order to run lbry from command line, you need more than the packaged app/deb.
|
||||
|
||||
######On OS X
|
||||
|
||||
You can install LBRY command line by running `curl -sL https://rawgit.com/lbryio/lbry-setup/master/lbry_setup_osx.sh | sudo bash` in a terminal. This script will install lbrynet and its dependancies, as well as the app.
|
||||
|
||||
######On Linux
|
||||
|
||||
On Ubuntu or Mint you can install the prerequisites and lbrynet by running
|
||||
|
||||
sudo apt-get install libgmp3-dev build-essential python2.7 python2.7-dev python-pip git
|
||||
git clone https://github.com/lbryio/lbry.git
|
||||
cd lbry
|
||||
sudo python setup.py install
|
||||
|
||||
#### Using LBRY
|
||||
|
||||
Q: How do I run lbry from command line?
|
||||
|
||||
A: The command is `lbrynet-daemon`
|
||||
|
||||
***********
|
||||
|
||||
Q: How do I stop lbry from the command line?
|
||||
|
||||
A: You can ctrl-c or run `stop-lbrynet-daemon`
|
||||
|
||||
***********
|
||||
|
||||
Q: How do I run lbry with lbrycrdd (the blockchain node application)?
|
||||
|
||||
A: Start lbry with the --wallet flag set: `lbrynet-daemon --wallet=lbrycrd`
|
||||
|
||||
Note: when you change the wallet it is persistant until you specify you want to use another wallet - lbryum - with the --wallet flag again.
|
||||
|
||||
***********
|
||||
|
||||
Q: Where are all the behind the scenes files?
|
||||
|
||||
A: On linux, the relevant directories are `~/.lbrynet`, `~/.lbrycrd`, and `~/.lbryum`, depending on which wallets you've used. On OS X, the folders of interest are `~/Library/Application Support/LBRY`, `~/.lbrycrd` and `~/.lbryum`, also depending on which wallets you've used.
|
||||
|
||||
***********
|
||||
|
||||
Q: How can I see the log in the console?
|
||||
|
||||
A: Run lbry with the --log-to-console flag set: `lbrynet-daemon --log-to-console`
|
||||
|
||||
***********
|
||||
|
||||
Q: How do I specify a web-UI to use?
|
||||
|
||||
A: If the files for the UI you'd like to use are storred locally on your computer, start lbry with the --ui flag: `lbrynet-daemon --ui=/full/path/to/ui/files/root/folder`
|
||||
|
||||
Note, once set with the UI flag the given UI will be cached by lbry and used as the default going forward. Also, it will only successfully load a UI if it contains a conforming requirements.txt file to specify required lbrynet and lbryum versions. [Here](https://github.com/lbryio/lbry-web-ui/blob/master/dist/requirements.txt) is an example requirements.txt file.
|
||||
|
||||
To reset your ui to pull from lbryio, or to try a UI still in development, run lbry with the --branch flag: `lbrynet=daemon --branch=master`
|
||||
|
||||
***********
|
||||
|
||||
Q: How do I see the list of API functions I can call, and how do I call them?
|
||||
|
||||
A: Here is an example script to get the documentation for the various API calls. To use any of the functions displayed, just provide any specified arguments in a dictionary.
|
||||
|
||||
Note: the lbry api can only be used while either the app or lbrynet-daemon command line are running
|
||||
|
||||
import sys
|
||||
from jsonrpc.proxy import JSONRPCProxy
|
||||
|
||||
try:
|
||||
from lbrynet.conf import API_CONNECTION_STRING
|
||||
except:
|
||||
print "You don't have lbrynet installed!"
|
||||
sys.exit(0)
|
||||
|
||||
api = JSONRPCProxy.from_url(API_CONNECTION_STRING)
|
||||
if not api.is_running():
|
||||
print api.daemon_status()
|
||||
else:
|
||||
for func in api.help():
|
||||
print "%s:\n%s" % (func, api.help({'function': func}))
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
Prerequisites
|
||||
-------------
|
||||
|
||||
To use the LBRYWallet, which enables spending and accepting LBRYcrds in exchange for data, the
|
||||
LBRYcrd application (insert link to LBRYcrd website here) must be installed and running. If
|
||||
this is not desired, the testing client can be used to simulate trading points, which is
|
||||
built into LBRYnet.
|
||||
|
||||
on Ubuntu:
|
||||
|
||||
sudo apt-get install libgmp3-dev build-essential python-dev python-pip
|
||||
|
||||
Getting the source
|
||||
------------------
|
||||
|
||||
Don't you already have it?
|
||||
|
||||
Setting up the environment
|
||||
--------------------------
|
||||
|
||||
It's recommended that you use a virtualenv
|
||||
|
||||
sudo apt-get install python-virtualenv
|
||||
cd <source base directory>
|
||||
virtualenv .
|
||||
source bin/activate
|
||||
|
||||
(to deactivate the virtualenv, enter 'deactivate')
|
||||
|
||||
python setup.py install
|
||||
|
||||
this will install all of the libraries and a few applications
|
||||
|
||||
For running the file sharing application, see RUNNING
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
#### Installing the LBRY app
|
||||
--------------------------
|
||||
|
||||
Installing LBRY is simple. You can get a dmg installer for OS X or a .deb for linux [here](https://github.com/lbryio/lbry/releases/latest).
|
||||
|
||||
##### OS X
|
||||
Just drag and drop LBRY.app into your applications folder (replacing any older versions). When it's running you'll have a LBRY icon in your status bar and the browser will open to the UI.
|
||||
|
||||
##### Linux
|
||||
Double click the .deb file and follow the prompts. The app can be started by searching "LBRY", and it can be turned off by clicking the red 'x' in the browser interface.
|
||||
|
||||
On both systems you can also open the UI while the app is running by going to lbry://lbry in Firefox or Safari, or localhost:5279 in Chrome.
|
||||
|
||||
|
||||
|
||||
#### Installing LBRY command line
|
||||
--------------------------
|
||||
|
||||
##### OS X
|
||||
You can install LBRY command line by running `curl -sL https://raw.githubusercontent.com/lbryio/lbry/master/packaging/osx/install_lbry_source.sh | sudo bash` in a terminal. This script will install lbrynet and its dependencies. You can start LBRY by running `lbrynet-daemon` from a terminal.
|
||||
|
||||
##### Linux
|
||||
On Ubuntu or Mint you can install the prerequisites and lbrynet by running
|
||||
|
||||
```
|
||||
sudo apt-get install libgmp3-dev build-essential python2.7 python2.7-dev python-pip git
|
||||
git clone https://github.com/lbryio/lbry.git
|
||||
cd lbry
|
||||
sudo python setup.py install
|
||||
```
|
||||
|
||||
To start LBRY, run `lbrynet-daemon` in a terminal.
|
||||
|
||||
#### On windows:
|
||||
|
||||
Install [mingw32](http://www.mingw.org/) base and c++ compiler.
|
||||
|
||||
Add C:\MinGW\bin to the windows PATH.
|
||||
|
||||
Enable distutils to compile with mingw32 by creating a distutils.cfg file in *PYTHONPATH\Lib\distutils* containing
|
||||
```
|
||||
[build]
|
||||
compiler = mingw32
|
||||
```
|
||||
|
||||
If using virtualenv, copy the *PYTHONPATH\Lib\distutils* directory to the virtualenv.
|
||||
|
||||
It's recommended to use [Unofficial Windows Binaries for Python Extension Packages](http://www.lfd.uci.edu/~gohlke/pythonlibs/) for as many of the required packages as possible.
|
||||
Currently, available binaries include:
|
||||
- Cython
|
||||
- Twisted
|
||||
- Zope.interface
|
||||
- pywin32
|
||||
- Yapsy
|
||||
- cx_Freeze
|
||||
- requests
|
||||
- gmpy
|
||||
|
||||
Install the each of the preceding binaries with `pip install *.whl`
|
||||
|
||||
Install pywin32 system files by run `python.exe Scripts\pywin32_postinstall.py -install` from an elevated command prompt.
|
||||
|
||||
Install the rest of the required modules with the standard `pip install module` command
|
||||
|
||||
The one module which requires additional work is [miniupnpc](https://pypi.python.org/pypi/miniupnpc/1.9).
|
||||
Download the source and compile with MinGW by running `mingw32make.bat`
|
||||
Then install the module by running `python setupmingw32.py install`
|
||||
@@ -1,22 +1,15 @@
|
||||
Copyright (c) 2015, LBRY, Inc.
|
||||
All rights reserved.
|
||||
The MIT License (MIT)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
Copyright (c) 2015-2016 LBRY Inc
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,
|
||||
distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
|
||||
following conditions:
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
||||
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
@@ -0,0 +1 @@
|
||||
include lbrynet/lbrynet_console/plugins/blindrepeater.yapsy-plugin
|
||||
+39
-24
@@ -1,12 +1,16 @@
|
||||
LBRYnet
|
||||
=======
|
||||
[](https://travis-ci.org/lbryio/lbry)
|
||||
|
||||
# LBRYnet
|
||||
|
||||
LBRYnet is a fully decentralized network for distributing data. It consists of peers uploading
|
||||
and downloading data from other peers, possibly in exchange for payments, and a distributed hash
|
||||
table, used by peers to discover other peers.
|
||||
|
||||
Overview
|
||||
--------
|
||||
## Installation
|
||||
|
||||
Download the [latest release](https://github.com/lbryio/lbry/releases/latest) or see [INSTALL.md](INSTALL.md) for manual installation.
|
||||
|
||||
## Overview
|
||||
|
||||
On LBRYnet, data is broken into chunks, and each chunk is specified by its sha384 hash sum. This
|
||||
guarantees that peers can verify the correctness of each chunk without having to know anything
|
||||
@@ -20,38 +24,49 @@ help peers find each other. For example, an application for which clients don't
|
||||
necessary chunks may use some identifier, chosen by the application, to find clients which do
|
||||
know all of the necessary chunks.
|
||||
|
||||
Running
|
||||
-------
|
||||
## For Developers
|
||||
|
||||
LBRYnet comes with an file sharing application, called 'lbrynet-console', which breaks
|
||||
The bundled LBRY application uses the lbrynet JSONRPC api found in `lbrynet.lbrynet_daemon.LBRYDaemon`. This api allows for applications and web services like the lbry browser UI to interact with lbrynet. If you've installed lbrynet, you can run `lbrynet-daemon` without running the app. While the app or `lbrynet-daemon` is running, you can use the following to show the help for all the available commands:
|
||||
|
||||
```
|
||||
from jsonrpc.proxy import JSONRPCProxy
|
||||
|
||||
try:
|
||||
from lbrynet.conf import API_CONNECTION_STRING
|
||||
except:
|
||||
print "You don't have lbrynet installed!"
|
||||
API_CONNECTION_STRING = "http://localhost:5279/lbryapi"
|
||||
|
||||
api = JSONRPCProxy.from_url(API_CONNECTION_STRING)
|
||||
if not api.is_running():
|
||||
print api.daemon_status()
|
||||
else:
|
||||
for func in api.help():
|
||||
print "%s:\n%s" % (func, api.help({'function': func}))
|
||||
```
|
||||
|
||||
If you've installed lbrynet, it comes with a file sharing application, called `lbrynet-daemon`, which breaks
|
||||
files into chunks, encrypts them with a symmetric key, computes their sha384 hash sum, generates
|
||||
a special file called a 'stream descriptor' containing the hash sums and some other file metadata,
|
||||
and makes the chunks available for download by other peers. A peer wishing to download the file
|
||||
must first obtain the 'stream descriptor' and then may open it with his 'lbrynet-console' client,
|
||||
must first obtain the 'stream descriptor' and then may open it with his `lbrynet-daemon` client,
|
||||
download all of the chunks by locating peers with the chunks via the DHT, and then combine the
|
||||
chunks into the original file, according to the metadata included in the 'stream descriptor'.
|
||||
|
||||
To install and use this client, see INSTALL and RUNNING
|
||||
For detailed instructions, see [INSTALL.md](INSTALL.md) and [RUNNING.md](RUNNING.md).
|
||||
|
||||
Installation
|
||||
------------
|
||||
Documentation: doc.lbry.io (may be out of date)
|
||||
|
||||
See INSTALL
|
||||
Source code: https://github.com/lbryio/lbry
|
||||
|
||||
Developers
|
||||
----------
|
||||
To contribute, [join us on Slack](https://lbry-slackin.herokuapp.com/) or contact josh@lbry.io. Pull requests are also welcome.
|
||||
|
||||
Documentation: doc.lbry.io
|
||||
Source code: trac.lbry.io/browser
|
||||
## Support
|
||||
|
||||
To contribute to the development of LBRYnet or lbrynet-console, contact jimmy@lbry.io
|
||||
Please open an issue and describe your situation in detail. We will respond as soon as we can.
|
||||
|
||||
Support
|
||||
-------
|
||||
For private issues, contact josh@lbry.io.
|
||||
|
||||
Send all support requests to jimmy@lbry.io
|
||||
## License
|
||||
|
||||
License
|
||||
-------
|
||||
|
||||
See LICENSE
|
||||
See [LICENSE](LICENSE)
|
||||
@@ -1,52 +0,0 @@
|
||||
To install LBRYnet and lbrynet-console, see INSTALL
|
||||
|
||||
lbrynet-console is a console application which makes use of the LBRYnet to share files.
|
||||
|
||||
In particular, lbrynet-console splits files into encrypted chunks of data compatible with
|
||||
LBRYnet, groups all metadata into a 'stream descriptor file' which can be sent directly to
|
||||
others wishing to obtain the file, or can be itself turned into a chunk compatible with
|
||||
LBRYnet and downloaded via LBRYnet by anyone knowing its sha384 hashsum. lbrynet-console
|
||||
also acts as a client whichreads a stream descriptor file, downloads the chunks of data
|
||||
specified by the hash sums found in the stream descriptor file, decrypts them according to
|
||||
metadata found in the stream, and reconstructs the original file. lbrynet-console features
|
||||
a server so that clients can connect to it and download the chunks and other data gotten
|
||||
from files created locally and files that have been downloaded from LBRYnet.
|
||||
|
||||
lbrynet-console also has a plugin system. There are two plugins: a live stream proof of
|
||||
concept which is currently far behind the development of the rest of the application and
|
||||
therefore will not run, and a plugin which attempts to determine which chunks on the
|
||||
network should be downloaded in order for the application to turn a profit. It will run,
|
||||
but its usefulness is extremely limited.
|
||||
|
||||
Passing '--help' to lbrynet-console will cause it to print out a quick help message
|
||||
describing other command line options to the application.
|
||||
|
||||
Once the application has been started, the user is presented with a numbered list of
|
||||
actions which looks something like this:
|
||||
|
||||
...
|
||||
[2] Toggle whether an LBRY File is running
|
||||
[3] Create an LBRY File from file
|
||||
[4] Publish a stream descriptor file to the DHT for an LBRY File
|
||||
...
|
||||
|
||||
To perform an action, type the desired number and then hit enter. For example, if you wish
|
||||
to create an LBRY file from a file as described in the beginning of this document, type 3 and
|
||||
hit enter.
|
||||
|
||||
If the application needs more input in order to for the action to be taken, the application
|
||||
will continue to print prompts for input until it has received what it needs.
|
||||
|
||||
For example, when creating an LBRY file from a file, the application needs to know which file
|
||||
it's supposed to use to create the LBRY file, so the user will be prompted for it:
|
||||
|
||||
File name:
|
||||
|
||||
The user should input the desired file name and hit enter, at which point the application
|
||||
will go about splitting the file and making it available on the network.
|
||||
|
||||
Some actions will produce sub-menus of actions, which work the same way.
|
||||
|
||||
A more detailed user guide is available at doc.lbry.io
|
||||
|
||||
Any issues may be reported to jimmy@lbry.io
|
||||
+146
@@ -0,0 +1,146 @@
|
||||
How to watch It's a Wonderful Life via LBRY
|
||||
|
||||
## Quickest quick guide
|
||||
|
||||
Create a directory called lbry, and go into that directory
|
||||
|
||||
Download the file https://raw.githubusercontent.com/lbryio/lbry-setup/master/lbry_setup.sh and run it in that directory
|
||||
|
||||
Once it's done building, type:
|
||||
|
||||
```
|
||||
lbrynet-console
|
||||
```
|
||||
|
||||
A console application will load, and after a moment you will be presented with a ">" signifying
|
||||
that the console is ready for commands.
|
||||
|
||||
If it's your first time running lbrynet-console, you should be given some credits to test the
|
||||
network. If they don't show up within a minute or two, let us know, and we'll send you some.
|
||||
|
||||
After your credits have shown up, type
|
||||
|
||||
```
|
||||
get wonderfullife
|
||||
```
|
||||
|
||||
into the prompt and hit enter.
|
||||
|
||||
You will be asked if you want to cancel, change options, save, and perhaps stream the file.
|
||||
|
||||
Enter 's' for save and then hit enter.
|
||||
|
||||
The file should start downloading. Enter the command 'status' to check on the status of files
|
||||
that are in the process of being downloaded.
|
||||
|
||||
To stop lbrynet-console, enter the command 'exit'.
|
||||
|
||||
|
||||
## Slightly longer install guide
|
||||
|
||||
### Installing lbrycrd, the full blockchain client
|
||||
|
||||
Note: this process takes upwards of an hour and is not necessary to use lbrynet.
|
||||
|
||||
```
|
||||
sudo apt-get install build-essential libtool autotools-dev autoconf pkg-config libssl-dev libboost-all-dev libdb-dev libdb++-dev libqt4-dev libprotobuf-dev protobuf-compiler git
|
||||
git clone --depth=1 -b alpha https://github.com/lbryio/lbrycrd.git
|
||||
cd lbrycrd
|
||||
./autogen.sh
|
||||
./configure --with-incompatible-bdb --without-gui
|
||||
|
||||
make
|
||||
```
|
||||
|
||||
When make has completed, create the directory where LBRYcrd data will be stored. ~/.lbrycrd is where LBRYcrd will look by default and so is recommended.
|
||||
|
||||
```
|
||||
mkdir ~/.lbrycrd
|
||||
echo 'rpcuser=rpcuser
|
||||
rpcpassword=rpcpassword' > ~/.lbrycrd/lbrycrd.conf
|
||||
# (use a long random password if your computer is on a network anyone else has access to. e.g, pwgen -s 20)
|
||||
cd ..
|
||||
|
||||
```
|
||||
|
||||
### Installing lbrynet from source
|
||||
|
||||
Acquire the LBRYnet source code from https://github.com/lbryio/lbry
|
||||
|
||||
```
|
||||
cd lbry
|
||||
sudo apt-get install libgmp3-dev build-essential python-dev python-pip
|
||||
```
|
||||
|
||||
(with virtualenv)
|
||||
|
||||
```
|
||||
python-virtualenv
|
||||
|
||||
virtualenv .
|
||||
|
||||
source bin/activate
|
||||
|
||||
python setup.py install
|
||||
|
||||
```
|
||||
|
||||
to deactivate the virtualenv later:
|
||||
|
||||
```
|
||||
deactivate
|
||||
```
|
||||
|
||||
to reactivate it, go to the directory in which you created it and:
|
||||
|
||||
```
|
||||
source bin/activate
|
||||
```
|
||||
|
||||
(without virtualenv)
|
||||
|
||||
```
|
||||
python setup.py build bdist_egg
|
||||
|
||||
sudo python setup.py install
|
||||
```
|
||||
|
||||
## Slightly longer running guide
|
||||
|
||||
### lbrynet-console can be set to use lbrycrdd instead of the built in lightweight client.
|
||||
|
||||
To run lbrynet-console with lbrycrdd:
|
||||
|
||||
```
|
||||
lbrynet-console </path/to/lbrycrdd>
|
||||
```
|
||||
|
||||
If lbrycrdd is not already running, lbrynet will launch it at that path, and will shut it down
|
||||
when lbrynet exits. If lbrycrdd is already running, lbrynet will not launch it and will not
|
||||
shut it down, but it will connect to it and use it.
|
||||
|
||||
### Running lbrycrdd manually
|
||||
|
||||
From the lbrycrd directory, run:
|
||||
|
||||
```
|
||||
./src/lbrycrdd -server -daemon
|
||||
```
|
||||
|
||||
If you want to mine LBC, also use the flag '-gen', so:
|
||||
|
||||
```
|
||||
./src/lbrycrdd -server -daemon -gen
|
||||
```
|
||||
|
||||
Warning: This will put a heavy load on your CPU
|
||||
|
||||
It will take a few minutes for your client to download the whole block chain.
|
||||
|
||||
To shut lbrycrdd down: from the lbrycrd directory, run:
|
||||
|
||||
```
|
||||
./src/lbrycrd-cli stop
|
||||
```
|
||||
|
||||
Any questions or problems, email jimmy@lbry.io
|
||||
@@ -0,0 +1,6 @@
|
||||
import logging
|
||||
|
||||
__version__ = "0.4.7"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
||||
@@ -0,0 +1,2 @@
|
||||
from events import *
|
||||
from api import AnalyticsApi as Api
|
||||
@@ -0,0 +1,71 @@
|
||||
import functools
|
||||
import json
|
||||
import logging
|
||||
|
||||
from requests import auth
|
||||
from requests_futures import sessions
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.analytics import utils
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def log_response(fn):
|
||||
def _log(future):
|
||||
if future.cancelled():
|
||||
log.warning('Request was unexpectedly cancelled')
|
||||
else:
|
||||
response = future.result()
|
||||
log.debug('Response (%s): %s', response.status_code, response.content)
|
||||
|
||||
@functools.wraps(fn)
|
||||
def wrapper(*args, **kwargs):
|
||||
future = fn(*args, **kwargs)
|
||||
future.add_done_callback(_log)
|
||||
return future
|
||||
return wrapper
|
||||
|
||||
|
||||
class AnalyticsApi(object):
|
||||
def __init__(self, session, url, write_key):
|
||||
self.session = session
|
||||
self.url = url
|
||||
self.write_key = write_key
|
||||
|
||||
@property
|
||||
def auth(self):
|
||||
return auth.HTTPBasicAuth(self.write_key, '')
|
||||
|
||||
@log_response
|
||||
def batch(self, events):
|
||||
"""Send multiple events in one request.
|
||||
|
||||
Each event needs to have its type specified.
|
||||
"""
|
||||
data = json.dumps({
|
||||
'batch': events,
|
||||
'sentAt': utils.now(),
|
||||
})
|
||||
log.debug('sending %s events', len(events))
|
||||
log.debug('Data: %s', data)
|
||||
return self.session.post(self.url + '/batch', json=data, auth=self.auth)
|
||||
|
||||
@log_response
|
||||
def track(self, event):
|
||||
"""Send a single tracking event"""
|
||||
log.debug('Sending track event: %s', event)
|
||||
import base64
|
||||
return self.session.post(self.url + '/track', json=event, auth=self.auth)
|
||||
|
||||
@classmethod
|
||||
def load(cls, session=None):
|
||||
"""Initialize an instance using values from lbry.io."""
|
||||
if not session:
|
||||
session = sessions.FuturesSession()
|
||||
return cls(
|
||||
session,
|
||||
conf.ANALYTICS_ENDPOINT,
|
||||
utils.deobfuscate(conf.ANALYTICS_TOKEN)
|
||||
)
|
||||
@@ -0,0 +1,47 @@
|
||||
from lbrynet.analytics import utils
|
||||
|
||||
|
||||
class Events(object):
|
||||
def __init__(self, context, lbry_id, session_id):
|
||||
self.context = context
|
||||
self.lbry_id = lbry_id
|
||||
self.session_id = session_id
|
||||
|
||||
def heartbeat(self):
|
||||
return {
|
||||
'userId': 'lbry',
|
||||
'event': 'Heartbeat',
|
||||
'properties': {
|
||||
'lbry_id': self.lbry_id,
|
||||
'session_id': self.session_id
|
||||
},
|
||||
'context': self.context,
|
||||
'timestamp': utils.now()
|
||||
}
|
||||
|
||||
|
||||
def make_context(platform, wallet, is_dev=False):
|
||||
# TODO: distinguish between developer and release instances
|
||||
return {
|
||||
'is_dev': is_dev,
|
||||
'app': {
|
||||
'name': 'lbrynet',
|
||||
'version': platform['lbrynet_version'],
|
||||
'ui_version': platform['ui_version'],
|
||||
'python_version': platform['python_version'],
|
||||
'wallet': {
|
||||
'name': wallet,
|
||||
# TODO: add in version info for lbrycrdd
|
||||
'version': platform['lbryum_version'] if wallet == 'lbryum' else None
|
||||
},
|
||||
},
|
||||
# TODO: expand os info to give linux/osx specific info
|
||||
'os': {
|
||||
'name': platform['os_system'],
|
||||
'version': platform['os_release']
|
||||
},
|
||||
'library': {
|
||||
'name': 'lbrynet-analytics',
|
||||
'version': '1.0.0'
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import datetime
|
||||
|
||||
from lbrynet.core.utils import *
|
||||
|
||||
|
||||
def now():
|
||||
"""Return utc now in isoformat with timezone"""
|
||||
return datetime.datetime.utcnow().isoformat() + 'Z'
|
||||
+59
-7
@@ -1,11 +1,11 @@
|
||||
"""
|
||||
Some network wide and also application specific parameters
|
||||
"""
|
||||
|
||||
|
||||
import os
|
||||
|
||||
|
||||
IS_DEVELOPMENT_VERSION = False
|
||||
|
||||
MAX_HANDSHAKE_SIZE = 2**16
|
||||
MAX_REQUEST_SIZE = 2**16
|
||||
MAX_BLOB_REQUEST_SIZE = 2**16
|
||||
@@ -13,13 +13,65 @@ MAX_RESPONSE_INFO_SIZE = 2**16
|
||||
MAX_BLOB_INFOS_TO_REQUEST = 20
|
||||
BLOBFILES_DIR = ".blobfiles"
|
||||
BLOB_SIZE = 2**21
|
||||
MIN_BLOB_DATA_PAYMENT_RATE = .5 # points/megabyte
|
||||
MIN_BLOB_INFO_PAYMENT_RATE = 2.0 # points/1000 infos
|
||||
MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE = 5.0 # points/1000 infos
|
||||
MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE = 5.0 # points/1000 infos
|
||||
|
||||
MIN_BLOB_DATA_PAYMENT_RATE = .005 # points/megabyte
|
||||
MIN_BLOB_INFO_PAYMENT_RATE = .02 # points/1000 infos
|
||||
MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE = .05 # points/1000 infos
|
||||
MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE = .05 # points/1000 infos
|
||||
MAX_CONNECTIONS_PER_STREAM = 5
|
||||
|
||||
KNOWN_DHT_NODES = [('104.236.42.182', 4000),
|
||||
('lbrynet1.lbry.io', 4444),
|
||||
('lbrynet2.lbry.io', 4444),
|
||||
('lbrynet3.lbry.io', 4444)]
|
||||
|
||||
POINTTRADER_SERVER = 'http://ec2-54-187-192-68.us-west-2.compute.amazonaws.com:2424'
|
||||
#POINTTRADER_SERVER = 'http://127.0.0.1:2424'
|
||||
|
||||
CRYPTSD_FILE_EXTENSION = ".cryptsd"
|
||||
if IS_DEVELOPMENT_VERSION:
|
||||
SEARCH_SERVERS = ["http://107.170.207.64:50005"]
|
||||
else:
|
||||
SEARCH_SERVERS = ["http://lighthouse1.lbry.io:50005",
|
||||
"http://lighthouse2.lbry.io:50005",
|
||||
"http://lighthouse3.lbry.io:50005"]
|
||||
|
||||
REFLECTOR_SERVERS = [("reflector.lbry.io", 5566)]
|
||||
|
||||
LOG_FILE_NAME = "lbrynet.log"
|
||||
LOG_POST_URL = "https://lbry.io/log-upload"
|
||||
|
||||
CRYPTSD_FILE_EXTENSION = ".cryptsd"
|
||||
|
||||
API_INTERFACE = "localhost"
|
||||
API_ADDRESS = "lbryapi"
|
||||
API_PORT = 5279
|
||||
if os.name == "nt":
|
||||
ICON_PATH = "icons"
|
||||
else:
|
||||
ICON_PATH = "app.icns"
|
||||
APP_NAME = "LBRY"
|
||||
API_CONNECTION_STRING = "http://%s:%i/%s" % (API_INTERFACE, API_PORT, API_ADDRESS)
|
||||
UI_ADDRESS = "http://%s:%i" % (API_INTERFACE, API_PORT)
|
||||
PROTOCOL_PREFIX = "lbry"
|
||||
|
||||
DEFAULT_WALLET = "lbryum"
|
||||
WALLET_TYPES = ["lbryum", "lbrycrd"]
|
||||
DEFAULT_TIMEOUT = 30
|
||||
DEFAULT_MAX_SEARCH_RESULTS = 25
|
||||
DEFAULT_MAX_KEY_FEE = {'USD': {'amount': 25.0, 'address': ''}}
|
||||
DEFAULT_SEARCH_TIMEOUT = 3.0
|
||||
DEFAULT_SD_DOWNLOAD_TIMEOUT = 3
|
||||
DEFAULT_CACHE_TIME = 150
|
||||
DEFAULT_UI_BRANCH = "master"
|
||||
|
||||
SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih']
|
||||
CURRENCIES = {
|
||||
'BTC': {'type': 'crypto'},
|
||||
'LBC': {'type': 'crypto'},
|
||||
'USD': {'type': 'fiat'},
|
||||
}
|
||||
|
||||
LOGGLY_TOKEN = 'LJEzATH4AzRgAwxjAP00LwZ2YGx3MwVgZTMuBQZ3MQuxLmOv'
|
||||
|
||||
ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
|
||||
ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
|
||||
+202
-82
@@ -1,14 +1,19 @@
|
||||
import logging
|
||||
import os
|
||||
import leveldb
|
||||
import time
|
||||
import json
|
||||
from twisted.internet import threads, defer, reactor, task
|
||||
import sqlite3
|
||||
from twisted.internet import threads, defer
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.enterprise import adbapi
|
||||
from lbrynet.core.HashBlob import BlobFile, TempBlob, BlobFileCreator, TempBlobCreator
|
||||
from lbrynet.core.server.DHTHashAnnouncer import DHTHashSupplier
|
||||
from lbrynet.core.utils import is_valid_blobhash
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.core.Error import NoSuchBlobError
|
||||
from lbrynet.core.sqlite_helpers import rerun_if_locked
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobManager(DHTHashSupplier):
|
||||
@@ -62,30 +67,37 @@ class BlobManager(DHTHashSupplier):
|
||||
def blob_paid_for(self, blob_hash, amount):
|
||||
pass
|
||||
|
||||
def get_all_verified_blobs(self):
|
||||
pass
|
||||
|
||||
|
||||
class DiskBlobManager(BlobManager):
|
||||
"""This class stores blobs on the hard disk"""
|
||||
def __init__(self, hash_announcer, blob_dir, db_dir):
|
||||
BlobManager.__init__(self, hash_announcer)
|
||||
self.blob_dir = blob_dir
|
||||
self.db_dir = db_dir
|
||||
self.db = None
|
||||
self.db_file = os.path.join(db_dir, "blobs.db")
|
||||
self.db_conn = None
|
||||
self.blob_type = BlobFile
|
||||
self.blob_creator_type = BlobFileCreator
|
||||
self.blobs = {}
|
||||
self.blob_hashes_to_delete = {} # {blob_hash: being_deleted (True/False)}
|
||||
self.blob_hashes_to_delete = {} # {blob_hash: being_deleted (True/False)}
|
||||
self._next_manage_call = None
|
||||
|
||||
def setup(self):
|
||||
d = threads.deferToThread(self._open_db)
|
||||
log.info("Setting up the DiskBlobManager. blob_dir: %s, db_file: %s", str(self.blob_dir),
|
||||
str(self.db_file))
|
||||
d = self._open_db()
|
||||
d.addCallback(lambda _: self._manage())
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
log.info("Stopping the DiskBlobManager")
|
||||
if self._next_manage_call is not None and self._next_manage_call.active():
|
||||
self._next_manage_call.cancel()
|
||||
self._next_manage_call = None
|
||||
self.db = None
|
||||
#d = self.db_conn.close()
|
||||
self.db_conn = None
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_blob(self, blob_hash, upload_allowed, length=None):
|
||||
@@ -101,7 +113,7 @@ class DiskBlobManager(BlobManager):
|
||||
def _make_new_blob(self, blob_hash, upload_allowed, length=None):
|
||||
blob = self.blob_type(self.blob_dir, blob_hash, upload_allowed, length)
|
||||
self.blobs[blob_hash] = blob
|
||||
d = threads.deferToThread(self._completed_blobs, [blob_hash])
|
||||
d = self._completed_blobs([blob_hash])
|
||||
|
||||
def check_completed(completed_blobs):
|
||||
|
||||
@@ -110,7 +122,7 @@ class DiskBlobManager(BlobManager):
|
||||
|
||||
if len(completed_blobs) == 1 and completed_blobs[0] == blob_hash:
|
||||
blob.verified = True
|
||||
inner_d = threads.deferToThread(self._get_blob_length, blob_hash)
|
||||
inner_d = self._get_blob_length(blob_hash)
|
||||
inner_d.addCallback(set_length)
|
||||
inner_d.addCallback(lambda _: blob)
|
||||
else:
|
||||
@@ -123,18 +135,18 @@ class DiskBlobManager(BlobManager):
|
||||
def blob_completed(self, blob, next_announce_time=None):
|
||||
if next_announce_time is None:
|
||||
next_announce_time = time.time()
|
||||
return threads.deferToThread(self._add_completed_blob, blob.blob_hash, blob.length,
|
||||
time.time(), next_announce_time)
|
||||
return self._add_completed_blob(blob.blob_hash, blob.length,
|
||||
time.time(), next_announce_time)
|
||||
|
||||
def completed_blobs(self, blobs_to_check):
|
||||
return threads.deferToThread(self._completed_blobs, blobs_to_check)
|
||||
return self._completed_blobs(blobs_to_check)
|
||||
|
||||
def hashes_to_announce(self):
|
||||
next_announce_time = time.time() + self.hash_reannounce_time
|
||||
return threads.deferToThread(self._get_blobs_to_announce, next_announce_time)
|
||||
return self._get_blobs_to_announce(next_announce_time)
|
||||
|
||||
def creator_finished(self, blob_creator):
|
||||
logging.debug("blob_creator.blob_hash: %s", blob_creator.blob_hash)
|
||||
log.debug("blob_creator.blob_hash: %s", blob_creator.blob_hash)
|
||||
assert blob_creator.blob_hash is not None
|
||||
assert blob_creator.blob_hash not in self.blobs
|
||||
assert blob_creator.length is not None
|
||||
@@ -155,18 +167,23 @@ class DiskBlobManager(BlobManager):
|
||||
self.blob_hashes_to_delete[blob_hash] = False
|
||||
|
||||
def update_all_last_verified_dates(self, timestamp):
|
||||
return threads.deferToThread(self._update_all_last_verified_dates, timestamp)
|
||||
return self._update_all_last_verified_dates(timestamp)
|
||||
|
||||
def immediate_announce_all_blobs(self):
|
||||
d = threads.deferToThread(self._get_all_verified_blob_hashes)
|
||||
d = self._get_all_verified_blob_hashes()
|
||||
d.addCallback(self.hash_announcer.immediate_announce)
|
||||
return d
|
||||
|
||||
def get_blob_length(self, blob_hash):
|
||||
return threads.deferToThread(self._get_blob_length, blob_hash)
|
||||
return self._get_blob_length(blob_hash)
|
||||
|
||||
def check_consistency(self):
|
||||
return threads.deferToThread(self._check_consistency)
|
||||
return self._check_consistency()
|
||||
|
||||
def get_all_verified_blobs(self):
|
||||
d = self._get_all_verified_blob_hashes()
|
||||
d.addCallback(self.completed_blobs)
|
||||
return d
|
||||
|
||||
def _manage(self):
|
||||
from twisted.internet import reactor
|
||||
@@ -185,19 +202,19 @@ class DiskBlobManager(BlobManager):
|
||||
return b_h
|
||||
|
||||
def set_not_deleting(err, b_h):
|
||||
logging.warning("Failed to delete blob %s. Reason: %s", str(b_h), err.getErrorMessage())
|
||||
log.warning("Failed to delete blob %s. Reason: %s", str(b_h), err.getErrorMessage())
|
||||
self.blob_hashes_to_delete[b_h] = False
|
||||
return err
|
||||
|
||||
def delete_from_db(result):
|
||||
b_hs = [r[1] for r in result if r[0] is True]
|
||||
if b_hs:
|
||||
d = threads.deferToThread(self._delete_blobs_from_db, b_hs)
|
||||
d = self._delete_blobs_from_db(b_hs)
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
|
||||
def log_error(err):
|
||||
logging.warning("Failed to delete completed blobs from the db: %s", err.getErrorMessage())
|
||||
log.warning("Failed to delete completed blobs from the db: %s", err.getErrorMessage())
|
||||
|
||||
d.addErrback(log_error)
|
||||
return d
|
||||
@@ -221,72 +238,127 @@ class DiskBlobManager(BlobManager):
|
||||
######### database calls #########
|
||||
|
||||
def _open_db(self):
|
||||
self.db = leveldb.LevelDB(os.path.join(self.db_dir, "blobs.db"))
|
||||
# check_same_thread=False is solely to quiet a spurious error that appears to be due
|
||||
# to a bug in twisted, where the connection is closed by a different thread than the
|
||||
# one that opened it. The individual connections in the pool are not used in multiple
|
||||
# threads.
|
||||
self.db_conn = adbapi.ConnectionPool('sqlite3', self.db_file, check_same_thread=False)
|
||||
return self.db_conn.runQuery("create table if not exists blobs (" +
|
||||
" blob_hash text primary key, " +
|
||||
" blob_length integer, " +
|
||||
" last_verified_time real, " +
|
||||
" next_announce_time real"
|
||||
")")
|
||||
|
||||
@rerun_if_locked
|
||||
def _add_completed_blob(self, blob_hash, length, timestamp, next_announce_time=None):
|
||||
logging.debug("Adding a completed blob. blob_hash=%s, length=%s", blob_hash, str(length))
|
||||
log.debug("Adding a completed blob. blob_hash=%s, length=%s", blob_hash, str(length))
|
||||
if next_announce_time is None:
|
||||
next_announce_time = timestamp
|
||||
self.db.Put(blob_hash, json.dumps((length, timestamp, next_announce_time)), sync=True)
|
||||
d = self.db_conn.runQuery("insert into blobs values (?, ?, ?, ?)",
|
||||
(blob_hash, length, timestamp, next_announce_time))
|
||||
d.addErrback(lambda err: err.trap(sqlite3.IntegrityError))
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _completed_blobs(self, blobs_to_check):
|
||||
blobs = []
|
||||
for b in blobs_to_check:
|
||||
if is_valid_blobhash(b):
|
||||
try:
|
||||
length, verified_time, next_announce_time = json.loads(self.db.Get(b))
|
||||
except KeyError:
|
||||
continue
|
||||
blobs_to_check = filter(is_valid_blobhash, blobs_to_check)
|
||||
|
||||
def get_blobs_in_db(db_transaction):
|
||||
blobs_in_db = [] # [(blob_hash, last_verified_time)]
|
||||
for b in blobs_to_check:
|
||||
result = db_transaction.execute("select last_verified_time from blobs where blob_hash = ?",
|
||||
(b,))
|
||||
row = result.fetchone()
|
||||
if row is not None:
|
||||
blobs_in_db.append((b, row[0]))
|
||||
return blobs_in_db
|
||||
|
||||
def get_valid_blobs(blobs_in_db):
|
||||
|
||||
def check_blob_verified_date(b, verified_time):
|
||||
file_path = os.path.join(self.blob_dir, b)
|
||||
if os.path.isfile(file_path):
|
||||
if verified_time > os.path.getctime(file_path):
|
||||
blobs.append(b)
|
||||
return blobs
|
||||
return True
|
||||
return False
|
||||
|
||||
def return_valid_blobs(results):
|
||||
valid_blobs = []
|
||||
for (b, verified_date), (success, result) in zip(blobs_in_db, results):
|
||||
if success is True and result is True:
|
||||
valid_blobs.append(b)
|
||||
return valid_blobs
|
||||
|
||||
ds = []
|
||||
for b, verified_date in blobs_in_db:
|
||||
ds.append(threads.deferToThread(check_blob_verified_date, b, verified_date))
|
||||
dl = defer.DeferredList(ds)
|
||||
dl.addCallback(return_valid_blobs)
|
||||
return dl
|
||||
|
||||
d = self.db_conn.runInteraction(get_blobs_in_db)
|
||||
d.addCallback(get_valid_blobs)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_blob_length(self, blob):
|
||||
length, verified_time, next_announce_time = json.loads(self.db.Get(blob))
|
||||
return length
|
||||
d = self.db_conn.runQuery("select blob_length from blobs where blob_hash = ?", (blob,))
|
||||
d.addCallback(lambda r: r[0][0] if len(r) else Failure(NoSuchBlobError(blob)))
|
||||
return d
|
||||
|
||||
#length, verified_time, next_announce_time = json.loads(self.db.Get(blob))
|
||||
#return length
|
||||
|
||||
@rerun_if_locked
|
||||
def _update_blob_verified_timestamp(self, blob, timestamp):
|
||||
length, old_verified_time, next_announce_time = json.loads(self.db.Get(blob))
|
||||
self.db.Put(blob, json.dumps((length, timestamp, next_announce_time)), sync=True)
|
||||
return self.db_conn.runQuery("update blobs set last_verified_time = ? where blob_hash = ?",
|
||||
(blob, timestamp))
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_blobs_to_announce(self, next_announce_time):
|
||||
# TODO: See if the following would be better for handling announce times:
|
||||
# TODO: Have a separate db for them, and read the whole thing into memory
|
||||
# TODO: on startup, and then write changes to db when they happen
|
||||
blobs = []
|
||||
batch = leveldb.WriteBatch()
|
||||
current_time = time.time()
|
||||
for blob_hash, blob_info in self.db.RangeIter():
|
||||
length, verified_time, announce_time = json.loads(blob_info)
|
||||
if announce_time < current_time:
|
||||
batch.Put(blob_hash, json.dumps((length, verified_time, next_announce_time)))
|
||||
blobs.append(blob_hash)
|
||||
self.db.Write(batch, sync=True)
|
||||
return blobs
|
||||
|
||||
def get_and_update(transaction):
|
||||
timestamp = time.time()
|
||||
r = transaction.execute("select blob_hash from blobs " +
|
||||
"where next_announce_time < ? and blob_hash is not null",
|
||||
(timestamp,))
|
||||
blobs = [b for b, in r.fetchall()]
|
||||
transaction.execute("update blobs set next_announce_time = ? where next_announce_time < ?",
|
||||
(next_announce_time, timestamp))
|
||||
return blobs
|
||||
|
||||
return self.db_conn.runInteraction(get_and_update)
|
||||
|
||||
@rerun_if_locked
|
||||
def _update_all_last_verified_dates(self, timestamp):
|
||||
batch = leveldb.WriteBatch()
|
||||
for blob_hash, blob_info in self.db.RangeIter():
|
||||
length, verified_time, announce_time = json.loads(blob_info)
|
||||
batch.Put(blob_hash, json.dumps((length, timestamp, announce_time)))
|
||||
self.db.Write(batch, sync=True)
|
||||
return self.db_conn.runQuery("update blobs set last_verified_date = ?", (timestamp,))
|
||||
|
||||
@rerun_if_locked
|
||||
def _delete_blobs_from_db(self, blob_hashes):
|
||||
batch = leveldb.WriteBatch()
|
||||
for blob_hash in blob_hashes:
|
||||
batch.Delete(blob_hash)
|
||||
self.db.Write(batch, sync=True)
|
||||
|
||||
def delete_blobs(transaction):
|
||||
for b in blob_hashes:
|
||||
transaction.execute("delete from blobs where blob_hash = ?", (b,))
|
||||
|
||||
return self.db_conn.runInteraction(delete_blobs)
|
||||
|
||||
@rerun_if_locked
|
||||
def _check_consistency(self):
|
||||
batch = leveldb.WriteBatch()
|
||||
|
||||
ALREADY_VERIFIED = 1
|
||||
NEWLY_VERIFIED = 2
|
||||
INVALID = 3
|
||||
|
||||
current_time = time.time()
|
||||
for blob_hash, blob_info in self.db.RangeIter():
|
||||
length, verified_time, announce_time = json.loads(blob_info)
|
||||
d = self.db_conn.runQuery("select blob_hash, blob_length, last_verified_time from blobs")
|
||||
|
||||
def check_blob(blob_hash, blob_length, verified_time):
|
||||
file_path = os.path.join(self.blob_dir, blob_hash)
|
||||
if os.path.isfile(file_path):
|
||||
if verified_time < os.path.getctime(file_path):
|
||||
if verified_time >= os.path.getctime(file_path):
|
||||
return ALREADY_VERIFIED
|
||||
else:
|
||||
h = get_lbry_hash_obj()
|
||||
len_so_far = 0
|
||||
f = open(file_path)
|
||||
@@ -296,19 +368,63 @@ class DiskBlobManager(BlobManager):
|
||||
break
|
||||
h.update(data)
|
||||
len_so_far += len(data)
|
||||
if len_so_far == length and h.hexdigest() == blob_hash:
|
||||
batch.Put(blob_hash, json.dumps((length, current_time, announce_time)))
|
||||
self.db.Write(batch, sync=True)
|
||||
if len_so_far == blob_length and h.hexdigest() == blob_hash:
|
||||
return NEWLY_VERIFIED
|
||||
return INVALID
|
||||
|
||||
def do_check(blobs):
|
||||
already_verified = []
|
||||
newly_verified = []
|
||||
invalid = []
|
||||
for blob_hash, blob_length, verified_time in blobs:
|
||||
status = check_blob(blob_hash, blob_length, verified_time)
|
||||
if status == ALREADY_VERIFIED:
|
||||
already_verified.append(blob_hash)
|
||||
elif status == NEWLY_VERIFIED:
|
||||
newly_verified.append(blob_hash)
|
||||
else:
|
||||
invalid.append(blob_hash)
|
||||
return already_verified, newly_verified, invalid
|
||||
|
||||
def update_newly_verified(transaction, blobs):
|
||||
for b in blobs:
|
||||
transaction.execute("update blobs set last_verified_time = ? where blob_hash = ?",
|
||||
(current_time, b))
|
||||
|
||||
def check_blobs(blobs):
|
||||
|
||||
@rerun_if_locked
|
||||
def update_and_return(status_lists):
|
||||
|
||||
already_verified, newly_verified, invalid = status_lists
|
||||
|
||||
d = self.db_conn.runInteraction(update_newly_verified, newly_verified)
|
||||
d.addCallback(lambda _: status_lists)
|
||||
return d
|
||||
|
||||
d = threads.deferToThread(do_check, blobs)
|
||||
|
||||
d.addCallback(update_and_return)
|
||||
return d
|
||||
|
||||
d.addCallback(check_blobs)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_all_verified_blob_hashes(self):
|
||||
blob_hashes = []
|
||||
for blob_hash, blob_info in self.db.RangeIter():
|
||||
length, verified_time, announce_time = json.loads(blob_info)
|
||||
file_path = os.path.join(self.blob_dir, blob_hash)
|
||||
if os.path.isfile(file_path):
|
||||
if verified_time > os.path.getctime(file_path):
|
||||
blob_hashes.append(blob_hash)
|
||||
return blob_hashes
|
||||
d = self.db_conn.runQuery("select blob_hash, last_verified_time from blobs")
|
||||
|
||||
def get_verified_blobs(blobs):
|
||||
verified_blobs = []
|
||||
for blob_hash, verified_time in blobs:
|
||||
file_path = os.path.join(self.blob_dir, blob_hash)
|
||||
if os.path.isfile(file_path):
|
||||
if verified_time > os.path.getctime(file_path):
|
||||
verified_blobs.append(blob_hash)
|
||||
return verified_blobs
|
||||
|
||||
d.addCallback(lambda blobs: threads.deferToThread(get_verified_blobs, blobs))
|
||||
return d
|
||||
|
||||
|
||||
class TempBlobManager(BlobManager):
|
||||
@@ -354,6 +470,10 @@ class TempBlobManager(BlobManager):
|
||||
blobs = [b.blob_hash for b in self.blobs.itervalues() if b.blob_hash in blobs_to_check and b.is_validated()]
|
||||
return defer.succeed(blobs)
|
||||
|
||||
def get_all_verified_blobs(self):
|
||||
d = self.completed_blobs(self.blobs)
|
||||
return d
|
||||
|
||||
def hashes_to_announce(self):
|
||||
now = time.time()
|
||||
blobs = [blob_hash for blob_hash, announce_time in self.blob_next_announces.iteritems() if announce_time < now]
|
||||
@@ -389,7 +509,7 @@ class TempBlobManager(BlobManager):
|
||||
if blob_hash in self.blobs:
|
||||
if self.blobs[blob_hash].length is not None:
|
||||
return defer.succeed(self.blobs[blob_hash].length)
|
||||
return defer.fail(ValueError("No such blob hash is known"))
|
||||
return defer.fail(NoSuchBlobError(blob_hash))
|
||||
|
||||
def immediate_announce_all_blobs(self):
|
||||
return self.hash_announcer.immediate_announce(self.blobs.iterkeys())
|
||||
@@ -400,7 +520,7 @@ class TempBlobManager(BlobManager):
|
||||
d = self._delete_blobs_marked_for_deletion()
|
||||
|
||||
def set_next_manage_call():
|
||||
logging.info("Setting the next manage call in %s", str(self))
|
||||
log.info("Setting the next manage call in %s", str(self))
|
||||
self._next_manage_call = reactor.callLater(1, self._manage)
|
||||
|
||||
d.addCallback(lambda _: set_next_manage_call())
|
||||
@@ -409,11 +529,11 @@ class TempBlobManager(BlobManager):
|
||||
|
||||
def remove_from_list(b_h):
|
||||
del self.blob_hashes_to_delete[b_h]
|
||||
logging.info("Deleted blob %s", blob_hash)
|
||||
log.info("Deleted blob %s", blob_hash)
|
||||
return b_h
|
||||
|
||||
def set_not_deleting(err, b_h):
|
||||
logging.warning("Failed to delete blob %s. Reason: %s", str(b_h), err.getErrorMessage())
|
||||
log.warning("Failed to delete blob %s. Reason: %s", str(b_h), err.getErrorMessage())
|
||||
self.blob_hashes_to_delete[b_h] = False
|
||||
return b_h
|
||||
|
||||
@@ -422,7 +542,7 @@ class TempBlobManager(BlobManager):
|
||||
if being_deleted is False:
|
||||
if blob_hash in self.blobs:
|
||||
self.blob_hashes_to_delete[blob_hash] = True
|
||||
logging.info("Found a blob marked for deletion: %s", blob_hash)
|
||||
log.info("Found a blob marked for deletion: %s", blob_hash)
|
||||
blob = self.blobs[blob_hash]
|
||||
d = blob.delete()
|
||||
|
||||
@@ -432,7 +552,7 @@ class TempBlobManager(BlobManager):
|
||||
ds.append(d)
|
||||
else:
|
||||
remove_from_list(blob_hash)
|
||||
d = defer.fail(Failure(ValueError("No such blob known")))
|
||||
logging.warning("Blob %s cannot be deleted because it is unknown")
|
||||
d = defer.fail(Failure(NoSuchBlobError(blob_hash)))
|
||||
log.warning("Blob %s cannot be deleted because it is unknown")
|
||||
ds.append(d)
|
||||
return defer.DeferredList(ds)
|
||||
@@ -1,6 +1,21 @@
|
||||
class DownloadOptionChoice(object):
|
||||
"""A possible choice that can be picked for some option.
|
||||
|
||||
An option can have one or more choices that can be picked from.
|
||||
"""
|
||||
def __init__(self, value, short_description, long_description, bool_options_description=None):
|
||||
self.value = value
|
||||
self.short_description = short_description
|
||||
self.long_description = long_description
|
||||
self.bool_options_description = bool_options_description
|
||||
|
||||
|
||||
class DownloadOption(object):
|
||||
def __init__(self, option_types, long_description, short_description, default):
|
||||
"""An option for a user to select a value from several different choices."""
|
||||
def __init__(self, option_types, long_description, short_description, default_value,
|
||||
default_value_description):
|
||||
self.option_types = option_types
|
||||
self.long_description = long_description
|
||||
self.short_description = short_description
|
||||
self.default = default
|
||||
self.default_value = default_value
|
||||
self.default_value_description = default_value_description
|
||||
@@ -22,15 +22,45 @@ class ConnectionClosedBeforeResponseError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class KeyFeeAboveMaxAllowed(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class UnknownNameError(Exception):
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
def __str__(self):
|
||||
return repr(self.name)
|
||||
|
||||
|
||||
class InvalidNameError(Exception):
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
def __str__(self):
|
||||
return repr(self.name)
|
||||
|
||||
|
||||
class UnknownStreamTypeError(Exception):
|
||||
def __init__(self, stream_type):
|
||||
self.stream_type = stream_type
|
||||
|
||||
def __str__(self):
|
||||
return repr(self.stream_type)
|
||||
|
||||
|
||||
class InvalidStreamDescriptorError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidStreamInfoError(Exception):
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
def __str__(self):
|
||||
return repr(self.name)
|
||||
|
||||
|
||||
class MisbehavingPeerError(Exception):
|
||||
pass
|
||||
@@ -45,4 +75,16 @@ class NoResponseError(MisbehavingPeerError):
|
||||
|
||||
|
||||
class InvalidResponseError(MisbehavingPeerError):
|
||||
pass
|
||||
|
||||
|
||||
class NoSuchBlobError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class NoSuchStreamHashError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidBlobHashError(Exception):
|
||||
pass
|
||||
@@ -13,6 +13,9 @@ from lbrynet.core.Error import DownloadCanceledError, InvalidDataError
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HashBlobReader(object):
|
||||
implements(interfaces.IConsumer)
|
||||
|
||||
@@ -56,6 +59,9 @@ class HashBlobWriter(object):
|
||||
" %s to %s" % (str(self.len_so_far),
|
||||
str(self.length_getter())))))
|
||||
else:
|
||||
if self.write_handle is None:
|
||||
log.debug("Tried to write to a write_handle that was None.")
|
||||
return
|
||||
self.write_handle.write(data)
|
||||
if self.len_so_far == self.length_getter():
|
||||
self.finished_cb(self)
|
||||
@@ -84,7 +90,7 @@ class HashBlob(object):
|
||||
if self.length is None and 0 <= length <= BLOB_SIZE:
|
||||
self.length = length
|
||||
return True
|
||||
logging.warning("Got an invalid length. Previous length: %s, Invalid length: %s", str(self.length), str(length))
|
||||
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s", str(self.length), str(length))
|
||||
return False
|
||||
|
||||
def get_length(self):
|
||||
@@ -126,8 +132,8 @@ class HashBlob(object):
|
||||
finished_deferred.callback(self)
|
||||
del self.writers[p]
|
||||
return True
|
||||
logging.warning("Somehow, the writer that was accepted as being valid was already removed. writer: %s",
|
||||
str(writer))
|
||||
log.warning("Somehow, the writer that was accepted as being valid was already removed. writer: %s",
|
||||
str(writer))
|
||||
return False
|
||||
|
||||
def errback_finished_deferred(err):
|
||||
@@ -199,14 +205,14 @@ class BlobFile(HashBlob):
|
||||
|
||||
def open_for_writing(self, peer):
|
||||
if not peer in self.writers:
|
||||
logging.debug("Opening %s to be written by %s", str(self), str(peer))
|
||||
log.debug("Opening %s to be written by %s", str(self), str(peer))
|
||||
write_file = tempfile.NamedTemporaryFile(delete=False, dir=self.blob_dir)
|
||||
finished_deferred = defer.Deferred()
|
||||
writer = HashBlobWriter(write_file, self.get_length, self.writer_finished)
|
||||
|
||||
self.writers[peer] = (writer, finished_deferred)
|
||||
return finished_deferred, writer.write, writer.cancel
|
||||
logging.warning("Tried to download the same file twice simultaneously from the same peer")
|
||||
log.warning("Tried to download the same file twice simultaneously from the same peer")
|
||||
return None, None, None
|
||||
|
||||
def open_for_reading(self):
|
||||
@@ -232,7 +238,7 @@ class BlobFile(HashBlob):
|
||||
d = threads.deferToThread(delete_from_file_system)
|
||||
|
||||
def log_error(err):
|
||||
logging.warning("An error occurred deleting %s: %s", str(self.file_path), err.getErrorMessage())
|
||||
log.warning("An error occurred deleting %s: %s", str(self.file_path), err.getErrorMessage())
|
||||
return err
|
||||
|
||||
d.addErrback(log_error)
|
||||
@@ -247,7 +253,7 @@ class BlobFile(HashBlob):
|
||||
|
||||
def _close_writer(self, writer):
|
||||
if writer.write_handle is not None:
|
||||
logging.debug("Closing %s", str(self))
|
||||
log.debug("Closing %s", str(self))
|
||||
name = writer.write_handle.name
|
||||
writer.write_handle.close()
|
||||
threads.deferToThread(os.remove, name)
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,468 +0,0 @@
|
||||
from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, ILBRYWallet
|
||||
from lbrynet.core.client.ClientRequest import ClientRequest
|
||||
from lbrynet.core.Error import UnknownNameError, InvalidStreamInfoError, RequestCanceledError
|
||||
from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException
|
||||
from twisted.internet import threads, reactor, defer, task
|
||||
from twisted.python.failure import Failure
|
||||
from collections import defaultdict, deque
|
||||
from zope.interface import implements
|
||||
from decimal import Decimal
|
||||
import datetime
|
||||
import logging
|
||||
import json
|
||||
import subprocess
|
||||
import socket
|
||||
import time
|
||||
import os
|
||||
|
||||
|
||||
class ReservedPoints(object):
|
||||
def __init__(self, identifier, amount):
|
||||
self.identifier = identifier
|
||||
self.amount = amount
|
||||
|
||||
|
||||
class LBRYcrdWallet(object):
|
||||
"""This class implements the LBRYWallet interface for the LBRYcrd payment system"""
|
||||
implements(ILBRYWallet)
|
||||
|
||||
def __init__(self, rpc_user, rpc_pass, rpc_url, rpc_port, start_lbrycrdd=False,
|
||||
wallet_dir=None, wallet_conf=None):
|
||||
self.rpc_conn_string = "http://%s:%s@%s:%s" % (rpc_user, rpc_pass, rpc_url, str(rpc_port))
|
||||
self.next_manage_call = None
|
||||
self.wallet_balance = Decimal(0.0)
|
||||
self.total_reserved_points = Decimal(0.0)
|
||||
self.peer_addresses = {} # {Peer: string}
|
||||
self.queued_payments = defaultdict(Decimal) # {address(string): amount(Decimal)}
|
||||
self.expected_balances = defaultdict(Decimal) # {address(string): amount(Decimal)}
|
||||
self.current_address_given_to_peer = {} # {Peer: address(string)}
|
||||
self.expected_balance_at_time = deque() # (Peer, address(string), amount(Decimal), time(datetime), count(int),
|
||||
# incremental_amount(float))
|
||||
self.max_expected_payment_time = datetime.timedelta(minutes=3)
|
||||
self.stopped = True
|
||||
self.start_lbrycrdd = start_lbrycrdd
|
||||
self.started_lbrycrdd = False
|
||||
self.wallet_dir = wallet_dir
|
||||
self.wallet_conf = wallet_conf
|
||||
self.lbrycrdd = None
|
||||
self.manage_running = False
|
||||
|
||||
def start(self):
|
||||
|
||||
def make_connection():
|
||||
if self.start_lbrycrdd is True:
|
||||
self._start_daemon()
|
||||
logging.info("Trying to connect to %s", self.rpc_conn_string)
|
||||
self.rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
logging.info("Connected!")
|
||||
|
||||
def start_manage():
|
||||
self.stopped = False
|
||||
self.manage()
|
||||
return True
|
||||
|
||||
d = threads.deferToThread(make_connection)
|
||||
d.addCallback(lambda _: start_manage())
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
self.stopped = True
|
||||
# If self.next_manage_call is None, then manage is currently running or else
|
||||
# start has not been called, so set stopped and do nothing else.
|
||||
if self.next_manage_call is not None:
|
||||
self.next_manage_call.cancel()
|
||||
self.next_manage_call = None
|
||||
|
||||
d = self.manage()
|
||||
if self.start_lbrycrdd is True:
|
||||
d.addBoth(lambda _: self._stop_daemon())
|
||||
return d
|
||||
|
||||
def manage(self):
|
||||
logging.info("Doing manage")
|
||||
self.next_manage_call = None
|
||||
have_set_manage_running = [False]
|
||||
|
||||
def check_if_manage_running():
|
||||
|
||||
d = defer.Deferred()
|
||||
|
||||
def fire_if_not_running():
|
||||
if self.manage_running is False:
|
||||
self.manage_running = True
|
||||
have_set_manage_running[0] = True
|
||||
d.callback(True)
|
||||
else:
|
||||
task.deferLater(reactor, 1, fire_if_not_running)
|
||||
|
||||
fire_if_not_running()
|
||||
return d
|
||||
|
||||
d = check_if_manage_running()
|
||||
|
||||
d.addCallback(lambda _: self._check_expected_balances())
|
||||
|
||||
d.addCallback(lambda _: self._send_payments())
|
||||
|
||||
d.addCallback(lambda _: threads.deferToThread(self._get_wallet_balance))
|
||||
|
||||
def set_wallet_balance(balance):
|
||||
self.wallet_balance = balance
|
||||
|
||||
d.addCallback(set_wallet_balance)
|
||||
|
||||
def set_next_manage_call():
|
||||
if not self.stopped:
|
||||
self.next_manage_call = reactor.callLater(60, self.manage)
|
||||
|
||||
d.addCallback(lambda _: set_next_manage_call())
|
||||
|
||||
def log_error(err):
|
||||
logging.error("Something went wrong during manage. Error message: %s", err.getErrorMessage())
|
||||
return err
|
||||
|
||||
d.addErrback(log_error)
|
||||
|
||||
def set_manage_not_running(arg):
|
||||
if have_set_manage_running[0] is True:
|
||||
self.manage_running = False
|
||||
return arg
|
||||
|
||||
d.addBoth(set_manage_not_running)
|
||||
return d
|
||||
|
||||
def get_info_exchanger(self):
|
||||
return LBRYcrdAddressRequester(self)
|
||||
|
||||
def get_wallet_info_query_handler_factory(self):
|
||||
return LBRYcrdAddressQueryHandlerFactory(self)
|
||||
|
||||
def get_balance(self):
|
||||
d = threads.deferToThread(self._get_wallet_balance)
|
||||
return d
|
||||
|
||||
def reserve_points(self, peer, amount):
|
||||
"""
|
||||
Ensure a certain amount of points are available to be sent as payment, before the service is rendered
|
||||
|
||||
@param peer: The peer to which the payment will ultimately be sent
|
||||
|
||||
@param amount: The amount of points to reserve
|
||||
|
||||
@return: A ReservedPoints object which is given to send_points once the service has been rendered
|
||||
"""
|
||||
rounded_amount = Decimal(str(round(amount, 8)))
|
||||
#if peer in self.peer_addresses:
|
||||
if self.wallet_balance >= self.total_reserved_points + rounded_amount:
|
||||
self.total_reserved_points += rounded_amount
|
||||
return ReservedPoints(peer, rounded_amount)
|
||||
return None
|
||||
|
||||
def cancel_point_reservation(self, reserved_points):
|
||||
"""
|
||||
Return all of the points that were reserved previously for some ReservedPoints object
|
||||
|
||||
@param reserved_points: ReservedPoints previously returned by reserve_points
|
||||
|
||||
@return: None
|
||||
"""
|
||||
self.total_reserved_points -= reserved_points.amount
|
||||
|
||||
def send_points(self, reserved_points, amount):
|
||||
"""
|
||||
Schedule a payment to be sent to a peer
|
||||
|
||||
@param reserved_points: ReservedPoints object previously returned by reserve_points
|
||||
|
||||
@param amount: amount of points to actually send, must be less than or equal to the
|
||||
amount reserved in reserved_points
|
||||
|
||||
@return: Deferred which fires when the payment has been scheduled
|
||||
"""
|
||||
rounded_amount = Decimal(str(round(amount, 8)))
|
||||
peer = reserved_points.identifier
|
||||
assert(rounded_amount <= reserved_points.amount)
|
||||
assert(peer in self.peer_addresses)
|
||||
self.queued_payments[self.peer_addresses[peer]] += rounded_amount
|
||||
# make any unused points available
|
||||
self.total_reserved_points -= (reserved_points.amount - rounded_amount)
|
||||
logging.info("ordering that %s points be sent to %s", str(rounded_amount),
|
||||
str(self.peer_addresses[peer]))
|
||||
peer.update_stats('points_sent', amount)
|
||||
return defer.succeed(True)
|
||||
|
||||
def add_expected_payment(self, peer, amount):
|
||||
"""Increase the number of points expected to be paid by a peer"""
|
||||
rounded_amount = Decimal(str(round(amount, 8)))
|
||||
assert(peer in self.current_address_given_to_peer)
|
||||
address = self.current_address_given_to_peer[peer]
|
||||
logging.info("expecting a payment at address %s in the amount of %s", str(address), str(rounded_amount))
|
||||
self.expected_balances[address] += rounded_amount
|
||||
expected_balance = self.expected_balances[address]
|
||||
expected_time = datetime.datetime.now() + self.max_expected_payment_time
|
||||
self.expected_balance_at_time.append((peer, address, expected_balance, expected_time, 0, amount))
|
||||
peer.update_stats('expected_points', amount)
|
||||
|
||||
def update_peer_address(self, peer, address):
|
||||
self.peer_addresses[peer] = address
|
||||
|
||||
def get_new_address_for_peer(self, peer):
|
||||
def set_address_for_peer(address):
|
||||
self.current_address_given_to_peer[peer] = address
|
||||
return address
|
||||
d = threads.deferToThread(self._get_new_address)
|
||||
d.addCallback(set_address_for_peer)
|
||||
return d
|
||||
|
||||
def get_stream_info_for_name(self, name):
|
||||
|
||||
def get_stream_info_from_value(result):
|
||||
r_dict = {}
|
||||
if 'value' in result:
|
||||
value = result['value']
|
||||
try:
|
||||
value_dict = json.loads(value)
|
||||
except ValueError:
|
||||
return Failure(InvalidStreamInfoError(name))
|
||||
if 'stream_hash' in value_dict:
|
||||
r_dict['stream_hash'] = value_dict['stream_hash']
|
||||
if 'name' in value_dict:
|
||||
r_dict['name'] = value_dict['name']
|
||||
if 'description' in value_dict:
|
||||
r_dict['description'] = value_dict['description']
|
||||
return r_dict
|
||||
return Failure(UnknownNameError(name))
|
||||
|
||||
d = threads.deferToThread(self._get_value_for_name, name)
|
||||
d.addCallback(get_stream_info_from_value)
|
||||
return d
|
||||
|
||||
def claim_name(self, stream_hash, name, amount):
|
||||
value = json.dumps({"stream_hash": stream_hash})
|
||||
d = threads.deferToThread(self._claim_name, name, value, amount)
|
||||
return d
|
||||
|
||||
def get_available_balance(self):
|
||||
return float(self.wallet_balance - self.total_reserved_points)
|
||||
|
||||
def get_new_address(self):
|
||||
return threads.deferToThread(self._get_new_address)
|
||||
|
||||
def _start_daemon(self):
|
||||
|
||||
if os.name == "nt":
|
||||
si = subprocess.STARTUPINFO
|
||||
si.dwFlags = subprocess.STARTF_USESHOWWINDOW
|
||||
si.wShowWindow = subprocess.SW_HIDE
|
||||
self.lbrycrdd = subprocess.Popen(["lbrycrdd.exe", "-datadir=%s" % self.wallet_dir,
|
||||
"-conf=%s" % self.wallet_conf], startupinfo=si)
|
||||
else:
|
||||
self.lbrycrdd = subprocess.Popen(["./lbrycrdd", "-datadir=%s" % self.wallet_dir,
|
||||
"-conf=%s" % self.wallet_conf])
|
||||
self.started_lbrycrdd = True
|
||||
|
||||
tries = 0
|
||||
while tries < 5:
|
||||
try:
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
rpc_conn.getinfo()
|
||||
break
|
||||
except (socket.error, JSONRPCException):
|
||||
tries += 1
|
||||
logging.warning("Failed to connect to lbrycrdd.")
|
||||
if tries < 5:
|
||||
time.sleep(2 ** tries)
|
||||
logging.warning("Trying again in %d seconds", 2 ** tries)
|
||||
else:
|
||||
logging.warning("Giving up.")
|
||||
else:
|
||||
self.lbrycrdd.terminate()
|
||||
raise ValueError("Couldn't open lbrycrdd")
|
||||
|
||||
def _stop_daemon(self):
|
||||
if self.lbrycrdd is not None and self.started_lbrycrdd is True:
|
||||
d = threads.deferToThread(self._rpc_stop)
|
||||
return d
|
||||
return defer.succeed(True)
|
||||
|
||||
def _check_expected_balances(self):
|
||||
now = datetime.datetime.now()
|
||||
balances_to_check = []
|
||||
try:
|
||||
while self.expected_balance_at_time[0][3] < now:
|
||||
balances_to_check.append(self.expected_balance_at_time.popleft())
|
||||
except IndexError:
|
||||
pass
|
||||
ds = []
|
||||
for balance_to_check in balances_to_check:
|
||||
d = threads.deferToThread(self._check_expected_balance, balance_to_check)
|
||||
ds.append(d)
|
||||
dl = defer.DeferredList(ds)
|
||||
|
||||
def handle_checks(results):
|
||||
from future_builtins import zip
|
||||
for balance, (success, result) in zip(balances_to_check, results):
|
||||
peer = balance[0]
|
||||
if success is True:
|
||||
if result is False:
|
||||
if balance[4] <= 1: # first or second strike, give them another chance
|
||||
new_expected_balance = (balance[0],
|
||||
balance[1],
|
||||
balance[2],
|
||||
datetime.datetime.now() + self.max_expected_payment_time,
|
||||
balance[4] + 1,
|
||||
balance[5])
|
||||
self.expected_balance_at_time.append(new_expected_balance)
|
||||
peer.update_score(-5.0)
|
||||
else:
|
||||
peer.update_score(-50.0)
|
||||
else:
|
||||
if balance[4] == 0:
|
||||
peer.update_score(balance[5])
|
||||
peer.update_stats('points_received', balance[5])
|
||||
else:
|
||||
logging.warning("Something went wrong checking a balance. Peer: %s, account: %s,"
|
||||
"expected balance: %s, expected time: %s, count: %s, error: %s",
|
||||
str(balance[0]), str(balance[1]), str(balance[2]), str(balance[3]),
|
||||
str(balance[4]), str(result.getErrorMessage()))
|
||||
|
||||
dl.addCallback(handle_checks)
|
||||
return dl
|
||||
|
||||
def _check_expected_balance(self, expected_balance):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
logging.info("Checking balance of address %s", str(expected_balance[1]))
|
||||
balance = rpc_conn.getreceivedbyaddress(expected_balance[1])
|
||||
logging.debug("received balance: %s", str(balance))
|
||||
logging.debug("expected balance: %s", str(expected_balance[2]))
|
||||
return balance >= expected_balance[2]
|
||||
|
||||
def _send_payments(self):
|
||||
logging.info("Trying to send payments, if there are any to be sent")
|
||||
|
||||
def do_send(payments):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
rpc_conn.sendmany("", payments)
|
||||
|
||||
payments_to_send = {}
|
||||
for address, points in self.queued_payments.items():
|
||||
logging.info("Should be sending %s points to %s", str(points), str(address))
|
||||
payments_to_send[address] = float(points)
|
||||
self.total_reserved_points -= points
|
||||
self.wallet_balance -= points
|
||||
del self.queued_payments[address]
|
||||
if payments_to_send:
|
||||
logging.info("Creating a transaction with outputs %s", str(payments_to_send))
|
||||
return threads.deferToThread(do_send, payments_to_send)
|
||||
logging.info("There were no payments to send")
|
||||
return defer.succeed(True)
|
||||
|
||||
def _get_wallet_balance(self):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
return rpc_conn.getbalance("")
|
||||
|
||||
def _get_new_address(self):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
return rpc_conn.getnewaddress()
|
||||
|
||||
def _get_value_for_name(self, name):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
return rpc_conn.getvalueforname(name)
|
||||
|
||||
def _claim_name(self, name, value, amount):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
return str(rpc_conn.claimname(name, value, amount))
|
||||
|
||||
def _rpc_stop(self):
|
||||
rpc_conn = AuthServiceProxy(self.rpc_conn_string)
|
||||
rpc_conn.stop()
|
||||
self.lbrycrdd.wait()
|
||||
|
||||
|
||||
class LBRYcrdAddressRequester(object):
|
||||
implements([IRequestCreator])
|
||||
|
||||
def __init__(self, wallet):
|
||||
self.wallet = wallet
|
||||
self._protocols = []
|
||||
|
||||
######### IRequestCreator #########
|
||||
|
||||
def send_next_request(self, peer, protocol):
|
||||
|
||||
if not protocol in self._protocols:
|
||||
r = ClientRequest({'lbrycrd_address': True}, 'lbrycrd_address')
|
||||
d = protocol.add_request(r)
|
||||
d.addCallback(self._handle_address_response, peer, r, protocol)
|
||||
d.addErrback(self._request_failed, peer)
|
||||
self._protocols.append(protocol)
|
||||
return defer.succeed(True)
|
||||
else:
|
||||
return defer.succeed(False)
|
||||
|
||||
######### internal calls #########
|
||||
|
||||
def _handle_address_response(self, response_dict, peer, request, protocol):
|
||||
assert request.response_identifier in response_dict, \
|
||||
"Expected %s in dict but did not get it" % request.response_identifier
|
||||
assert protocol in self._protocols, "Responding protocol is not in our list of protocols"
|
||||
address = response_dict[request.response_identifier]
|
||||
self.wallet.update_peer_address(peer, address)
|
||||
|
||||
def _request_failed(self, err, peer):
|
||||
if not err.check(RequestCanceledError):
|
||||
logging.warning("A peer failed to send a valid public key response. Error: %s, peer: %s",
|
||||
err.getErrorMessage(), str(peer))
|
||||
#return err
|
||||
|
||||
|
||||
class LBRYcrdAddressQueryHandlerFactory(object):
|
||||
implements(IQueryHandlerFactory)
|
||||
|
||||
def __init__(self, wallet):
|
||||
self.wallet = wallet
|
||||
|
||||
######### IQueryHandlerFactory #########
|
||||
|
||||
def build_query_handler(self):
|
||||
q_h = LBRYcrdAddressQueryHandler(self.wallet)
|
||||
return q_h
|
||||
|
||||
def get_primary_query_identifier(self):
|
||||
return 'lbrycrd_address'
|
||||
|
||||
def get_description(self):
|
||||
return "LBRYcrd Address - an address for receiving payments via LBRYcrd"
|
||||
|
||||
|
||||
class LBRYcrdAddressQueryHandler(object):
|
||||
implements(IQueryHandler)
|
||||
|
||||
def __init__(self, wallet):
|
||||
self.wallet = wallet
|
||||
self.query_identifiers = ['lbrycrd_address']
|
||||
self.address = None
|
||||
self.peer = None
|
||||
|
||||
######### IQueryHandler #########
|
||||
|
||||
def register_with_request_handler(self, request_handler, peer):
|
||||
self.peer = peer
|
||||
request_handler.register_query_handler(self, self.query_identifiers)
|
||||
|
||||
def handle_queries(self, queries):
|
||||
|
||||
def create_response(address):
|
||||
self.address = address
|
||||
fields = {'lbrycrd_address': address}
|
||||
return fields
|
||||
|
||||
if self.query_identifiers[0] in queries:
|
||||
d = self.wallet.get_new_address_for_peer(self.peer)
|
||||
d.addCallback(create_response)
|
||||
return d
|
||||
if self.address is None:
|
||||
logging.warning("Expected a request for an address, but did not receive one")
|
||||
return defer.fail(Failure(ValueError("Expected but did not receive an address request")))
|
||||
else:
|
||||
return defer.succeed({})
|
||||
+30
-22
@@ -1,7 +1,7 @@
|
||||
from collections import defaultdict
|
||||
import logging
|
||||
import leveldb
|
||||
import os
|
||||
import unqlite
|
||||
import time
|
||||
from Crypto.Hash import SHA512
|
||||
from Crypto.PublicKey import RSA
|
||||
@@ -12,7 +12,10 @@ from lbrynet.pointtraderclient import pointtraderclient
|
||||
from twisted.internet import defer, threads
|
||||
from zope.interface import implements
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.core.LBRYcrdWallet import ReservedPoints
|
||||
from lbrynet.core.LBRYWallet import ReservedPoints
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PTCWallet(object):
|
||||
@@ -71,7 +74,7 @@ class PTCWallet(object):
|
||||
|
||||
def save_key(success, private_key):
|
||||
if success is True:
|
||||
threads.deferToThread(self.save_private_key, private_key.exportKey())
|
||||
self._save_private_key(private_key.exportKey())
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -95,8 +98,8 @@ class PTCWallet(object):
|
||||
def start_manage():
|
||||
self.manage()
|
||||
return True
|
||||
d = threads.deferToThread(self._open_db)
|
||||
d.addCallback(lambda _: threads.deferToThread(self.get_wallet_private_key))
|
||||
d = self._open_db()
|
||||
d.addCallback(lambda _: self._get_wallet_private_key())
|
||||
d.addCallback(ensure_private_key_exists)
|
||||
d.addCallback(lambda _: start_manage())
|
||||
return d
|
||||
@@ -160,8 +163,8 @@ class PTCWallet(object):
|
||||
ds.append(d)
|
||||
del self.queued_payments[peer]
|
||||
else:
|
||||
logging.warning("Don't have a payment address for peer %s. Can't send %s points.",
|
||||
str(peer), str(points))
|
||||
log.warning("Don't have a payment address for peer %s. Can't send %s points.",
|
||||
str(peer), str(points))
|
||||
return defer.DeferredList(ds)
|
||||
|
||||
def get_balance(self):
|
||||
@@ -207,20 +210,25 @@ class PTCWallet(object):
|
||||
if self.peer_pub_keys[peer] in self.received_payments:
|
||||
received_balance = sum([a[0] for a in self.received_payments[self.peer_pub_keys[peer]]])
|
||||
if min_expected_balance > received_balance:
|
||||
logging.warning("Account in bad standing: %s (pub_key: %s), expected amount = %s, received_amount = %s",
|
||||
str(peer), self.peer_pub_keys[peer], str(min_expected_balance), str(received_balance))
|
||||
log.warning("Account in bad standing: %s (pub_key: %s), expected amount = %s, received_amount = %s",
|
||||
str(peer), self.peer_pub_keys[peer], str(min_expected_balance), str(received_balance))
|
||||
|
||||
def _open_db(self):
|
||||
self.db = leveldb.LevelDB(os.path.join(self.db_dir, "ptcwallet.db"))
|
||||
def open_db():
|
||||
self.db = unqlite.UnQLite(os.path.join(self.db_dir, "ptcwallet.db"))
|
||||
return threads.deferToThread(open_db)
|
||||
|
||||
def save_private_key(self, private_key):
|
||||
self.db.Put("private_key", private_key)
|
||||
def _save_private_key(self, private_key):
|
||||
def save_key():
|
||||
self.db['private_key'] = private_key
|
||||
return threads.deferToThread(save_key)
|
||||
|
||||
def get_wallet_private_key(self):
|
||||
try:
|
||||
return self.db.Get("private_key")
|
||||
except KeyError:
|
||||
def _get_wallet_private_key(self):
|
||||
def get_key():
|
||||
if 'private_key' in self.db:
|
||||
return self.db['private_key']
|
||||
return None
|
||||
return threads.deferToThread(get_key)
|
||||
|
||||
|
||||
class PointTraderKeyExchanger(object):
|
||||
@@ -256,9 +264,9 @@ class PointTraderKeyExchanger(object):
|
||||
|
||||
def _request_failed(self, err, peer):
|
||||
if not err.check(RequestCanceledError):
|
||||
logging.warning("A peer failed to send a valid public key response. Error: %s, peer: %s",
|
||||
err.getErrorMessage(), str(peer))
|
||||
#return err
|
||||
log.warning("A peer failed to send a valid public key response. Error: %s, peer: %s",
|
||||
err.getErrorMessage(), str(peer))
|
||||
return err
|
||||
|
||||
|
||||
class PointTraderKeyQueryHandlerFactory(object):
|
||||
@@ -301,15 +309,15 @@ class PointTraderKeyQueryHandler(object):
|
||||
try:
|
||||
RSA.importKey(new_encoded_pub_key)
|
||||
except (ValueError, TypeError, IndexError):
|
||||
logging.warning("Client sent an invalid public key.")
|
||||
log.warning("Client sent an invalid public key.")
|
||||
return defer.fail(Failure(ValueError("Client sent an invalid public key")))
|
||||
self.public_key = new_encoded_pub_key
|
||||
self.wallet.set_public_key_for_peer(self.peer, self.public_key)
|
||||
logging.debug("Received the client's public key: %s", str(self.public_key))
|
||||
log.debug("Received the client's public key: %s", str(self.public_key))
|
||||
fields = {'public_key': self.wallet.encoded_public_key}
|
||||
return defer.succeed(fields)
|
||||
if self.public_key is None:
|
||||
logging.warning("Expected a public key, but did not receive one")
|
||||
log.warning("Expected a public key, but did not receive one")
|
||||
return defer.fail(Failure(ValueError("Expected but did not receive a public key")))
|
||||
else:
|
||||
return defer.succeed({})
|
||||
+32
-91
@@ -1,5 +1,6 @@
|
||||
from zope.interface import implements
|
||||
from lbrynet.interfaces import IRateLimiter
|
||||
from twisted.internet import task
|
||||
|
||||
|
||||
class DummyRateLimiter(object):
|
||||
@@ -10,22 +11,20 @@ class DummyRateLimiter(object):
|
||||
self.total_ul_bytes = 0
|
||||
self.target_dl = 0
|
||||
self.target_ul = 0
|
||||
self.ul_delay = 0.00
|
||||
self.dl_delay = 0.00
|
||||
self.next_tick = None
|
||||
self.tick_call = None
|
||||
|
||||
def start(self):
|
||||
self.tick_call = task.LoopingCall(self.tick)
|
||||
self.tick_call.start(1)
|
||||
|
||||
def tick(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self.dl_bytes_this_second = 0
|
||||
self.ul_bytes_this_second = 0
|
||||
self.next_tick = reactor.callLater(1.0, self.tick)
|
||||
|
||||
def stop(self):
|
||||
if self.next_tick is not None:
|
||||
self.next_tick.cancel()
|
||||
self.next_tick = None
|
||||
if self.tick_call is not None:
|
||||
self.tick_call.stop()
|
||||
self.tick_call = None
|
||||
|
||||
def set_dl_limit(self, limit):
|
||||
pass
|
||||
@@ -33,12 +32,6 @@ class DummyRateLimiter(object):
|
||||
def set_ul_limit(self, limit):
|
||||
pass
|
||||
|
||||
def ul_wait_time(self):
|
||||
return self.ul_delay
|
||||
|
||||
def dl_wait_time(self):
|
||||
return self.dl_delay
|
||||
|
||||
def report_dl_bytes(self, num_bytes):
|
||||
self.dl_bytes_this_second += num_bytes
|
||||
self.total_dl_bytes += num_bytes
|
||||
@@ -58,66 +51,32 @@ class RateLimiter(object):
|
||||
def __init__(self, max_dl_bytes=None, max_ul_bytes=None):
|
||||
self.max_dl_bytes = max_dl_bytes
|
||||
self.max_ul_bytes = max_ul_bytes
|
||||
self.dl_bytes_this_second = 0
|
||||
self.ul_bytes_this_second = 0
|
||||
self.dl_bytes_this_interval = 0
|
||||
self.ul_bytes_this_interval = 0
|
||||
self.total_dl_bytes = 0
|
||||
self.total_ul_bytes = 0
|
||||
self.next_tick = None
|
||||
self.next_unthrottle_dl = None
|
||||
self.next_unthrottle_ul = None
|
||||
|
||||
self.next_dl_check = None
|
||||
self.next_ul_check = None
|
||||
|
||||
self.dl_check_interval = 1.0
|
||||
self.ul_check_interval = 1.0
|
||||
self.tick_call = None
|
||||
self.tick_interval = 0.1
|
||||
|
||||
self.dl_throttled = False
|
||||
self.ul_throttled = False
|
||||
|
||||
self.protocols = []
|
||||
|
||||
def start(self):
|
||||
self.tick_call = task.LoopingCall(self.tick)
|
||||
self.tick_call.start(self.tick_interval)
|
||||
|
||||
def tick(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
# happens once per second
|
||||
if self.next_dl_check is not None:
|
||||
self.next_dl_check.cancel()
|
||||
self.next_dl_check = None
|
||||
if self.next_ul_check is not None:
|
||||
self.next_ul_check.cancel()
|
||||
self.next_ul_check = None
|
||||
if self.max_dl_bytes is not None:
|
||||
if self.dl_bytes_this_second == 0:
|
||||
self.dl_check_interval = 1.0
|
||||
else:
|
||||
self.dl_check_interval = min(1.0, self.dl_check_interval *
|
||||
self.max_dl_bytes / self.dl_bytes_this_second)
|
||||
self.next_dl_check = reactor.callLater(self.dl_check_interval, self.check_dl)
|
||||
if self.max_ul_bytes is not None:
|
||||
if self.ul_bytes_this_second == 0:
|
||||
self.ul_check_interval = 1.0
|
||||
else:
|
||||
self.ul_check_interval = min(1.0, self.ul_check_interval *
|
||||
self.max_ul_bytes / self.ul_bytes_this_second)
|
||||
self.next_ul_check = reactor.callLater(self.ul_check_interval, self.check_ul)
|
||||
self.dl_bytes_this_second = 0
|
||||
self.ul_bytes_this_second = 0
|
||||
self.dl_bytes_this_interval = 0
|
||||
self.ul_bytes_this_interval = 0
|
||||
self.unthrottle_dl()
|
||||
self.unthrottle_ul()
|
||||
self.next_tick = reactor.callLater(1.0, self.tick)
|
||||
|
||||
def stop(self):
|
||||
if self.next_tick is not None:
|
||||
self.next_tick.cancel()
|
||||
self.next_tick = None
|
||||
if self.next_dl_check is not None:
|
||||
self.next_dl_check.cancel()
|
||||
self.next_dl_check = None
|
||||
if self.next_ul_check is not None:
|
||||
self.next_ul_check.cancel()
|
||||
self.next_ul_check = None
|
||||
if self.tick_call is not None:
|
||||
self.tick_call.stop()
|
||||
self.tick_call = None
|
||||
|
||||
def set_dl_limit(self, limit):
|
||||
self.max_dl_bytes = limit
|
||||
@@ -129,27 +88,15 @@ class RateLimiter(object):
|
||||
|
||||
def check_dl(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self.next_dl_check = None
|
||||
|
||||
if self.dl_bytes_this_second > self.max_dl_bytes:
|
||||
self.throttle_dl()
|
||||
else:
|
||||
self.next_dl_check = reactor.callLater(self.dl_check_interval, self.check_dl)
|
||||
self.dl_check_interval = min(self.dl_check_interval * 2, 1.0)
|
||||
if self.max_dl_bytes is not None and self.dl_bytes_this_interval > self.max_dl_bytes * self.tick_interval:
|
||||
from twisted.internet import reactor
|
||||
reactor.callLater(0, self.throttle_dl)
|
||||
|
||||
def check_ul(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self.next_ul_check = None
|
||||
|
||||
if self.ul_bytes_this_second > self.max_ul_bytes:
|
||||
self.throttle_ul()
|
||||
else:
|
||||
self.next_ul_check = reactor.callLater(self.ul_check_interval, self.check_ul)
|
||||
self.ul_check_interval = min(self.ul_check_interval * 2, 1.0)
|
||||
if self.max_ul_bytes is not None and self.ul_bytes_this_interval > self.max_ul_bytes * self.tick_interval:
|
||||
from twisted.internet import reactor
|
||||
reactor.callLater(0, self.throttle_ul)
|
||||
|
||||
def throttle_dl(self):
|
||||
if self.dl_throttled is False:
|
||||
@@ -175,23 +122,17 @@ class RateLimiter(object):
|
||||
protocol.unthrottle_upload()
|
||||
self.ul_throttled = False
|
||||
|
||||
#deprecated
|
||||
|
||||
def ul_wait_time(self):
|
||||
return 0
|
||||
|
||||
def dl_wait_time(self):
|
||||
return 0
|
||||
|
||||
#called by protocols
|
||||
|
||||
def report_dl_bytes(self, num_bytes):
|
||||
self.dl_bytes_this_second += num_bytes
|
||||
self.dl_bytes_this_interval += num_bytes
|
||||
self.total_dl_bytes += num_bytes
|
||||
self.check_dl()
|
||||
|
||||
def report_ul_bytes(self, num_bytes):
|
||||
self.ul_bytes_this_second += num_bytes
|
||||
self.ul_bytes_this_interval += num_bytes
|
||||
self.total_ul_bytes += num_bytes
|
||||
self.check_ul()
|
||||
|
||||
def register_protocol(self, protocol):
|
||||
if protocol not in self.protocols:
|
||||
|
||||
+71
-32
@@ -13,6 +13,9 @@ from lbrynet.core.PaymentRateManager import BasePaymentRateManager
|
||||
from twisted.internet import threads, defer
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LBRYSession(object):
|
||||
"""This class manages all important services common to any application that uses the network:
|
||||
the hash announcer, which informs other peers that this peer is associated with some hash. Usually,
|
||||
@@ -25,7 +28,7 @@ class LBRYSession(object):
|
||||
def __init__(self, blob_data_payment_rate, db_dir=None, lbryid=None, peer_manager=None, dht_node_port=None,
|
||||
known_dht_nodes=None, peer_finder=None, hash_announcer=None,
|
||||
blob_dir=None, blob_manager=None, peer_port=None, use_upnp=True,
|
||||
rate_limiter=None, wallet=None):
|
||||
rate_limiter=None, wallet=None, dht_node_class=node.Node):
|
||||
"""
|
||||
@param blob_data_payment_rate: The default payment rate for blob data
|
||||
|
||||
@@ -92,11 +95,11 @@ class LBRYSession(object):
|
||||
self.rate_limiter = rate_limiter
|
||||
|
||||
self.external_ip = '127.0.0.1'
|
||||
self.upnp_handler = None
|
||||
self.upnp_redirects_set = False
|
||||
|
||||
self.upnp_redirects = []
|
||||
|
||||
self.wallet = wallet
|
||||
|
||||
self.dht_node_class = dht_node_class
|
||||
self.dht_node = None
|
||||
|
||||
self.base_payment_rate_manager = BasePaymentRateManager(blob_data_payment_rate)
|
||||
@@ -104,7 +107,7 @@ class LBRYSession(object):
|
||||
def setup(self):
|
||||
"""Create the blob directory and database if necessary, start all desired services"""
|
||||
|
||||
logging.debug("Setting up the lbry session")
|
||||
log.debug("Setting up the lbry session")
|
||||
|
||||
if self.lbryid is None:
|
||||
self.lbryid = generate_id()
|
||||
@@ -124,7 +127,7 @@ class LBRYSession(object):
|
||||
d.addCallback(lambda _: self._setup_dht())
|
||||
else:
|
||||
if self.hash_announcer is None and self.peer_port is not None:
|
||||
logging.warning("The server has no way to advertise its available blobs.")
|
||||
log.warning("The server has no way to advertise its available blobs.")
|
||||
self.hash_announcer = DummyHashAnnouncer()
|
||||
|
||||
d.addCallback(lambda _: self._setup_other_components())
|
||||
@@ -135,41 +138,63 @@ class LBRYSession(object):
|
||||
ds = []
|
||||
if self.dht_node is not None:
|
||||
ds.append(defer.maybeDeferred(self.dht_node.stop))
|
||||
ds.append(defer.maybeDeferred(self.rate_limiter.stop))
|
||||
ds.append(defer.maybeDeferred(self.peer_finder.stop))
|
||||
ds.append(defer.maybeDeferred(self.hash_announcer.stop))
|
||||
ds.append(defer.maybeDeferred(self.wallet.stop))
|
||||
ds.append(defer.maybeDeferred(self.blob_manager.stop))
|
||||
if self.upnp_redirects_set is True:
|
||||
if self.rate_limiter is not None:
|
||||
ds.append(defer.maybeDeferred(self.rate_limiter.stop))
|
||||
if self.peer_finder is not None:
|
||||
ds.append(defer.maybeDeferred(self.peer_finder.stop))
|
||||
if self.hash_announcer is not None:
|
||||
ds.append(defer.maybeDeferred(self.hash_announcer.stop))
|
||||
if self.wallet is not None:
|
||||
ds.append(defer.maybeDeferred(self.wallet.stop))
|
||||
if self.blob_manager is not None:
|
||||
ds.append(defer.maybeDeferred(self.blob_manager.stop))
|
||||
if self.use_upnp is True:
|
||||
ds.append(defer.maybeDeferred(self._unset_upnp))
|
||||
return defer.DeferredList(ds)
|
||||
|
||||
def _try_upnp(self):
|
||||
|
||||
logging.debug("In _try_upnp")
|
||||
log.debug("In _try_upnp")
|
||||
|
||||
def threaded_try_upnp():
|
||||
if self.use_upnp is False:
|
||||
logging.debug("Not using upnp")
|
||||
log.debug("Not using upnp")
|
||||
return False
|
||||
u = miniupnpc.UPnP()
|
||||
num_devices_found = u.discover()
|
||||
if num_devices_found > 0:
|
||||
self.upnp_handler = u
|
||||
u.selectigd()
|
||||
external_ip = u.externalipaddress()
|
||||
if external_ip != '0.0.0.0':
|
||||
self.external_ip = external_ip
|
||||
if self.peer_port is not None:
|
||||
u.addportmapping(self.peer_port, 'TCP', u.lanaddr, self.peer_port, 'LBRY peer port', '')
|
||||
if u.getspecificportmapping(self.peer_port, 'TCP') is None:
|
||||
u.addportmapping(self.peer_port, 'TCP', u.lanaddr, self.peer_port, 'LBRY peer port', '')
|
||||
self.upnp_redirects.append((self.peer_port, 'TCP'))
|
||||
log.info("Set UPnP redirect for TCP port %d", self.peer_port)
|
||||
else:
|
||||
# see comment below
|
||||
log.warning("UPnP redirect already set for TCP port %d", self.peer_port)
|
||||
self.upnp_redirects.append((self.peer_port, 'TCP'))
|
||||
if self.dht_node_port is not None:
|
||||
u.addportmapping(self.dht_node_port, 'UDP', u.lanaddr, self.dht_node_port, 'LBRY DHT port', '')
|
||||
self.upnp_redirects_set = True
|
||||
if u.getspecificportmapping(self.dht_node_port, 'UDP') is None:
|
||||
u.addportmapping(self.dht_node_port, 'UDP', u.lanaddr, self.dht_node_port, 'LBRY DHT port', '')
|
||||
self.upnp_redirects.append((self.dht_node_port, 'UDP'))
|
||||
log.info("Set UPnP redirect for UPD port %d", self.dht_node_port)
|
||||
else:
|
||||
# TODO: check that the existing redirect was put up by an old lbrynet session before grabbing it
|
||||
# if such a disconnected redirect exists, then upnp won't work unless the redirect is appended
|
||||
# or is torn down and set back up. a bad shutdown of lbrynet could leave such a redirect up
|
||||
# and cause problems on the next start.
|
||||
# this could be problematic if a previous lbrynet session didn't make the redirect, and it was
|
||||
# made by another application
|
||||
log.warning("UPnP redirect already set for UDP port %d", self.dht_node_port)
|
||||
self.upnp_redirects.append((self.dht_node_port, 'UDP'))
|
||||
return True
|
||||
return False
|
||||
|
||||
def upnp_failed(err):
|
||||
logging.warning("UPnP failed. Reason: %s", err.getErrorMessage())
|
||||
log.warning("UPnP failed. Reason: %s", err.getErrorMessage())
|
||||
return False
|
||||
|
||||
d = threads.deferToThread(threaded_try_upnp)
|
||||
@@ -180,7 +205,7 @@ class LBRYSession(object):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
logging.debug("Starting the dht")
|
||||
log.debug("Starting the dht")
|
||||
|
||||
def match_port(h, p):
|
||||
return h, p
|
||||
@@ -204,8 +229,11 @@ class LBRYSession(object):
|
||||
d.addCallback(match_port, port)
|
||||
ds.append(d)
|
||||
|
||||
self.dht_node = node.Node(udpPort=self.dht_node_port, lbryid=self.lbryid,
|
||||
externalIP=self.external_ip)
|
||||
self.dht_node = self.dht_node_class(
|
||||
udpPort=self.dht_node_port,
|
||||
lbryid=self.lbryid,
|
||||
externalIP=self.external_ip
|
||||
)
|
||||
self.peer_finder = DHTPeerFinder(self.dht_node, self.peer_manager)
|
||||
if self.hash_announcer is None:
|
||||
self.hash_announcer = DHTHashAnnouncer(self.dht_node, self.peer_port)
|
||||
@@ -216,7 +244,7 @@ class LBRYSession(object):
|
||||
return dl
|
||||
|
||||
def _setup_other_components(self):
|
||||
logging.debug("Setting up the rest of the components")
|
||||
log.debug("Setting up the rest of the components")
|
||||
|
||||
if self.rate_limiter is None:
|
||||
self.rate_limiter = RateLimiter()
|
||||
@@ -227,19 +255,30 @@ class LBRYSession(object):
|
||||
else:
|
||||
self.blob_manager = DiskBlobManager(self.hash_announcer, self.blob_dir, self.db_dir)
|
||||
|
||||
self.rate_limiter.tick()
|
||||
self.rate_limiter.start()
|
||||
d1 = self.blob_manager.setup()
|
||||
d2 = self.wallet.start()
|
||||
return defer.DeferredList([d1, d2], fireOnOneErrback=True)
|
||||
|
||||
dl = defer.DeferredList([d1, d2], fireOnOneErrback=True, consumeErrors=True)
|
||||
|
||||
dl.addErrback(lambda err: err.value.subFailure)
|
||||
return dl
|
||||
|
||||
def _unset_upnp(self):
|
||||
|
||||
def threaded_unset_upnp():
|
||||
u = self.upnp_handler
|
||||
if self.peer_port is not None:
|
||||
u.deleteportmapping(self.peer_port, 'TCP')
|
||||
if self.dht_node_port is not None:
|
||||
u.deleteportmapping(self.dht_node_port, 'UDP')
|
||||
self.upnp_redirects_set = False
|
||||
u = miniupnpc.UPnP()
|
||||
num_devices_found = u.discover()
|
||||
if num_devices_found > 0:
|
||||
u.selectigd()
|
||||
for port, protocol in self.upnp_redirects:
|
||||
if u.getspecificportmapping(port, protocol) is None:
|
||||
log.warning("UPnP redirect for %s %d was removed by something else.", protocol, port)
|
||||
else:
|
||||
u.deleteportmapping(port, protocol)
|
||||
log.info("Removed UPnP redirect for %s %d.", protocol, port)
|
||||
self.upnp_redirects = []
|
||||
|
||||
return threads.deferToThread(threaded_unset_upnp)
|
||||
d = threads.deferToThread(threaded_unset_upnp)
|
||||
d.addErrback(lambda err: str(err))
|
||||
return d
|
||||
@@ -3,6 +3,9 @@ from twisted.internet import interfaces, defer
|
||||
from zope.interface import implements
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StreamCreator(object):
|
||||
"""Classes which derive from this class create a 'stream', which can be any
|
||||
collection of associated blobs and associated metadata. These classes
|
||||
@@ -42,7 +45,7 @@ class StreamCreator(object):
|
||||
|
||||
def stop(self):
|
||||
"""Stop creating the stream. Create the terminating zero-length blob."""
|
||||
logging.debug("stop has been called for StreamCreator")
|
||||
log.debug("stop has been called for StreamCreator")
|
||||
self.stopped = True
|
||||
if self.current_blob is not None:
|
||||
current_blob = self.current_blob
|
||||
|
||||
@@ -3,6 +3,10 @@ import json
|
||||
import logging
|
||||
from twisted.internet import threads
|
||||
from lbrynet.core.client.StandaloneBlobDownloader import StandaloneBlobDownloader
|
||||
from lbrynet.core.Error import UnknownStreamTypeError, InvalidStreamDescriptorError
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StreamDescriptorReader(object):
|
||||
@@ -81,7 +85,7 @@ class PlainStreamDescriptorWriter(StreamDescriptorWriter):
|
||||
def _write_stream_descriptor(self, raw_data):
|
||||
|
||||
def write_file():
|
||||
logging.debug("Writing the sd file to disk")
|
||||
log.debug("Writing the sd file to disk")
|
||||
with open(self.sd_file_name, 'w') as sd_file:
|
||||
sd_file.write(raw_data)
|
||||
return self.sd_file_name
|
||||
@@ -96,13 +100,26 @@ class BlobStreamDescriptorWriter(StreamDescriptorWriter):
|
||||
self.blob_manager = blob_manager
|
||||
|
||||
def _write_stream_descriptor(self, raw_data):
|
||||
logging.debug("Creating the new blob for the stream descriptor")
|
||||
log.debug("Creating the new blob for the stream descriptor")
|
||||
blob_creator = self.blob_manager.get_blob_creator()
|
||||
blob_creator.write(raw_data)
|
||||
logging.debug("Wrote the data to the new blob")
|
||||
log.debug("Wrote the data to the new blob")
|
||||
return blob_creator.close()
|
||||
|
||||
|
||||
class StreamMetadata(object):
|
||||
FROM_BLOB = 1
|
||||
FROM_PLAIN = 2
|
||||
|
||||
def __init__(self, validator, options, factories):
|
||||
self.validator = validator
|
||||
self.options = options
|
||||
self.factories = factories
|
||||
self.metadata_source = None
|
||||
self.source_blob_hash = None
|
||||
self.source_file = None
|
||||
|
||||
|
||||
class StreamDescriptorIdentifier(object):
|
||||
"""Tries to determine the type of stream described by the stream descriptor using the
|
||||
'stream_type' field. Keeps a list of StreamDescriptorValidators and StreamDownloaderFactorys
|
||||
@@ -110,9 +127,10 @@ class StreamDescriptorIdentifier(object):
|
||||
"""
|
||||
def __init__(self):
|
||||
self._sd_info_validators = {} # {stream_type: IStreamDescriptorValidator}
|
||||
self._stream_options = {} # {stream_type: IStreamOptions}
|
||||
self._stream_downloader_factories = defaultdict(list) # {stream_type: [IStreamDownloaderFactory]}
|
||||
|
||||
def add_stream_info_validator(self, stream_type, sd_info_validator):
|
||||
def add_stream_type(self, stream_type, sd_info_validator, stream_options):
|
||||
"""
|
||||
This is how the StreamDescriptorIdentifier learns about new types of stream descriptors.
|
||||
|
||||
@@ -126,9 +144,13 @@ class StreamDescriptorIdentifier(object):
|
||||
will then be called. If the validation step fails, an exception will be thrown, preventing the stream
|
||||
descriptor from being further processed.
|
||||
|
||||
@param stream_options: A class implementing the IStreamOptions interface. This class's constructor will be
|
||||
passed the sd_info_validator object containing the raw metadata from the stream descriptor file.
|
||||
|
||||
@return: None
|
||||
"""
|
||||
self._sd_info_validators[stream_type] = sd_info_validator
|
||||
self._stream_options[stream_type] = stream_options
|
||||
|
||||
def add_stream_downloader_factory(self, stream_type, factory):
|
||||
"""
|
||||
@@ -146,35 +168,59 @@ class StreamDescriptorIdentifier(object):
|
||||
"""
|
||||
self._stream_downloader_factories[stream_type].append(factory)
|
||||
|
||||
def get_info_and_factories_for_sd_file(self, sd_path):
|
||||
def _return_metadata(self, options_validator_factories, source_type, source):
|
||||
validator, options, factories = options_validator_factories
|
||||
m = StreamMetadata(validator, options, factories)
|
||||
m.metadata_source = source_type
|
||||
if source_type == StreamMetadata.FROM_BLOB:
|
||||
m.source_blob_hash = source
|
||||
if source_type == StreamMetadata.FROM_PLAIN:
|
||||
m.source_file = source
|
||||
return m
|
||||
|
||||
def get_metadata_for_sd_file(self, sd_path):
|
||||
sd_reader = PlainStreamDescriptorReader(sd_path)
|
||||
d = sd_reader.get_info()
|
||||
d.addCallback(self._return_info_and_factories)
|
||||
d.addCallback(self._return_options_and_validator_and_factories)
|
||||
d.addCallback(self._return_metadata, StreamMetadata.FROM_PLAIN, sd_path)
|
||||
return d
|
||||
|
||||
def get_info_and_factories_for_sd_blob(self, sd_blob):
|
||||
def get_metadata_for_sd_blob(self, sd_blob):
|
||||
sd_reader = BlobStreamDescriptorReader(sd_blob)
|
||||
d = sd_reader.get_info()
|
||||
d.addCallback(self._return_info_and_factories)
|
||||
d.addCallback(self._return_options_and_validator_and_factories)
|
||||
d.addCallback(self._return_metadata, StreamMetadata.FROM_BLOB, sd_blob.blob_hash)
|
||||
return d
|
||||
|
||||
def _get_factories(self, stream_type):
|
||||
assert stream_type in self._stream_downloader_factories, "Unrecognized stream type: " + str(stream_type)
|
||||
if not stream_type in self._stream_downloader_factories:
|
||||
raise UnknownStreamTypeError(stream_type)
|
||||
return self._stream_downloader_factories[stream_type]
|
||||
|
||||
def _get_validator(self, stream_type):
|
||||
assert stream_type in self._sd_info_validators, "Unrecognized stream type: " + str(stream_type)
|
||||
if not stream_type in self._sd_info_validators:
|
||||
raise UnknownStreamTypeError(stream_type)
|
||||
return self._sd_info_validators[stream_type]
|
||||
|
||||
def _return_info_and_factories(self, sd_info):
|
||||
assert 'stream_type' in sd_info, 'Invalid stream descriptor. No stream_type parameter.'
|
||||
def _get_options(self, stream_type):
|
||||
if not stream_type in self._stream_downloader_factories:
|
||||
raise UnknownStreamTypeError(stream_type)
|
||||
return self._stream_options[stream_type]
|
||||
|
||||
def _return_options_and_validator_and_factories(self, sd_info):
|
||||
if not 'stream_type' in sd_info:
|
||||
raise InvalidStreamDescriptorError('No stream_type parameter in stream descriptor.')
|
||||
stream_type = sd_info['stream_type']
|
||||
factories = self._get_factories(stream_type)
|
||||
validator = self._get_validator(stream_type)(sd_info)
|
||||
factories = [f for f in self._get_factories(stream_type) if f.can_download(validator)]
|
||||
|
||||
d = validator.validate()
|
||||
|
||||
d.addCallback(lambda _: (validator, factories))
|
||||
def get_options():
|
||||
options = self._get_options(stream_type)
|
||||
return validator, options, factories
|
||||
|
||||
d.addCallback(lambda _: get_options())
|
||||
return d
|
||||
|
||||
|
||||
@@ -192,4 +238,4 @@ def download_sd_blob(session, blob_hash, payment_rate_manager):
|
||||
"""
|
||||
downloader = StandaloneBlobDownloader(blob_hash, session.blob_manager, session.peer_finder,
|
||||
session.rate_limiter, payment_rate_manager, session.wallet)
|
||||
return downloader.download()
|
||||
return downloader.download()
|
||||
@@ -5,10 +5,14 @@ from twisted.python.failure import Failure
|
||||
from zope.interface import implements
|
||||
from lbrynet.core.Error import PriceDisagreementError, DownloadCanceledError, InsufficientFundsError
|
||||
from lbrynet.core.Error import InvalidResponseError, RequestCanceledError, NoResponseError
|
||||
from lbrynet.core.Error import ConnectionClosedBeforeResponseError
|
||||
from lbrynet.core.client.ClientRequest import ClientRequest, ClientBlobRequest
|
||||
from lbrynet.interfaces import IRequestCreator
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobRequester(object):
|
||||
implements(IRequestCreator)
|
||||
|
||||
@@ -68,6 +72,7 @@ class BlobRequester(object):
|
||||
sent_request = True
|
||||
else:
|
||||
d_r.cancel(InsufficientFundsError())
|
||||
d_r.finished_deferred.addErrback(lambda _: True)
|
||||
return defer.fail(InsufficientFundsError())
|
||||
if sent_request is True:
|
||||
if p_r is not None:
|
||||
@@ -84,7 +89,7 @@ class BlobRequester(object):
|
||||
######### internal calls #########
|
||||
|
||||
def _download_succeeded(self, arg, peer, blob):
|
||||
logging.info("Blob %s has been successfully downloaded from %s", str(blob), str(peer))
|
||||
log.info("Blob %s has been successfully downloaded from %s", str(blob), str(peer))
|
||||
self._update_local_score(peer, 5.0)
|
||||
peer.update_stats('blobs_downloaded', 1)
|
||||
peer.update_score(5.0)
|
||||
@@ -105,8 +110,8 @@ class BlobRequester(object):
|
||||
|
||||
def _handle_download_error(self, err, peer, blob_to_download):
|
||||
if not err.check(DownloadCanceledError, PriceDisagreementError, RequestCanceledError):
|
||||
logging.warning("An error occurred while downloading %s from %s. Error: %s",
|
||||
blob_to_download.blob_hash, str(peer), err.getTraceback())
|
||||
log.warning("An error occurred while downloading %s from %s. Error: %s",
|
||||
blob_to_download.blob_hash, str(peer), err.getTraceback())
|
||||
if err.check(PriceDisagreementError):
|
||||
# Don't kill the whole connection just because a price couldn't be agreed upon.
|
||||
# Other information might be desired by other request creators at a better rate.
|
||||
@@ -123,7 +128,7 @@ class BlobRequester(object):
|
||||
else:
|
||||
blob_hash = blobs_without_sources[0].blob_hash
|
||||
r = blob_hash
|
||||
logging.debug("Blob requester peer search response: %s", str(r))
|
||||
log.debug("Blob requester peer search response: %s", str(r))
|
||||
return defer.succeed(r)
|
||||
|
||||
def _find_peers_for_hash(self, h):
|
||||
@@ -137,6 +142,12 @@ class BlobRequester(object):
|
||||
return [p for p in peers if not p in bad_peers]
|
||||
|
||||
d.addCallback(choose_best_peers)
|
||||
|
||||
def lookup_failed(err):
|
||||
log.error("An error occurred looking up peers for a hash: %s", err.getTraceback())
|
||||
return []
|
||||
|
||||
d.addErrback(lookup_failed)
|
||||
return d
|
||||
|
||||
def _should_send_request_to(self, peer):
|
||||
@@ -201,7 +212,7 @@ class BlobRequester(object):
|
||||
request = ClientBlobRequest(request_dict, response_identifier, counting_write_func, d,
|
||||
cancel_func, blob_to_download)
|
||||
|
||||
logging.info("Requesting blob %s from %s", str(blob_to_download), str(peer))
|
||||
log.info("Requesting blob %s from %s", str(blob_to_download), str(peer))
|
||||
return request
|
||||
|
||||
def _price_settled(self, protocol):
|
||||
@@ -238,11 +249,11 @@ class BlobRequester(object):
|
||||
def _handle_availability(self, response_dict, peer, request):
|
||||
if not request.response_identifier in response_dict:
|
||||
raise InvalidResponseError("response identifier not in response")
|
||||
logging.debug("Received a response to the availability request")
|
||||
log.debug("Received a response to the availability request")
|
||||
blob_hashes = response_dict[request.response_identifier]
|
||||
for blob_hash in blob_hashes:
|
||||
if blob_hash in request.request_dict['requested_blobs']:
|
||||
logging.debug("The server has indicated it has the following blob available: %s", blob_hash)
|
||||
log.debug("The server has indicated it has the following blob available: %s", blob_hash)
|
||||
self._available_blobs[peer].append(blob_hash)
|
||||
if blob_hash in self._unavailable_blobs[peer]:
|
||||
self._unavailable_blobs[peer].remove(blob_hash)
|
||||
@@ -296,12 +307,13 @@ class BlobRequester(object):
|
||||
return
|
||||
if reason.check(NoResponseError):
|
||||
self._incompatible_peers.append(peer)
|
||||
return
|
||||
logging.warning("Blob requester: a request of type '%s' failed. Reason: %s, Error type: %s",
|
||||
str(request_type), reason.getErrorMessage(), reason.type)
|
||||
log.warning("Blob requester: a request of type '%s' failed. Reason: %s, Error type: %s",
|
||||
str(request_type), reason.getErrorMessage(), reason.type)
|
||||
self._update_local_score(peer, -10.0)
|
||||
if isinstance(reason, InvalidResponseError):
|
||||
if isinstance(reason, InvalidResponseError) or isinstance(reason, NoResponseError):
|
||||
peer.update_score(-10.0)
|
||||
else:
|
||||
peer.update_score(-2.0)
|
||||
if reason.check(ConnectionClosedBeforeResponseError):
|
||||
return
|
||||
return reason
|
||||
@@ -1,6 +1,6 @@
|
||||
import json
|
||||
import logging
|
||||
from twisted.internet import error, defer, reactor
|
||||
from twisted.internet import error, defer
|
||||
from twisted.internet.protocol import Protocol, ClientFactory
|
||||
from twisted.python import failure
|
||||
from lbrynet.conf import MAX_RESPONSE_INFO_SIZE as MAX_RESPONSE_SIZE
|
||||
@@ -11,6 +11,9 @@ from lbrynet.interfaces import IRequestSender, IRateLimited
|
||||
from zope.interface import implements
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ClientProtocol(Protocol):
|
||||
implements(IRequestSender, IRateLimited)
|
||||
|
||||
@@ -39,7 +42,7 @@ class ClientProtocol(Protocol):
|
||||
else:
|
||||
self._response_buff += data
|
||||
if len(self._response_buff) > MAX_RESPONSE_SIZE:
|
||||
logging.warning("Response is too large. Size %s", len(self._response_buff))
|
||||
log.warning("Response is too large. Size %s", len(self._response_buff))
|
||||
self.transport.loseConnection()
|
||||
response, extra_data = self._get_valid_response(self._response_buff)
|
||||
if response is not None:
|
||||
@@ -55,7 +58,7 @@ class ClientProtocol(Protocol):
|
||||
else:
|
||||
err = reason
|
||||
#if self._response_deferreds:
|
||||
# logging.warning("Lost connection with active response deferreds. %s", str(self._response_deferreds))
|
||||
# log.warning("Lost connection with active response deferreds. %s", str(self._response_deferreds))
|
||||
for key, d in self._response_deferreds.items():
|
||||
del self._response_deferreds[key]
|
||||
d.errback(err)
|
||||
@@ -70,7 +73,7 @@ class ClientProtocol(Protocol):
|
||||
return defer.fail(failure.Failure(ValueError("There is already a request for that response active")))
|
||||
self._next_request.update(request.request_dict)
|
||||
d = defer.Deferred()
|
||||
logging.debug("Adding a request. Request: %s", str(request))
|
||||
log.debug("Adding a request. Request: %s", str(request.request_dict))
|
||||
self._response_deferreds[request.response_identifier] = d
|
||||
return d
|
||||
|
||||
@@ -102,12 +105,14 @@ class ClientProtocol(Protocol):
|
||||
######### Internal request handling #########
|
||||
|
||||
def _handle_request_error(self, err):
|
||||
logging.error("An unexpected error occurred creating or sending a request to %s. Error message: %s",
|
||||
str(self.peer), err.getTraceback())
|
||||
log.error("An unexpected error occurred creating or sending a request to %s. Error message: %s",
|
||||
str(self.peer), err.getTraceback())
|
||||
self.transport.loseConnection()
|
||||
|
||||
def _ask_for_request(self):
|
||||
|
||||
log.debug("In _ask_for_request")
|
||||
|
||||
if self.connection_closed is True or self.connection_closing is True:
|
||||
return
|
||||
|
||||
@@ -117,7 +122,7 @@ class ClientProtocol(Protocol):
|
||||
self._send_request_message(request_msg)
|
||||
else:
|
||||
# The connection manager has indicated that this connection should be terminated
|
||||
logging.info("Closing the connection to %s due to having no further requests to send", str(self.peer))
|
||||
log.info("Closing the connection to %s due to having no further requests to send", str(self.peer))
|
||||
self.transport.loseConnection()
|
||||
|
||||
d = self._connection_manager.get_next_request(self.peer, self)
|
||||
@@ -153,14 +158,15 @@ class ClientProtocol(Protocol):
|
||||
# If an error gets to this point, log it and kill the connection.
|
||||
if not err.check(MisbehavingPeerError, ConnectionClosedBeforeResponseError, DownloadCanceledError,
|
||||
RequestCanceledError):
|
||||
logging.error("The connection to %s is closing due to an unexpected error: %s", str(self.peer),
|
||||
err.getErrorMessage())
|
||||
if not err.check(RequestCanceledError):
|
||||
self.transport.loseConnection()
|
||||
log.error("The connection to %s is closing due to an unexpected error: %s", str(self.peer),
|
||||
err.getErrorMessage())
|
||||
if not err.check(RequestCanceledError): # The connection manager is closing the connection, so
|
||||
# there's no need to do it here.
|
||||
return err
|
||||
|
||||
def _handle_response(self, response):
|
||||
ds = []
|
||||
logging.debug("Handling a response. Current expected responses: %s", str(self._response_deferreds))
|
||||
log.debug("Handling a response. Current expected responses: %s", str(self._response_deferreds))
|
||||
for key, val in response.items():
|
||||
if key in self._response_deferreds:
|
||||
d = self._response_deferreds[key]
|
||||
@@ -178,12 +184,27 @@ class ClientProtocol(Protocol):
|
||||
d.addErrback(self._handle_response_error)
|
||||
ds.append(d)
|
||||
|
||||
dl = defer.DeferredList(ds)
|
||||
dl = defer.DeferredList(ds, consumeErrors=True)
|
||||
|
||||
dl.addCallback(lambda _: self._ask_for_request())
|
||||
def get_next_request(results):
|
||||
failed = False
|
||||
for success, result in results:
|
||||
if success is False:
|
||||
failed = True
|
||||
log.info("The connection is closing due to an error: %s", str(result.getTraceback()))
|
||||
if failed is False:
|
||||
log.debug("Asking for another request.")
|
||||
from twisted.internet import reactor
|
||||
reactor.callLater(0, self._ask_for_request)
|
||||
#self._ask_for_request()
|
||||
else:
|
||||
log.debug("Not asking for another request.")
|
||||
self.transport.loseConnection()
|
||||
|
||||
dl.addCallback(get_next_request)
|
||||
|
||||
def _downloading_finished(self, arg):
|
||||
logging.debug("The blob has finished downloading")
|
||||
log.debug("The blob has finished downloading")
|
||||
self._blob_download_request = None
|
||||
self._downloading_blob = False
|
||||
return arg
|
||||
@@ -194,8 +215,8 @@ class ClientProtocol(Protocol):
|
||||
# TODO: always be this way. it's done this way now because the client has no other way
|
||||
# TODO: of telling the server it wants the download to stop. It would be great if the
|
||||
# TODO: protocol had such a mechanism.
|
||||
logging.info("Closing the connection to %s because the download of blob %s was canceled",
|
||||
str(self.peer), str(self._blob_download_request.blob))
|
||||
log.info("Closing the connection to %s because the download of blob %s was canceled",
|
||||
str(self.peer), str(self._blob_download_request.blob))
|
||||
#self.transport.loseConnection()
|
||||
#return True
|
||||
return err
|
||||
|
||||
@@ -7,49 +7,62 @@ from lbrynet.core.client.ClientProtocol import ClientProtocolFactory
|
||||
from lbrynet.core.Error import InsufficientFundsError
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PeerConnectionHandler(object):
|
||||
def __init__(self, request_creators, factory):
|
||||
self.request_creators = request_creators
|
||||
self.factory = factory
|
||||
self.connection = None
|
||||
|
||||
|
||||
class ConnectionManager(object):
|
||||
implements(interfaces.IConnectionManager)
|
||||
|
||||
def __init__(self, downloader, rate_limiter, primary_request_creators, secondary_request_creators):
|
||||
self.downloader = downloader
|
||||
self.rate_limiter = rate_limiter
|
||||
self.primary_request_creators = primary_request_creators
|
||||
self.secondary_request_creators = secondary_request_creators
|
||||
self.peer_connections = {} # {Peer: {'connection': connection,
|
||||
# 'request_creators': [IRequestCreator if using this connection]}}
|
||||
self.connections_closing = {} # {Peer: deferred (fired when the connection is closed)}
|
||||
self.next_manage_call = None
|
||||
self._primary_request_creators = primary_request_creators
|
||||
self._secondary_request_creators = secondary_request_creators
|
||||
self._peer_connections = {} # {Peer: PeerConnectionHandler}
|
||||
self._connections_closing = {} # {Peer: deferred (fired when the connection is closed)}
|
||||
self._next_manage_call = None
|
||||
self.stopped = True
|
||||
|
||||
def start(self):
|
||||
from twisted.internet import reactor
|
||||
|
||||
if self.next_manage_call is not None and self.next_manage_call.active() is True:
|
||||
self.next_manage_call.cancel()
|
||||
self.next_manage_call = reactor.callLater(0, self._manage)
|
||||
self.stopped = False
|
||||
|
||||
if self._next_manage_call is not None and self._next_manage_call.active() is True:
|
||||
self._next_manage_call.cancel()
|
||||
self._next_manage_call = reactor.callLater(0, self._manage)
|
||||
return defer.succeed(True)
|
||||
|
||||
def stop(self):
|
||||
if self.next_manage_call is not None and self.next_manage_call.active() is True:
|
||||
self.next_manage_call.cancel()
|
||||
self.next_manage_call = None
|
||||
self.stopped = True
|
||||
if self._next_manage_call is not None and self._next_manage_call.active() is True:
|
||||
self._next_manage_call.cancel()
|
||||
self._next_manage_call = None
|
||||
closing_deferreds = []
|
||||
for peer in self.peer_connections.keys():
|
||||
for peer in self._peer_connections.keys():
|
||||
|
||||
def close_connection(p):
|
||||
logging.info("Abruptly closing a connection to %s due to downloading being paused",
|
||||
str(p))
|
||||
log.info("Abruptly closing a connection to %s due to downloading being paused",
|
||||
str(p))
|
||||
|
||||
if self.peer_connections[p]['factory'].p is not None:
|
||||
d = self.peer_connections[p]['factory'].p.cancel_requests()
|
||||
if self._peer_connections[p].factory.p is not None:
|
||||
d = self._peer_connections[p].factory.p.cancel_requests()
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
|
||||
def disconnect_peer():
|
||||
self.peer_connections[p]['connection'].disconnect()
|
||||
if p in self.peer_connections:
|
||||
del self.peer_connections[p]
|
||||
d = defer.Deferred()
|
||||
self.connections_closing[p] = d
|
||||
self._connections_closing[p] = d
|
||||
self._peer_connections[p].connection.disconnect()
|
||||
if p in self._peer_connections:
|
||||
del self._peer_connections[p]
|
||||
return d
|
||||
|
||||
d.addBoth(lambda _: disconnect_peer())
|
||||
@@ -60,26 +73,27 @@ class ConnectionManager(object):
|
||||
|
||||
def get_next_request(self, peer, protocol):
|
||||
|
||||
logging.debug("Trying to get the next request for peer %s", str(peer))
|
||||
log.debug("Trying to get the next request for peer %s", str(peer))
|
||||
|
||||
if not peer in self.peer_connections:
|
||||
logging.debug("The peer has already been told to shut down.")
|
||||
if not peer in self._peer_connections or self.stopped is True:
|
||||
log.debug("The peer has already been told to shut down.")
|
||||
return defer.succeed(False)
|
||||
|
||||
def handle_error(err):
|
||||
if err.check(InsufficientFundsError):
|
||||
self.downloader.insufficient_funds()
|
||||
return False
|
||||
else:
|
||||
return err
|
||||
err.trap(InsufficientFundsError)
|
||||
self.downloader.insufficient_funds(err)
|
||||
return False
|
||||
|
||||
def check_if_request_sent(request_sent, request_creator):
|
||||
if peer not in self._peer_connections:
|
||||
# This can happen if the connection is told to close
|
||||
return False
|
||||
if request_sent is False:
|
||||
if request_creator in self.peer_connections[peer]['request_creators']:
|
||||
self.peer_connections[peer]['request_creators'].remove(request_creator)
|
||||
if request_creator in self._peer_connections[peer].request_creators:
|
||||
self._peer_connections[peer].request_creators.remove(request_creator)
|
||||
else:
|
||||
if not request_creator in self.peer_connections[peer]['request_creators']:
|
||||
self.peer_connections[peer]['request_creators'].append(request_creator)
|
||||
if not request_creator in self._peer_connections[peer].request_creators:
|
||||
self._peer_connections[peer].request_creators.append(request_creator)
|
||||
return request_sent
|
||||
|
||||
def check_requests(requests):
|
||||
@@ -89,7 +103,7 @@ class ConnectionManager(object):
|
||||
def get_secondary_requests_if_necessary(have_request):
|
||||
if have_request is True:
|
||||
ds = []
|
||||
for s_r_c in self.secondary_request_creators:
|
||||
for s_r_c in self._secondary_request_creators:
|
||||
d = s_r_c.send_next_request(peer, protocol)
|
||||
ds.append(d)
|
||||
dl = defer.DeferredList(ds)
|
||||
@@ -100,7 +114,7 @@ class ConnectionManager(object):
|
||||
|
||||
ds = []
|
||||
|
||||
for p_r_c in self.primary_request_creators:
|
||||
for p_r_c in self._primary_request_creators:
|
||||
d = p_r_c.send_next_request(peer, protocol)
|
||||
d.addErrback(handle_error)
|
||||
d.addCallback(check_if_request_sent, p_r_c)
|
||||
@@ -112,11 +126,11 @@ class ConnectionManager(object):
|
||||
return dl
|
||||
|
||||
def protocol_disconnected(self, peer, protocol):
|
||||
if peer in self.peer_connections:
|
||||
del self.peer_connections[peer]
|
||||
if peer in self.connections_closing:
|
||||
d = self.connections_closing[peer]
|
||||
del self.connections_closing[peer]
|
||||
if peer in self._peer_connections:
|
||||
del self._peer_connections[peer]
|
||||
if peer in self._connections_closing:
|
||||
d = self._connections_closing[peer]
|
||||
del self._connections_closing[peer]
|
||||
d.callback(True)
|
||||
|
||||
def _rank_request_creator_connections(self):
|
||||
@@ -125,30 +139,30 @@ class ConnectionManager(object):
|
||||
connections open that it likes
|
||||
"""
|
||||
def count_peers(request_creator):
|
||||
return len([p for p in self.peer_connections.itervalues() if request_creator in p['request_creators']])
|
||||
return len([p for p in self._peer_connections.itervalues() if request_creator in p.request_creators])
|
||||
|
||||
return sorted(self.primary_request_creators, key=count_peers)
|
||||
return sorted(self._primary_request_creators, key=count_peers)
|
||||
|
||||
def _connect_to_peer(self, peer):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
if peer is not None:
|
||||
logging.debug("Trying to connect to %s", str(peer))
|
||||
if peer is not None and self.stopped is False:
|
||||
log.debug("Trying to connect to %s", str(peer))
|
||||
factory = ClientProtocolFactory(peer, self.rate_limiter, self)
|
||||
self._peer_connections[peer] = PeerConnectionHandler(self._primary_request_creators[:],
|
||||
factory)
|
||||
connection = reactor.connectTCP(peer.host, peer.port, factory)
|
||||
self.peer_connections[peer] = {'connection': connection,
|
||||
'request_creators': self.primary_request_creators[:],
|
||||
'factory': factory}
|
||||
self._peer_connections[peer].connection = connection
|
||||
|
||||
def _manage(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
def get_new_peers(request_creators):
|
||||
logging.debug("Trying to get a new peer to connect to")
|
||||
log.debug("Trying to get a new peer to connect to")
|
||||
if len(request_creators) > 0:
|
||||
logging.debug("Got a creator to check: %s", str(request_creators[0]))
|
||||
log.debug("Got a creator to check: %s", str(request_creators[0]))
|
||||
d = request_creators[0].get_new_peers()
|
||||
d.addCallback(lambda h: h if h is not None else get_new_peers(request_creators[1:]))
|
||||
return d
|
||||
@@ -158,20 +172,21 @@ class ConnectionManager(object):
|
||||
def pick_best_peer(peers):
|
||||
# TODO: Eventually rank them based on past performance/reputation. For now
|
||||
# TODO: just pick the first to which we don't have an open connection
|
||||
logging.debug("Got a list of peers to choose from: %s", str(peers))
|
||||
|
||||
log.debug("Got a list of peers to choose from: %s", str(peers))
|
||||
if peers is None:
|
||||
return None
|
||||
for peer in peers:
|
||||
if not peer in self.peer_connections:
|
||||
logging.debug("Got a good peer. Returning peer %s", str(peer))
|
||||
if not peer in self._peer_connections:
|
||||
log.debug("Got a good peer. Returning peer %s", str(peer))
|
||||
return peer
|
||||
logging.debug("Couldn't find a good peer to connect to")
|
||||
log.debug("Couldn't find a good peer to connect to")
|
||||
return None
|
||||
|
||||
if len(self.peer_connections) < MAX_CONNECTIONS_PER_STREAM:
|
||||
if len(self._peer_connections) < MAX_CONNECTIONS_PER_STREAM:
|
||||
ordered_request_creators = self._rank_request_creator_connections()
|
||||
d = get_new_peers(ordered_request_creators)
|
||||
d.addCallback(pick_best_peer)
|
||||
d.addCallback(self._connect_to_peer)
|
||||
|
||||
self.next_manage_call = reactor.callLater(1, self._manage)
|
||||
self._next_manage_call = reactor.callLater(1, self._manage)
|
||||
@@ -32,8 +32,9 @@ class DHTPeerFinder(object):
|
||||
bin_hash = binascii.unhexlify(blob_hash)
|
||||
|
||||
def filter_peers(peer_list):
|
||||
peers = set(peer_list)
|
||||
good_peers = []
|
||||
for host, port in peer_list:
|
||||
for host, port in peers:
|
||||
peer = self.peer_manager.get_peer(host, port)
|
||||
if peer.is_available() is True:
|
||||
good_peers.append(peer)
|
||||
|
||||
@@ -5,6 +5,9 @@ from zope.interface import implements
|
||||
from lbrynet import interfaces
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DownloadManager(object):
|
||||
implements(interfaces.IDownloadManager)
|
||||
|
||||
@@ -24,7 +27,7 @@ class DownloadManager(object):
|
||||
|
||||
def start_downloading(self):
|
||||
d = self.blob_info_finder.get_initial_blobs()
|
||||
logging.debug("Requested the initial blobs from the info finder")
|
||||
log.debug("Requested the initial blobs from the info finder")
|
||||
d.addCallback(self.add_blobs_to_download)
|
||||
d.addCallback(lambda _: self.resume_downloading())
|
||||
return d
|
||||
@@ -33,7 +36,7 @@ class DownloadManager(object):
|
||||
|
||||
def check_start(result, manager):
|
||||
if isinstance(result, failure.Failure):
|
||||
logging.error("Failed to start the %s: %s", manager, result.getErrorMessage())
|
||||
log.error("Failed to start the %s: %s", manager, result.getErrorMessage())
|
||||
return False
|
||||
return True
|
||||
|
||||
@@ -49,7 +52,7 @@ class DownloadManager(object):
|
||||
|
||||
def check_stop(result, manager):
|
||||
if isinstance(result, failure.Failure):
|
||||
logging.error("Failed to stop the %s: %s", manager. result.getErrorMessage())
|
||||
log.error("Failed to stop the %s: %s", manager, result.getErrorMessage())
|
||||
return False
|
||||
return True
|
||||
|
||||
@@ -63,21 +66,21 @@ class DownloadManager(object):
|
||||
|
||||
def add_blobs_to_download(self, blob_infos):
|
||||
|
||||
logging.debug("Adding %s to blobs", str(blob_infos))
|
||||
log.debug("Adding %s blobs to blobs", len(blob_infos))
|
||||
|
||||
def add_blob_to_list(blob, blob_num):
|
||||
self.blobs[blob_num] = blob
|
||||
logging.info("Added blob (hash: %s, number %s) to the list", str(blob.blob_hash), str(blob_num))
|
||||
log.debug("Added blob (hash: %s, number %s) to the list", str(blob.blob_hash), str(blob_num))
|
||||
|
||||
def error_during_add(err):
|
||||
logging.warning("An error occurred adding the blob to blobs. Error:%s", err.getErrorMessage())
|
||||
log.warning("An error occurred adding the blob to blobs. Error:%s", err.getErrorMessage())
|
||||
return err
|
||||
|
||||
ds = []
|
||||
for blob_info in blob_infos:
|
||||
if not blob_info.blob_num in self.blobs:
|
||||
self.blob_infos[blob_info.blob_num] = blob_info
|
||||
logging.debug("Trying to get the blob associated with blob hash %s", str(blob_info.blob_hash))
|
||||
log.debug("Trying to get the blob associated with blob hash %s", str(blob_info.blob_hash))
|
||||
d = self.blob_manager.get_blob(blob_info.blob_hash, self.upload_allowed, blob_info.length)
|
||||
d.addCallback(add_blob_to_list, blob_info.blob_num)
|
||||
d.addErrback(error_during_add)
|
||||
@@ -112,4 +115,4 @@ class DownloadManager(object):
|
||||
if not self.blobs:
|
||||
return self.calculate_total_bytes()
|
||||
else:
|
||||
return sum([b.length for b in self.needed_blobs() if b.length is not None])
|
||||
return sum([b.length for b in self.needed_blobs() if b.length is not None])
|
||||
@@ -5,9 +5,15 @@ from lbrynet.core.BlobInfo import BlobInfo
|
||||
from lbrynet.core.client.BlobRequester import BlobRequester
|
||||
from lbrynet.core.client.ConnectionManager import ConnectionManager
|
||||
from lbrynet.core.client.DownloadManager import DownloadManager
|
||||
from lbrynet.core.Error import InvalidBlobHashError
|
||||
from lbrynet.core.utils import is_valid_blobhash
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.internet import defer
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SingleBlobMetadataHandler(object):
|
||||
implements(interfaces.IMetadataHandler)
|
||||
|
||||
@@ -18,7 +24,7 @@ class SingleBlobMetadataHandler(object):
|
||||
######## IMetadataHandler #########
|
||||
|
||||
def get_initial_blobs(self):
|
||||
logging.debug("Returning the blob info")
|
||||
log.debug("Returning the blob info")
|
||||
return defer.succeed([BlobInfo(self.blob_hash, 0, None)])
|
||||
|
||||
def final_blob_num(self):
|
||||
@@ -72,7 +78,7 @@ class SingleProgressManager(object):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
logging.debug("The blob %s has been downloaded. Calling the finished callback", str(blob))
|
||||
log.debug("The blob %s has been downloaded. Calling the finished callback", str(blob))
|
||||
if self.finished is False:
|
||||
self.finished = True
|
||||
reactor.callLater(0, self.finished_callback, blob)
|
||||
@@ -99,6 +105,9 @@ class StandaloneBlobDownloader(object):
|
||||
self.finished_deferred = None
|
||||
|
||||
def download(self):
|
||||
if not is_valid_blobhash(self.blob_hash):
|
||||
return defer.fail(Failure(InvalidBlobHashError(self.blob_hash)))
|
||||
|
||||
def cancel_download(d):
|
||||
self.stop()
|
||||
|
||||
@@ -127,7 +136,10 @@ class StandaloneBlobDownloader(object):
|
||||
|
||||
def _blob_downloaded(self, blob):
|
||||
self.stop()
|
||||
self.finished_deferred.callback(blob)
|
||||
if not self.finished_deferred.called:
|
||||
self.finished_deferred.callback(blob)
|
||||
|
||||
def insufficient_funds(self):
|
||||
return self.stop()
|
||||
def insufficient_funds(self, err):
|
||||
self.stop()
|
||||
if not self.finished_deferred.called:
|
||||
self.finished_deferred.errback(err)
|
||||
@@ -4,6 +4,9 @@ from twisted.internet import defer
|
||||
from zope.interface import implements
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StreamProgressManager(object):
|
||||
implements(IProgressManager)
|
||||
|
||||
@@ -61,18 +64,18 @@ class StreamProgressManager(object):
|
||||
return defer.succeed(None)
|
||||
|
||||
def _finished_with_blob(self, blob_num):
|
||||
logging.debug("In _finished_with_blob, blob_num = %s", str(blob_num))
|
||||
log.debug("In _finished_with_blob, blob_num = %s", str(blob_num))
|
||||
if self.delete_blob_after_finished is True:
|
||||
logging.debug("delete_blob_after_finished is True")
|
||||
log.debug("delete_blob_after_finished is True")
|
||||
blobs = self.download_manager.blobs
|
||||
if blob_num in blobs:
|
||||
logging.debug("Telling the blob manager, %s, to delete blob %s", str(self.blob_manager),
|
||||
blobs[blob_num].blob_hash)
|
||||
log.debug("Telling the blob manager, %s, to delete blob %s", str(self.blob_manager),
|
||||
blobs[blob_num].blob_hash)
|
||||
self.blob_manager.delete_blobs([blobs[blob_num].blob_hash])
|
||||
else:
|
||||
logging.debug("Blob number %s was not in blobs", str(blob_num))
|
||||
log.debug("Blob number %s was not in blobs", str(blob_num))
|
||||
else:
|
||||
logging.debug("delete_blob_after_finished is False")
|
||||
log.debug("delete_blob_after_finished is False")
|
||||
|
||||
|
||||
class FullStreamProgressManager(StreamProgressManager):
|
||||
@@ -115,6 +118,8 @@ class FullStreamProgressManager(StreamProgressManager):
|
||||
|
||||
def finished_outputting_blob():
|
||||
self.last_blob_outputted += 1
|
||||
|
||||
def check_if_finished():
|
||||
final_blob_num = self.download_manager.final_blob_num()
|
||||
if final_blob_num is not None and final_blob_num == self.last_blob_outputted:
|
||||
self._finished_outputting()
|
||||
@@ -126,14 +131,18 @@ class FullStreamProgressManager(StreamProgressManager):
|
||||
current_blob_num = self.last_blob_outputted + 1
|
||||
|
||||
if current_blob_num in blobs and blobs[current_blob_num].is_validated():
|
||||
logging.info("Outputting blob %s", str(self.last_blob_outputted + 1))
|
||||
log.info("Outputting blob %s", str(self.last_blob_outputted + 1))
|
||||
self.provided_blob_nums.append(self.last_blob_outputted + 1)
|
||||
d = self.download_manager.handle_blob(self.last_blob_outputted + 1)
|
||||
d.addCallback(lambda _: finished_outputting_blob())
|
||||
d.addCallback(lambda _: self._finished_with_blob(current_blob_num))
|
||||
d.addCallback(lambda _: check_if_finished())
|
||||
|
||||
def log_error(err):
|
||||
logging.warning("Error occurred in the output loop. Error: %s", err.getErrorMessage())
|
||||
log.warning("Error occurred in the output loop. Error: %s", err.getErrorMessage())
|
||||
if self.outputting_d is not None and not self.outputting_d.called:
|
||||
self.outputting_d.callback(True)
|
||||
self.outputting_d = None
|
||||
|
||||
d.addErrback(log_error)
|
||||
else:
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
import json
|
||||
import logging
|
||||
import logging.handlers
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
from requests_futures.sessions import FuturesSession
|
||||
|
||||
import lbrynet
|
||||
from lbrynet import conf
|
||||
from lbrynet.core import utils
|
||||
|
||||
session = FuturesSession()
|
||||
|
||||
|
||||
def bg_cb(sess, resp):
|
||||
""" Don't do anything with the response """
|
||||
pass
|
||||
|
||||
|
||||
class HTTPSHandler(logging.Handler):
|
||||
def __init__(self, url, fqdn=False, localname=None, facility=None):
|
||||
logging.Handler.__init__(self)
|
||||
self.url = url
|
||||
self.fqdn = fqdn
|
||||
self.localname = localname
|
||||
self.facility = facility
|
||||
|
||||
def get_full_message(self, record):
|
||||
if record.exc_info:
|
||||
return '\n'.join(traceback.format_exception(*record.exc_info))
|
||||
else:
|
||||
return record.getMessage()
|
||||
|
||||
def emit(self, record):
|
||||
try:
|
||||
payload = self.format(record)
|
||||
session.post(self.url, data=payload, background_callback=bg_cb)
|
||||
except (KeyboardInterrupt, SystemExit):
|
||||
raise
|
||||
except:
|
||||
self.handleError(record)
|
||||
|
||||
|
||||
DEFAULT_FORMAT = "%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s"
|
||||
DEFAULT_FORMATTER = logging.Formatter(DEFAULT_FORMAT)
|
||||
LOGGLY_URL = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}"
|
||||
|
||||
|
||||
def remove_handlers(log, handler_name):
|
||||
for handler in log.handlers:
|
||||
if handler.name == handler_name:
|
||||
log.removeHandler(handler)
|
||||
|
||||
|
||||
def _log_decorator(fn):
|
||||
def helper(*args, **kwargs):
|
||||
log = kwargs.pop('log', logging.getLogger())
|
||||
level = kwargs.pop('level', logging.INFO)
|
||||
if not isinstance(level, int):
|
||||
# despite the name, getLevelName returns
|
||||
# the numeric level when passed a text level
|
||||
level = logging.getLevelName(level)
|
||||
handler = fn(*args, **kwargs)
|
||||
if handler.name:
|
||||
remove_handlers(log, handler.name)
|
||||
handler.setLevel(level)
|
||||
log.addHandler(handler)
|
||||
if log.level > level:
|
||||
log.setLevel(level)
|
||||
return helper
|
||||
|
||||
|
||||
def disable_third_party_loggers():
|
||||
logging.getLogger('requests').setLevel(logging.WARNING)
|
||||
|
||||
|
||||
def disable_noisy_loggers():
|
||||
logging.getLogger('BitcoinRPC').setLevel(logging.INFO)
|
||||
logging.getLogger('lbrynet.analytics.api').setLevel(logging.INFO)
|
||||
logging.getLogger('lbrynet.core.client').setLevel(logging.INFO)
|
||||
logging.getLogger('lbrynet.core.server').setLevel(logging.INFO)
|
||||
logging.getLogger('lbrynet.dht').setLevel(logging.INFO)
|
||||
logging.getLogger('lbrynet.lbrynet_daemon').setLevel(logging.INFO)
|
||||
logging.getLogger('lbrynet.core.LBRYWallet').setLevel(logging.INFO)
|
||||
logging.getLogger('lbrynet.lbryfile').setLevel(logging.INFO)
|
||||
logging.getLogger('lbrynet.lbryfilemanager').setLevel(logging.INFO)
|
||||
|
||||
|
||||
@_log_decorator
|
||||
def configure_console(**kwargs):
|
||||
"""Convenience function to configure a logger that outputs to stdout"""
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setFormatter(DEFAULT_FORMATTER)
|
||||
handler.name = 'console'
|
||||
return handler
|
||||
|
||||
|
||||
@_log_decorator
|
||||
def configure_file_handler(file_name, **kwargs):
|
||||
handler = logging.handlers.RotatingFileHandler(file_name, maxBytes=2097152, backupCount=5)
|
||||
handler.setFormatter(DEFAULT_FORMATTER)
|
||||
handler.name = 'file'
|
||||
return handler
|
||||
|
||||
|
||||
def get_loggly_url(token=None, version=None):
|
||||
token = token or utils.deobfuscate(conf.LOGGLY_TOKEN)
|
||||
version = version or lbrynet.__version__
|
||||
return LOGGLY_URL.format(token=token, tag='lbrynet-' + version)
|
||||
|
||||
|
||||
@_log_decorator
|
||||
def configure_loggly_handler(url=None, **kwargs):
|
||||
url = url or get_loggly_url()
|
||||
json_format = {
|
||||
"loggerName": "%(name)s",
|
||||
"asciTime": "%(asctime)s",
|
||||
"fileName": "%(filename)s",
|
||||
"functionName": "%(funcName)s",
|
||||
"levelNo": "%(levelno)s",
|
||||
"lineNo": "%(lineno)d",
|
||||
"levelName": "%(levelname)s",
|
||||
"message": "%(message)s",
|
||||
}
|
||||
json_format.update(kwargs)
|
||||
formatter = logging.Formatter(json.dumps(json_format))
|
||||
handler = HTTPSHandler(url)
|
||||
handler.setFormatter(formatter)
|
||||
handler.name = 'loggly'
|
||||
return handler
|
||||
@@ -4,6 +4,9 @@ from zope.interface import implements
|
||||
from lbrynet.interfaces import IQueryHandlerFactory, IQueryHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobAvailabilityHandlerFactory(object):
|
||||
implements(IQueryHandlerFactory)
|
||||
|
||||
@@ -37,10 +40,11 @@ class BlobAvailabilityHandler(object):
|
||||
|
||||
def handle_queries(self, queries):
|
||||
if self.query_identifiers[0] in queries:
|
||||
logging.debug("Received the client's list of requested blobs")
|
||||
log.debug("Received the client's list of requested blobs")
|
||||
d = self._get_available_blobs(queries[self.query_identifiers[0]])
|
||||
|
||||
def set_field(available_blobs):
|
||||
log.debug("available blobs: %s", str(available_blobs))
|
||||
return {'available_blobs': available_blobs}
|
||||
|
||||
d.addCallback(set_field)
|
||||
|
||||
@@ -6,6 +6,9 @@ from zope.interface import implements
|
||||
from lbrynet.interfaces import IQueryHandlerFactory, IQueryHandler, IBlobSender
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobRequestHandlerFactory(object):
|
||||
implements(IQueryHandlerFactory)
|
||||
|
||||
@@ -58,7 +61,7 @@ class BlobRequestHandler(object):
|
||||
response['blob_data_payment_rate'] = 'RATE_ACCEPTED'
|
||||
|
||||
if self.query_identifiers[1] in queries:
|
||||
logging.debug("Received the client's request to send a blob")
|
||||
log.debug("Received the client's request to send a blob")
|
||||
response_fields = {}
|
||||
response['incoming_blob'] = response_fields
|
||||
|
||||
@@ -75,11 +78,11 @@ class BlobRequestHandler(object):
|
||||
if read_handle is not None:
|
||||
self.currently_uploading = blob
|
||||
self.read_handle = read_handle
|
||||
logging.debug("Sending %s to client", str(blob))
|
||||
log.info("Sending %s to client", str(blob))
|
||||
response_fields['blob_hash'] = blob.blob_hash
|
||||
response_fields['length'] = blob.length
|
||||
return response
|
||||
logging.debug("We can not send %s", str(blob))
|
||||
log.warning("We can not send %s", str(blob))
|
||||
response_fields['error'] = "BLOB_UNAVAILABLE"
|
||||
return response
|
||||
|
||||
@@ -129,14 +132,15 @@ class BlobRequestHandler(object):
|
||||
|
||||
def start_transfer():
|
||||
self.file_sender = FileSender()
|
||||
logging.info("Starting the file upload")
|
||||
log.debug("Starting the file upload")
|
||||
assert self.read_handle is not None, "self.read_handle was None when trying to start the transfer"
|
||||
d = self.file_sender.beginFileTransfer(self.read_handle, consumer, count_bytes)
|
||||
return d
|
||||
|
||||
def set_expected_payment():
|
||||
logging.info("Setting expected payment")
|
||||
log.debug("Setting expected payment")
|
||||
if self.blob_bytes_uploaded != 0 and self.blob_data_payment_rate is not None:
|
||||
# TODO: explain why 2**20
|
||||
self.wallet.add_expected_payment(self.peer,
|
||||
self.currently_uploading.length * 1.0 *
|
||||
self.blob_data_payment_rate / 2**20)
|
||||
@@ -151,6 +155,6 @@ class BlobRequestHandler(object):
|
||||
self.currently_uploading = None
|
||||
self.file_sender = None
|
||||
if reason is not None and isinstance(reason, Failure):
|
||||
logging.warning("Upload has failed. Reason: %s", reason.getErrorMessage())
|
||||
log.info("Upload has failed. Reason: %s", reason.getErrorMessage())
|
||||
|
||||
return _send_file()
|
||||
return _send_file()
|
||||
@@ -1,5 +1,5 @@
|
||||
import binascii
|
||||
from twisted.internet import defer, task, reactor
|
||||
from twisted.internet import defer, reactor
|
||||
import collections
|
||||
|
||||
|
||||
@@ -78,4 +78,4 @@ class DHTHashSupplier(object):
|
||||
self.hash_reannounce_time = 60 * 60 # 1 hour
|
||||
|
||||
def hashes_to_announce(self):
|
||||
pass
|
||||
pass
|
||||
@@ -6,6 +6,9 @@ from zope.interface import implements
|
||||
from lbrynet.core.server.ServerRequestHandler import ServerRequestHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ServerProtocol(Protocol):
|
||||
"""ServerProtocol needs to:
|
||||
|
||||
@@ -26,7 +29,7 @@ class ServerProtocol(Protocol):
|
||||
#Protocol stuff
|
||||
|
||||
def connectionMade(self):
|
||||
logging.debug("Got a connection")
|
||||
log.debug("Got a connection")
|
||||
peer_info = self.transport.getPeer()
|
||||
self.peer = self.factory.peer_manager.get_peer(peer_info.host, peer_info.port)
|
||||
self.request_handler = ServerRequestHandler(self)
|
||||
@@ -34,7 +37,7 @@ class ServerProtocol(Protocol):
|
||||
if enabled is True:
|
||||
query_handler = query_handler_factory.build_query_handler()
|
||||
query_handler.register_with_request_handler(self.request_handler, self.peer)
|
||||
logging.debug("Setting the request handler")
|
||||
log.debug("Setting the request handler")
|
||||
self.factory.rate_limiter.register_protocol(self)
|
||||
|
||||
def connectionLost(self, reason=failure.Failure(error.ConnectionDone())):
|
||||
@@ -42,10 +45,10 @@ class ServerProtocol(Protocol):
|
||||
self.request_handler.stopProducing()
|
||||
self.factory.rate_limiter.unregister_protocol(self)
|
||||
if not reason.check(error.ConnectionDone):
|
||||
logging.warning("Closing a connection. Reason: %s", reason.getErrorMessage())
|
||||
log.warning("Closing a connection. Reason: %s", reason.getErrorMessage())
|
||||
|
||||
def dataReceived(self, data):
|
||||
logging.debug("Receiving %s bytes of data from the transport", str(len(data)))
|
||||
log.debug("Receiving %s bytes of data from the transport", str(len(data)))
|
||||
self.factory.rate_limiter.report_dl_bytes(len(data))
|
||||
if self.request_handler is not None:
|
||||
self.request_handler.data_received(data)
|
||||
@@ -53,7 +56,7 @@ class ServerProtocol(Protocol):
|
||||
#IConsumer stuff
|
||||
|
||||
def registerProducer(self, producer, streaming):
|
||||
logging.debug("Registering the producer")
|
||||
log.debug("Registering the producer")
|
||||
assert streaming is True
|
||||
|
||||
def unregisterProducer(self):
|
||||
@@ -61,7 +64,7 @@ class ServerProtocol(Protocol):
|
||||
self.transport.loseConnection()
|
||||
|
||||
def write(self, data):
|
||||
logging.debug("Writing %s bytes of data to the transport", str(len(data)))
|
||||
log.debug("Writing %s bytes of data to the transport", str(len(data)))
|
||||
self.transport.write(data)
|
||||
self.factory.rate_limiter.report_ul_bytes(len(data))
|
||||
|
||||
|
||||
@@ -5,6 +5,9 @@ from zope.interface import implements
|
||||
from lbrynet.interfaces import IRequestHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ServerRequestHandler(object):
|
||||
"""This class handles requests from clients. It can upload blobs and return request for information about
|
||||
more blobs that are associated with streams"""
|
||||
@@ -52,7 +55,7 @@ class ServerRequestHandler(object):
|
||||
chunk = self.response_buff[:self.CHUNK_SIZE]
|
||||
self.response_buff = self.response_buff[self.CHUNK_SIZE:]
|
||||
if chunk != '':
|
||||
logging.debug("writing %s bytes to the client", str(len(chunk)))
|
||||
log.debug("writing %s bytes to the client", str(len(chunk)))
|
||||
self.consumer.write(chunk)
|
||||
reactor.callLater(0, self._produce_more)
|
||||
|
||||
@@ -76,7 +79,7 @@ class ServerRequestHandler(object):
|
||||
|
||||
def get_more_data():
|
||||
if self.producer is not None:
|
||||
logging.debug("Requesting more data from the producer")
|
||||
log.debug("Requesting more data from the producer")
|
||||
self.producer.resumeProducing()
|
||||
|
||||
reactor.callLater(0, get_more_data)
|
||||
@@ -84,8 +87,8 @@ class ServerRequestHandler(object):
|
||||
#From Protocol
|
||||
|
||||
def data_received(self, data):
|
||||
logging.debug("Received data")
|
||||
logging.debug("%s", str(data))
|
||||
log.debug("Received data")
|
||||
log.debug("%s", str(data))
|
||||
if self.request_received is False:
|
||||
self.request_buff = self.request_buff + data
|
||||
msg = self.try_to_parse_request(self.request_buff)
|
||||
@@ -96,10 +99,10 @@ class ServerRequestHandler(object):
|
||||
d.addCallback(lambda _: self.blob_sender.send_blob_if_requested(self))
|
||||
d.addCallbacks(lambda _: self.finished_response(), self.request_failure_handler)
|
||||
else:
|
||||
logging.info("Request buff not a valid json message")
|
||||
logging.info("Request buff: %s", str(self.request_buff))
|
||||
log.debug("Request buff not a valid json message")
|
||||
log.debug("Request buff: %s", str(self.request_buff))
|
||||
else:
|
||||
logging.warning("The client sent data when we were uploading a file. This should not happen")
|
||||
log.warning("The client sent data when we were uploading a file. This should not happen")
|
||||
|
||||
######### IRequestHandler #########
|
||||
|
||||
@@ -112,7 +115,7 @@ class ServerRequestHandler(object):
|
||||
#response handling
|
||||
|
||||
def request_failure_handler(self, err):
|
||||
logging.warning("An error occurred handling a request. Error: %s", err.getErrorMessage())
|
||||
log.warning("An error occurred handling a request. Error: %s", err.getErrorMessage())
|
||||
self.stopProducing()
|
||||
return err
|
||||
|
||||
@@ -122,15 +125,15 @@ class ServerRequestHandler(object):
|
||||
|
||||
def send_response(self, msg):
|
||||
m = json.dumps(msg)
|
||||
logging.info("Sending a response of length %s", str(len(m)))
|
||||
logging.debug("Response: %s", str(m))
|
||||
log.debug("Sending a response of length %s", str(len(m)))
|
||||
log.debug("Response: %s", str(m))
|
||||
self.response_buff = self.response_buff + m
|
||||
self._produce_more()
|
||||
return True
|
||||
|
||||
def handle_request(self, msg):
|
||||
logging.debug("Handling a request")
|
||||
logging.debug(str(msg))
|
||||
log.debug("Handling a request")
|
||||
log.debug(str(msg))
|
||||
|
||||
def create_response_message(results):
|
||||
response = {}
|
||||
@@ -140,11 +143,11 @@ class ServerRequestHandler(object):
|
||||
else:
|
||||
# result is a Failure
|
||||
return result
|
||||
logging.debug("Finished making the response message. Response: %s", str(response))
|
||||
log.debug("Finished making the response message. Response: %s", str(response))
|
||||
return response
|
||||
|
||||
def log_errors(err):
|
||||
logging.warning("An error occurred handling a client request. Error message: %s", err.getErrorMessage())
|
||||
log.warning("An error occurred handling a client request. Error message: %s", err.getErrorMessage())
|
||||
return err
|
||||
|
||||
def send_response(response):
|
||||
@@ -168,4 +171,4 @@ class ServerRequestHandler(object):
|
||||
msg = json.loads(request_buff)
|
||||
return msg
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
@@ -0,0 +1,23 @@
|
||||
import sqlite3
|
||||
from twisted.internet import task, reactor
|
||||
import logging
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def rerun_if_locked(f):
|
||||
|
||||
def rerun(err, *args, **kwargs):
|
||||
if err.check(sqlite3.OperationalError) and err.value.message == "database is locked":
|
||||
log.warning("database was locked. rerunning %s with args %s, kwargs %s",
|
||||
str(f), str(args), str(kwargs))
|
||||
return task.deferLater(reactor, 0, wrapper, *args, **kwargs)
|
||||
return err
|
||||
|
||||
def wrapper(*args, **kwargs):
|
||||
d = f(*args, **kwargs)
|
||||
d.addErrback(rerun, *args, **kwargs)
|
||||
return d
|
||||
|
||||
return wrapper
|
||||
+22
-2
@@ -1,6 +1,10 @@
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
import base64
|
||||
import distutils.version
|
||||
import random
|
||||
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
|
||||
|
||||
blobhash_length = get_lbry_hash_obj().digest_size * 2 # digest_size is in bytes, and blob hashes are hex encoded
|
||||
|
||||
@@ -25,4 +29,20 @@ def is_valid_blobhash(blobhash):
|
||||
for l in blobhash:
|
||||
if l not in "0123456789abcdef":
|
||||
return False
|
||||
return True
|
||||
return True
|
||||
|
||||
|
||||
def version_is_greater_than(a, b):
|
||||
"""Returns True if version a is more recent than version b"""
|
||||
try:
|
||||
return distutils.version.StrictVersion(a) > distutils.version.StrictVersion(b)
|
||||
except ValueError:
|
||||
return distutils.version.LooseVersion(a) > distutils.version.LooseVersion(b)
|
||||
|
||||
|
||||
def deobfuscate(obfustacated):
|
||||
return base64.b64decode(obfustacated.decode('rot13'))
|
||||
|
||||
|
||||
def obfuscate(plain):
|
||||
return base64.b64encode(plain).encode('rot13')
|
||||
@@ -5,6 +5,9 @@ from lbrynet.conf import BLOB_SIZE
|
||||
from lbrynet.core.BlobInfo import BlobInfo
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CryptBlobInfo(BlobInfo):
|
||||
def __init__(self, blob_hash, blob_num, length, iv):
|
||||
BlobInfo.__init__(self, blob_hash, blob_num, length)
|
||||
@@ -78,12 +81,12 @@ class CryptStreamBlobMaker(object):
|
||||
return done, num_bytes_to_write
|
||||
|
||||
def close(self):
|
||||
logging.debug("closing blob %s with plaintext len %s", str(self.blob_num), str(self.length))
|
||||
log.debug("closing blob %s with plaintext len %s", str(self.blob_num), str(self.length))
|
||||
if self.length != 0:
|
||||
self._close_buffer()
|
||||
d = self.blob.close()
|
||||
d.addCallback(self._return_info)
|
||||
logging.debug("called the finished_callback from CryptStreamBlobMaker.close")
|
||||
log.debug("called the finished_callback from CryptStreamBlobMaker.close")
|
||||
return d
|
||||
|
||||
def _write_buffer(self):
|
||||
|
||||
@@ -12,6 +12,9 @@ from lbrynet.core.StreamCreator import StreamCreator
|
||||
from lbrynet.cryptstream.CryptBlob import CryptStreamBlobMaker
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CryptStreamCreator(StreamCreator):
|
||||
"""Create a new stream with blobs encrypted by a symmetric cipher.
|
||||
|
||||
@@ -57,18 +60,18 @@ class CryptStreamCreator(StreamCreator):
|
||||
return defer.succeed(True)
|
||||
|
||||
def _finalize(self):
|
||||
logging.debug("_finalize has been called")
|
||||
log.debug("_finalize has been called")
|
||||
self.blob_count += 1
|
||||
iv = self.iv_generator.next()
|
||||
final_blob_creator = self.blob_manager.get_blob_creator()
|
||||
logging.debug("Created the finished_deferred")
|
||||
log.debug("Created the finished_deferred")
|
||||
final_blob = self._get_blob_maker(iv, final_blob_creator)
|
||||
logging.debug("Created the final blob")
|
||||
logging.debug("Calling close on final blob")
|
||||
log.debug("Created the final blob")
|
||||
log.debug("Calling close on final blob")
|
||||
d = final_blob.close()
|
||||
d.addCallback(self._blob_finished)
|
||||
self.finished_deferreds.append(d)
|
||||
logging.debug("called close on final blob, returning from make_final_blob")
|
||||
log.debug("called close on final blob, returning from make_final_blob")
|
||||
return d
|
||||
|
||||
def _write(self, data):
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import logging
|
||||
from zope.interface import implements
|
||||
from lbrynet.interfaces import IStreamDownloader
|
||||
from lbrynet.core.client.BlobRequester import BlobRequester
|
||||
@@ -9,6 +10,9 @@ from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StartFailedError(Exception):
|
||||
pass
|
||||
|
||||
@@ -71,6 +75,9 @@ class CryptStreamDownloader(object):
|
||||
|
||||
self.points_paid = 0.0
|
||||
|
||||
def __str__(self):
|
||||
return str(self.stream_name)
|
||||
|
||||
def toggle_running(self):
|
||||
if self.stopped is True:
|
||||
return self.start()
|
||||
@@ -79,10 +86,6 @@ class CryptStreamDownloader(object):
|
||||
|
||||
def start(self):
|
||||
|
||||
def set_finished_deferred():
|
||||
self.finished_deferred = defer.Deferred()
|
||||
return self.finished_deferred
|
||||
|
||||
if self.starting is True:
|
||||
raise CurrentlyStartingError()
|
||||
if self.stopping is True:
|
||||
@@ -92,11 +95,13 @@ class CryptStreamDownloader(object):
|
||||
assert self.download_manager is None
|
||||
self.starting = True
|
||||
self.completed = False
|
||||
self.finished_deferred = defer.Deferred()
|
||||
fd = self.finished_deferred
|
||||
d = self._start()
|
||||
d.addCallback(lambda _: set_finished_deferred())
|
||||
d.addCallback(lambda _: fd)
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
def stop(self, err=None):
|
||||
|
||||
def check_if_stop_succeeded(success):
|
||||
self.stopping = False
|
||||
@@ -112,8 +117,8 @@ class CryptStreamDownloader(object):
|
||||
assert self.download_manager is not None
|
||||
self.stopping = True
|
||||
d = self.download_manager.stop_downloading()
|
||||
self._fire_completed_deferred()
|
||||
d.addCallback(check_if_stop_succeeded)
|
||||
d.addCallback(lambda _: self._fire_completed_deferred(err))
|
||||
return d
|
||||
|
||||
def _start_failed(self):
|
||||
@@ -196,10 +201,15 @@ class CryptStreamDownloader(object):
|
||||
self._get_primary_request_creators(download_manager),
|
||||
self._get_secondary_request_creators(download_manager))
|
||||
|
||||
def _fire_completed_deferred(self):
|
||||
def _fire_completed_deferred(self, err=None):
|
||||
self.finished_deferred, d = None, self.finished_deferred
|
||||
if d is not None:
|
||||
d.callback(self._get_finished_deferred_callback_value())
|
||||
if err is not None:
|
||||
d.errback(err)
|
||||
else:
|
||||
d.callback(self._get_finished_deferred_callback_value())
|
||||
else:
|
||||
log.debug("Not firing the completed deferred because d is None")
|
||||
|
||||
def _get_finished_deferred_callback_value(self):
|
||||
return None
|
||||
@@ -209,5 +219,5 @@ class CryptStreamDownloader(object):
|
||||
self.completed = True
|
||||
return self.stop()
|
||||
|
||||
def insufficient_funds(self):
|
||||
return self.stop()
|
||||
def insufficient_funds(self, err):
|
||||
return self.stop(err=err)
|
||||
Whitespace-only changes.
@@ -0,0 +1,28 @@
|
||||
import logging
|
||||
# import os
|
||||
|
||||
|
||||
def migrate_db(db_dir, start, end):
|
||||
current = start
|
||||
old_dirs = []
|
||||
# if os.name == "nt":
|
||||
# return old_dirs
|
||||
# while current < end:
|
||||
# if current == 0:
|
||||
# from lbrynet.db_migrator.migrate0to1 import do_migration
|
||||
# old_dirs.append(do_migration(db_dir))
|
||||
# current += 1
|
||||
return old_dirs
|
||||
|
||||
|
||||
def run_migration_script():
|
||||
import sys
|
||||
log_format = "(%(asctime)s)[%(filename)s:%(lineno)s] %(funcName)s(): %(message)s"
|
||||
logging.basicConfig(level=logging.DEBUG, format=log_format, filename="migrator.log")
|
||||
sys.stdout = open("migrator.out.log", 'w')
|
||||
sys.stderr = open("migrator.err.log", 'w')
|
||||
migrate_db(sys.argv[1], int(sys.argv[2]), int(sys.argv[3]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_migration_script()
|
||||
@@ -0,0 +1,307 @@
|
||||
# import sqlite3
|
||||
# import unqlite
|
||||
# import leveldb
|
||||
# import shutil
|
||||
# import os
|
||||
# import logging
|
||||
# import json
|
||||
#
|
||||
#
|
||||
# log = logging.getLogger(__name__)
|
||||
#
|
||||
#
|
||||
# known_dbs = ['lbryfile_desc.db', 'lbryfiles.db', 'valuable_blobs.db', 'blobs.db',
|
||||
# 'lbryfile_blob.db', 'lbryfile_info.db', 'settings.db', 'blind_settings.db',
|
||||
# 'blind_peers.db', 'blind_info.db', 'lbryfile_info.db', 'lbryfile_manager.db',
|
||||
# 'live_stream.db', 'stream_info.db', 'stream_blob.db', 'stream_desc.db']
|
||||
#
|
||||
#
|
||||
# def do_move(from_dir, to_dir):
|
||||
# for known_db in known_dbs:
|
||||
# known_db_path = os.path.join(from_dir, known_db)
|
||||
# if os.path.exists(known_db_path):
|
||||
# log.debug("Moving %s to %s",
|
||||
# os.path.abspath(known_db_path),
|
||||
# os.path.abspath(os.path.join(to_dir, known_db)))
|
||||
# shutil.move(known_db_path, os.path.join(to_dir, known_db))
|
||||
# else:
|
||||
# log.debug("Did not find %s", os.path.abspath(known_db_path))
|
||||
#
|
||||
#
|
||||
# def do_migration(db_dir):
|
||||
# old_dir = os.path.join(db_dir, "_0_to_1_old")
|
||||
# new_dir = os.path.join(db_dir, "_0_to_1_new")
|
||||
# try:
|
||||
# log.info("Moving dbs from the real directory to %s", os.path.abspath(old_dir))
|
||||
# os.makedirs(old_dir)
|
||||
# do_move(db_dir, old_dir)
|
||||
# except:
|
||||
# log.error("An error occurred moving the old db files.")
|
||||
# raise
|
||||
# try:
|
||||
# log.info("Creating the new directory in %s", os.path.abspath(new_dir))
|
||||
# os.makedirs(new_dir)
|
||||
#
|
||||
# except:
|
||||
# log.error("An error occurred creating the new directory.")
|
||||
# raise
|
||||
# try:
|
||||
# log.info("Doing the migration")
|
||||
# migrate_blob_db(old_dir, new_dir)
|
||||
# migrate_lbryfile_db(old_dir, new_dir)
|
||||
# migrate_livestream_db(old_dir, new_dir)
|
||||
# migrate_ptc_db(old_dir, new_dir)
|
||||
# migrate_lbryfile_manager_db(old_dir, new_dir)
|
||||
# migrate_settings_db(old_dir, new_dir)
|
||||
# migrate_repeater_db(old_dir, new_dir)
|
||||
# log.info("Migration succeeded")
|
||||
# except:
|
||||
# log.error("An error occurred during the migration. Restoring.")
|
||||
# do_move(old_dir, db_dir)
|
||||
# raise
|
||||
# try:
|
||||
# log.info("Moving dbs in the new directory to the real directory")
|
||||
# do_move(new_dir, db_dir)
|
||||
# db_revision = open(os.path.join(db_dir, 'db_revision'), mode='w+')
|
||||
# db_revision.write("1")
|
||||
# db_revision.close()
|
||||
# os.rmdir(new_dir)
|
||||
# except:
|
||||
# log.error("An error occurred moving the new db files.")
|
||||
# raise
|
||||
# return old_dir
|
||||
#
|
||||
#
|
||||
# def migrate_blob_db(old_db_dir, new_db_dir):
|
||||
# old_blob_db_path = os.path.join(old_db_dir, "blobs.db")
|
||||
# if not os.path.exists(old_blob_db_path):
|
||||
# return True
|
||||
#
|
||||
# old_db = leveldb.LevelDB(old_blob_db_path)
|
||||
# new_db_conn = sqlite3.connect(os.path.join(new_db_dir, "blobs.db"))
|
||||
# c = new_db_conn.cursor()
|
||||
# c.execute("create table if not exists blobs (" +
|
||||
# " blob_hash text primary key, " +
|
||||
# " blob_length integer, " +
|
||||
# " last_verified_time real, " +
|
||||
# " next_announce_time real"
|
||||
# ")")
|
||||
# new_db_conn.commit()
|
||||
# c = new_db_conn.cursor()
|
||||
# for blob_hash, blob_info in old_db.RangeIter():
|
||||
# blob_length, verified_time, announce_time = json.loads(blob_info)
|
||||
# c.execute("insert into blobs values (?, ?, ?, ?)",
|
||||
# (blob_hash, blob_length, verified_time, announce_time))
|
||||
# new_db_conn.commit()
|
||||
# new_db_conn.close()
|
||||
#
|
||||
#
|
||||
# def migrate_lbryfile_db(old_db_dir, new_db_dir):
|
||||
# old_lbryfile_db_path = os.path.join(old_db_dir, "lbryfiles.db")
|
||||
# if not os.path.exists(old_lbryfile_db_path):
|
||||
# return True
|
||||
#
|
||||
# stream_info_db = leveldb.LevelDB(os.path.join(old_db_dir, "lbryfile_info.db"))
|
||||
# stream_blob_db = leveldb.LevelDB(os.path.join(old_db_dir, "lbryfile_blob.db"))
|
||||
# stream_desc_db = leveldb.LevelDB(os.path.join(old_db_dir, "lbryfile_desc.db"))
|
||||
#
|
||||
# db_conn = sqlite3.connect(os.path.join(new_db_dir, "lbryfile_info.db"))
|
||||
# c = db_conn.cursor()
|
||||
# c.execute("create table if not exists lbry_files (" +
|
||||
# " stream_hash text primary key, " +
|
||||
# " key text, " +
|
||||
# " stream_name text, " +
|
||||
# " suggested_file_name text" +
|
||||
# ")")
|
||||
# c.execute("create table if not exists lbry_file_blobs (" +
|
||||
# " blob_hash text, " +
|
||||
# " stream_hash text, " +
|
||||
# " position integer, " +
|
||||
# " iv text, " +
|
||||
# " length integer, " +
|
||||
# " foreign key(stream_hash) references lbry_files(stream_hash)" +
|
||||
# ")")
|
||||
# c.execute("create table if not exists lbry_file_descriptors (" +
|
||||
# " sd_blob_hash TEXT PRIMARY KEY, " +
|
||||
# " stream_hash TEXT, " +
|
||||
# " foreign key(stream_hash) references lbry_files(stream_hash)" +
|
||||
# ")")
|
||||
# db_conn.commit()
|
||||
# c = db_conn.cursor()
|
||||
# for stream_hash, stream_info in stream_info_db.RangeIter():
|
||||
# key, name, suggested_file_name = json.loads(stream_info)
|
||||
# c.execute("insert into lbry_files values (?, ?, ?, ?)",
|
||||
# (stream_hash, key, name, suggested_file_name))
|
||||
# db_conn.commit()
|
||||
# c = db_conn.cursor()
|
||||
# for blob_hash_stream_hash, blob_info in stream_blob_db.RangeIter():
|
||||
# b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
# position, iv, length = json.loads(blob_info)
|
||||
# c.execute("insert into lbry_file_blobs values (?, ?, ?, ?, ?)",
|
||||
# (b_h, s_h, position, iv, length))
|
||||
# db_conn.commit()
|
||||
# c = db_conn.cursor()
|
||||
# for sd_blob_hash, stream_hash in stream_desc_db.RangeIter():
|
||||
# c.execute("insert into lbry_file_descriptors values (?, ?)",
|
||||
# (sd_blob_hash, stream_hash))
|
||||
# db_conn.commit()
|
||||
# db_conn.close()
|
||||
#
|
||||
#
|
||||
# def migrate_livestream_db(old_db_dir, new_db_dir):
|
||||
# old_db_path = os.path.join(old_db_dir, "stream_info.db")
|
||||
# if not os.path.exists(old_db_path):
|
||||
# return True
|
||||
# stream_info_db = leveldb.LevelDB(os.path.join(old_db_dir, "stream_info.db"))
|
||||
# stream_blob_db = leveldb.LevelDB(os.path.join(old_db_dir, "stream_blob.db"))
|
||||
# stream_desc_db = leveldb.LevelDB(os.path.join(old_db_dir, "stream_desc.db"))
|
||||
#
|
||||
# db_conn = sqlite3.connect(os.path.join(new_db_dir, "live_stream.db"))
|
||||
#
|
||||
# c = db_conn.cursor()
|
||||
#
|
||||
# c.execute("create table if not exists live_streams (" +
|
||||
# " stream_hash text primary key, " +
|
||||
# " public_key text, " +
|
||||
# " key text, " +
|
||||
# " stream_name text, " +
|
||||
# " next_announce_time real" +
|
||||
# ")")
|
||||
# c.execute("create table if not exists live_stream_blobs (" +
|
||||
# " blob_hash text, " +
|
||||
# " stream_hash text, " +
|
||||
# " position integer, " +
|
||||
# " revision integer, " +
|
||||
# " iv text, " +
|
||||
# " length integer, " +
|
||||
# " signature text, " +
|
||||
# " foreign key(stream_hash) references live_streams(stream_hash)" +
|
||||
# ")")
|
||||
# c.execute("create table if not exists live_stream_descriptors (" +
|
||||
# " sd_blob_hash TEXT PRIMARY KEY, " +
|
||||
# " stream_hash TEXT, " +
|
||||
# " foreign key(stream_hash) references live_streams(stream_hash)" +
|
||||
# ")")
|
||||
#
|
||||
# db_conn.commit()
|
||||
#
|
||||
# c = db_conn.cursor()
|
||||
# for stream_hash, stream_info in stream_info_db.RangeIter():
|
||||
# public_key, key, name, next_announce_time = json.loads(stream_info)
|
||||
# c.execute("insert into live_streams values (?, ?, ?, ?, ?)",
|
||||
# (stream_hash, public_key, key, name, next_announce_time))
|
||||
# db_conn.commit()
|
||||
# c = db_conn.cursor()
|
||||
# for blob_hash_stream_hash, blob_info in stream_blob_db.RangeIter():
|
||||
# b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
# position, revision, iv, length, signature = json.loads(blob_info)
|
||||
# c.execute("insert into live_stream_blobs values (?, ?, ?, ?, ?, ?, ?)",
|
||||
# (b_h, s_h, position, revision, iv, length, signature))
|
||||
# db_conn.commit()
|
||||
# c = db_conn.cursor()
|
||||
# for sd_blob_hash, stream_hash in stream_desc_db.RangeIter():
|
||||
# c.execute("insert into live_stream_descriptors values (?, ?)",
|
||||
# (sd_blob_hash, stream_hash))
|
||||
# db_conn.commit()
|
||||
# db_conn.close()
|
||||
#
|
||||
#
|
||||
# def migrate_ptc_db(old_db_dir, new_db_dir):
|
||||
# old_db_path = os.path.join(old_db_dir, "ptcwallet.db")
|
||||
# if not os.path.exists(old_db_path):
|
||||
# return True
|
||||
# old_db = leveldb.LevelDB(old_db_path)
|
||||
# try:
|
||||
# p_key = old_db.Get("private_key")
|
||||
# new_db = unqlite.UnQLite(os.path.join(new_db_dir, "ptcwallet.db"))
|
||||
# new_db['private_key'] = p_key
|
||||
# except KeyError:
|
||||
# pass
|
||||
#
|
||||
#
|
||||
# def migrate_lbryfile_manager_db(old_db_dir, new_db_dir):
|
||||
# old_db_path = os.path.join(old_db_dir, "lbryfiles.db")
|
||||
# if not os.path.exists(old_db_path):
|
||||
# return True
|
||||
# old_db = leveldb.LevelDB(old_db_path)
|
||||
# new_db = sqlite3.connect(os.path.join(new_db_dir, "lbryfile_info.db"))
|
||||
# c = new_db.cursor()
|
||||
# c.execute("create table if not exists lbry_file_options (" +
|
||||
# " blob_data_rate real, " +
|
||||
# " status text," +
|
||||
# " stream_hash text,"
|
||||
# " foreign key(stream_hash) references lbry_files(stream_hash)" +
|
||||
# ")")
|
||||
# new_db.commit()
|
||||
# LBRYFILE_STATUS = "t"
|
||||
# LBRYFILE_OPTIONS = "o"
|
||||
# c = new_db.cursor()
|
||||
# for k, v in old_db.RangeIter():
|
||||
# key_type, stream_hash = json.loads(k)
|
||||
# if key_type == LBRYFILE_STATUS:
|
||||
# try:
|
||||
# rate = json.loads(old_db.Get(json.dumps((LBRYFILE_OPTIONS, stream_hash))))[0]
|
||||
# except KeyError:
|
||||
# rate = None
|
||||
# c.execute("insert into lbry_file_options values (?, ?, ?)",
|
||||
# (rate, v, stream_hash))
|
||||
# new_db.commit()
|
||||
# new_db.close()
|
||||
#
|
||||
#
|
||||
# def migrate_settings_db(old_db_dir, new_db_dir):
|
||||
# old_settings_db_path = os.path.join(old_db_dir, "settings.db")
|
||||
# if not os.path.exists(old_settings_db_path):
|
||||
# return True
|
||||
# old_db = leveldb.LevelDB(old_settings_db_path)
|
||||
# new_db = unqlite.UnQLite(os.path.join(new_db_dir, "settings.db"))
|
||||
# for k, v in old_db.RangeIter():
|
||||
# new_db[k] = v
|
||||
#
|
||||
#
|
||||
# def migrate_repeater_db(old_db_dir, new_db_dir):
|
||||
# old_repeater_db_path = os.path.join(old_db_dir, "valuable_blobs.db")
|
||||
# if not os.path.exists(old_repeater_db_path):
|
||||
# return True
|
||||
# old_db = leveldb.LevelDB(old_repeater_db_path)
|
||||
# info_db = sqlite3.connect(os.path.join(new_db_dir, "blind_info.db"))
|
||||
# peer_db = sqlite3.connect(os.path.join(new_db_dir, "blind_peers.db"))
|
||||
# unql_db = unqlite.UnQLite(os.path.join(new_db_dir, "blind_settings.db"))
|
||||
# BLOB_INFO_TYPE = 'b'
|
||||
# SETTING_TYPE = 's'
|
||||
# PEER_TYPE = 'p'
|
||||
# info_c = info_db.cursor()
|
||||
# info_c.execute("create table if not exists valuable_blobs (" +
|
||||
# " blob_hash text primary key, " +
|
||||
# " blob_length integer, " +
|
||||
# " reference text, " +
|
||||
# " peer_host text, " +
|
||||
# " peer_port integer, " +
|
||||
# " peer_score text" +
|
||||
# ")")
|
||||
# info_db.commit()
|
||||
# peer_c = peer_db.cursor()
|
||||
# peer_c.execute("create table if not exists approved_peers (" +
|
||||
# " ip_address text, " +
|
||||
# " port integer" +
|
||||
# ")")
|
||||
# peer_db.commit()
|
||||
# info_c = info_db.cursor()
|
||||
# peer_c = peer_db.cursor()
|
||||
# for k, v in old_db.RangeIter():
|
||||
# key_type, key_rest = json.loads(k)
|
||||
# if key_type == PEER_TYPE:
|
||||
# host, port = key_rest
|
||||
# peer_c.execute("insert into approved_peers values (?, ?)",
|
||||
# (host, port))
|
||||
# elif key_type == SETTING_TYPE:
|
||||
# unql_db[key_rest] = v
|
||||
# elif key_type == BLOB_INFO_TYPE:
|
||||
# blob_hash = key_rest
|
||||
# length, reference, peer_host, peer_port, peer_score = json.loads(v)
|
||||
# info_c.execute("insert into valuable_blobs values (?, ?, ?, ?, ?, ?)",
|
||||
# (blob_hash, length, reference, peer_host, peer_port, peer_score))
|
||||
# info_db.commit()
|
||||
# peer_db.commit()
|
||||
# info_db.close()
|
||||
# peer_db.close()
|
||||
@@ -8,10 +8,7 @@
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
|
||||
import UserDict
|
||||
#import sqlite3
|
||||
import cPickle as pickle
|
||||
import time
|
||||
import os
|
||||
import constants
|
||||
|
||||
|
||||
|
||||
@@ -99,7 +99,10 @@ class Bencode(Encoding):
|
||||
"""
|
||||
if len(data) == 0:
|
||||
raise DecodeError, 'Cannot decode empty string'
|
||||
return self._decodeRecursive(data)[0]
|
||||
try:
|
||||
return self._decodeRecursive(data)[0]
|
||||
except ValueError as e:
|
||||
raise DecodeError, e.message
|
||||
|
||||
@staticmethod
|
||||
def _decodeRecursive(data, startIndex=0):
|
||||
|
||||
+20
-11
@@ -7,9 +7,9 @@
|
||||
# The docstrings in this module contain epytext markup; API documentation
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
|
||||
import hashlib, random, struct, time, math, binascii
|
||||
import hashlib, random, struct, time, binascii
|
||||
import argparse
|
||||
from twisted.internet import defer
|
||||
from twisted.internet import defer, error
|
||||
import constants
|
||||
import routingtable
|
||||
import datastore
|
||||
@@ -22,6 +22,9 @@ from hashwatcher import HashWatcher
|
||||
import logging
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def rpcmethod(func):
|
||||
""" Decorator to expose Node methods as remote procedure calls
|
||||
|
||||
@@ -139,7 +142,13 @@ class Node(object):
|
||||
"""
|
||||
# Prepare the underlying Kademlia protocol
|
||||
if self.port is not None:
|
||||
self._listeningPort = twisted.internet.reactor.listenUDP(self.port, self._protocol) #IGNORE:E1101
|
||||
try:
|
||||
self._listeningPort = twisted.internet.reactor.listenUDP(self.port, self._protocol)
|
||||
except error.CannotListenError as e:
|
||||
import traceback
|
||||
log.error("Couldn't bind to port %d. %s", self.port, traceback.format_exc())
|
||||
raise ValueError("%s lbrynet may already be running." % str(e))
|
||||
#IGNORE:E1101
|
||||
# Create temporary contact information for the list of addresses of known nodes
|
||||
if knownNodeAddresses != None:
|
||||
bootstrapContacts = []
|
||||
@@ -231,12 +240,12 @@ class Node(object):
|
||||
known_nodes = {}
|
||||
|
||||
def log_error(err, n):
|
||||
logging.error("error storing blob_hash %s at %s", binascii.hexlify(blob_hash), str(n))
|
||||
logging.error(binascii.hexlify(err.getErrorMessage()))
|
||||
logging.error(err.getTraceback())
|
||||
log.debug("error storing blob_hash %s at %s", binascii.hexlify(blob_hash), str(n))
|
||||
log.debug(err.getErrorMessage())
|
||||
log.debug(err.getTraceback())
|
||||
|
||||
def log_success(res):
|
||||
logging.debug("Response to store request: %s", str(res))
|
||||
log.debug("Response to store request: %s", str(res))
|
||||
return res
|
||||
|
||||
def announce_to_peer(responseTuple):
|
||||
@@ -332,7 +341,7 @@ class Node(object):
|
||||
# originalPublisherID = self.id
|
||||
#
|
||||
# def log_error(err):
|
||||
# logging.error(err.getErrorMessage())
|
||||
# log.error(err.getErrorMessage())
|
||||
#
|
||||
# # Prepare a callback for doing "STORE" RPC calls
|
||||
# def executeStoreRPCs(nodes):
|
||||
@@ -507,7 +516,7 @@ class Node(object):
|
||||
else:
|
||||
raise TypeError, 'No NodeID given. Therefore we can\'t store this node'
|
||||
|
||||
if self_store is True and self.externalIP is not None:
|
||||
if self_store is True and self.externalIP:
|
||||
contact = Contact(self.id, self.externalIP, self.port, None, None)
|
||||
compact_ip = contact.compact_ip()
|
||||
elif '_rpcNodeContact' in kwargs:
|
||||
@@ -756,7 +765,7 @@ class Node(object):
|
||||
searchIteration()
|
||||
|
||||
def log_error(err):
|
||||
logging.error(err.getErrorMessage())
|
||||
log.error(err.getErrorMessage())
|
||||
|
||||
# Send parallel, asynchronous FIND_NODE RPCs to the shortlist of contacts
|
||||
def searchIteration():
|
||||
@@ -1008,4 +1017,4 @@ def main():
|
||||
twisted.internet.reactor.run()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
main()
|
||||
@@ -7,6 +7,7 @@
|
||||
# The docstrings in this module contain epytext markup; API documentation
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
|
||||
import binascii
|
||||
import time
|
||||
|
||||
from twisted.internet import protocol, defer
|
||||
@@ -23,6 +24,13 @@ reactor = twisted.internet.reactor
|
||||
|
||||
class TimeoutError(Exception):
|
||||
""" Raised when a RPC times out """
|
||||
def __init__(self, remote_contact_id):
|
||||
# remote_contact_id is a binary blob so we need to convert it
|
||||
# into something more readable
|
||||
msg = 'Timeout connecting to {}'.format(binascii.hexlify(remote_contact_id))
|
||||
Exception.__init__(self, msg)
|
||||
self.remote_contact_id = remote_contact_id
|
||||
|
||||
|
||||
class KademliaProtocol(protocol.DatagramProtocol):
|
||||
""" Implements all low-level network-related functions of a Kademlia node """
|
||||
|
||||
+12
-9
@@ -6,6 +6,9 @@ import sys
|
||||
from lbrynet.core.utils import generate_id
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def print_usage():
|
||||
print "Usage:\n%s UDP_PORT KNOWN_NODE_IP KNOWN_NODE_PORT HASH"
|
||||
|
||||
@@ -13,15 +16,15 @@ def print_usage():
|
||||
def join_network(udp_port, known_nodes):
|
||||
lbryid = generate_id()
|
||||
|
||||
logging.info('Creating Node...')
|
||||
log.info('Creating Node...')
|
||||
node = Node(udpPort=udp_port, lbryid=lbryid)
|
||||
|
||||
logging.info('Joining network...')
|
||||
log.info('Joining network...')
|
||||
d = node.joinNetwork(known_nodes)
|
||||
|
||||
def log_network_size():
|
||||
logging.info("Approximate number of nodes in DHT: %s", str(node.getApproximateTotalDHTNodes()))
|
||||
logging.info("Approximate number of blobs in DHT: %s", str(node.getApproximateTotalHashes()))
|
||||
log.info("Approximate number of nodes in DHT: %s", str(node.getApproximateTotalDHTNodes()))
|
||||
log.info("Approximate number of blobs in DHT: %s", str(node.getApproximateTotalHashes()))
|
||||
|
||||
d.addCallback(lambda _: log_network_size())
|
||||
|
||||
@@ -36,7 +39,7 @@ def get_hosts(node, h):
|
||||
print "Hosts returned from the DHT: "
|
||||
print hosts
|
||||
|
||||
logging.info("Looking up %s", h)
|
||||
log.info("Looking up %s", h)
|
||||
d = node.getPeersForBlob(h)
|
||||
d.addCallback(print_hosts)
|
||||
return d
|
||||
@@ -48,9 +51,9 @@ def announce_hash(node, h):
|
||||
def log_results(results):
|
||||
for success, result in results:
|
||||
if success:
|
||||
logging.info("Succeeded: %s", str(result))
|
||||
log.info("Succeeded: %s", str(result))
|
||||
else:
|
||||
logging.info("Failed: %s", str(result.getErrorMessage()))
|
||||
log.info("Failed: %s", str(result.getErrorMessage()))
|
||||
|
||||
d.addCallback(log_results)
|
||||
return d
|
||||
@@ -80,11 +83,11 @@ def run_dht_script(dht_func):
|
||||
d.addCallback(run_dht_func)
|
||||
|
||||
def log_err(err):
|
||||
logging.error("An error occurred: %s", err.getTraceback())
|
||||
log.error("An error occurred: %s", err.getTraceback())
|
||||
return err
|
||||
|
||||
def shut_down():
|
||||
logging.info("Shutting down")
|
||||
log.info("Shutting down")
|
||||
reactor.stop()
|
||||
|
||||
d.addErrback(log_err)
|
||||
|
||||
+2
-2
@@ -20,7 +20,7 @@
|
||||
|
||||
|
||||
|
||||
import os, sys, time, signal, hashlib, random
|
||||
import sys, hashlib, random
|
||||
import twisted.internet.reactor
|
||||
from lbrynet.dht.node import Node
|
||||
#from entangled.kademlia.datastore import SQLiteDataStore
|
||||
@@ -106,7 +106,7 @@ def stop():
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
import sys, os
|
||||
import sys
|
||||
if len(sys.argv) < 2:
|
||||
print 'Usage:\n%s UDP_PORT [KNOWN_NODE_IP KNOWN_NODE_PORT]' % sys.argv[0]
|
||||
print 'or:\n%s UDP_PORT [FILE_WITH_KNOWN_NODES]' % sys.argv[0]
|
||||
|
||||
+26
-9
@@ -445,10 +445,7 @@ class IQueryHandlerFactory(Interface):
|
||||
"""
|
||||
|
||||
|
||||
class IStreamDownloaderFactory(Interface):
|
||||
"""
|
||||
Construct IStreamDownloaders and provide options that will be passed to those IStreamDownloaders.
|
||||
"""
|
||||
class IStreamDownloaderOptions(Interface):
|
||||
def get_downloader_options(self, sd_validator, payment_rate_manager):
|
||||
"""
|
||||
Return the list of options that can be used to modify IStreamDownloader behavior
|
||||
@@ -459,8 +456,28 @@ class IStreamDownloaderFactory(Interface):
|
||||
@param payment_rate_manager: The payment rate manager currently in effect for the downloader
|
||||
@type payment_rate_manager: PaymentRateManager
|
||||
|
||||
@return: [(option_description, default)]
|
||||
@rtype: [(string, string)]
|
||||
@return: [DownloadOption]
|
||||
@rtype: [DownloadOption]
|
||||
"""
|
||||
|
||||
|
||||
class IStreamDownloaderFactory(Interface):
|
||||
"""
|
||||
Construct IStreamDownloaders and provide options that will be passed to those IStreamDownloaders.
|
||||
"""
|
||||
|
||||
def can_download(self, sd_validator, payment_rate_manager):
|
||||
"""
|
||||
Decide whether the downloaders created by this factory can download the stream described by sd_validator
|
||||
|
||||
@param sd_validator: object containing stream metadata
|
||||
@type sd_validator: object which implements IStreamDescriptorValidator interface
|
||||
|
||||
@param payment_rate_manager: The payment rate manager currently in effect for the downloader
|
||||
@type payment_rate_manager: PaymentRateManager
|
||||
|
||||
@return: True if the downloaders can download the stream, False otherwise
|
||||
@rtype: bool
|
||||
"""
|
||||
|
||||
def make_downloader(self, sd_validator, options, payment_rate_manager):
|
||||
@@ -470,10 +487,10 @@ class IStreamDownloaderFactory(Interface):
|
||||
@param sd_validator: object containing stream metadata which will be given to the IStreamDownloader
|
||||
@type sd_validator: object which implements IStreamDescriptorValidator interface
|
||||
|
||||
@param options: a list of strings that will be used by the IStreamDownloaderFactory to
|
||||
@param options: a list of values that will be used by the IStreamDownloaderFactory to
|
||||
construct the IStreamDownloader. the options are in the same order as they were given
|
||||
by get_downloader_options.
|
||||
@type options: [string]
|
||||
@type options: [Object]
|
||||
|
||||
@param payment_rate_manager: the PaymentRateManager which the IStreamDownloader should use.
|
||||
@type payment_rate_manager: PaymentRateManager
|
||||
@@ -504,7 +521,7 @@ class IStreamDownloader(Interface):
|
||||
@rtype: Deferred which fires with anything
|
||||
"""
|
||||
|
||||
def insufficient_funds(self):
|
||||
def insufficient_funds(self, err):
|
||||
"""
|
||||
this function informs the stream downloader that funds are too low to finish downloading.
|
||||
|
||||
|
||||
@@ -1,13 +1,18 @@
|
||||
import logging
|
||||
import leveldb
|
||||
import json
|
||||
import sqlite3
|
||||
import os
|
||||
from twisted.internet import threads, defer
|
||||
from lbrynet.core.Error import DuplicateStreamHashError
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.enterprise import adbapi
|
||||
from lbrynet.core.Error import DuplicateStreamHashError, NoSuchStreamHashError
|
||||
from lbrynet.core.sqlite_helpers import rerun_if_locked
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DBLBRYFileMetadataManager(object):
|
||||
"""Store and provide access to LBRY file metadata using leveldb files"""
|
||||
"""Store and provide access to LBRY file metadata using sqlite"""
|
||||
|
||||
def __init__(self, db_dir):
|
||||
self.db_dir = db_dir
|
||||
@@ -16,163 +21,226 @@ class DBLBRYFileMetadataManager(object):
|
||||
self.stream_desc_db = None
|
||||
|
||||
def setup(self):
|
||||
return threads.deferToThread(self._open_db)
|
||||
return self._open_db()
|
||||
|
||||
def stop(self):
|
||||
self.stream_info_db = None
|
||||
self.stream_blob_db = None
|
||||
self.stream_desc_db = None
|
||||
self.db_conn = None
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_all_streams(self):
|
||||
return threads.deferToThread(self._get_all_streams)
|
||||
return self._get_all_streams()
|
||||
|
||||
def save_stream(self, stream_hash, file_name, key, suggested_file_name, blobs):
|
||||
d = threads.deferToThread(self._store_stream, stream_hash, file_name, key, suggested_file_name)
|
||||
d = self._store_stream(stream_hash, file_name, key, suggested_file_name)
|
||||
d.addCallback(lambda _: self.add_blobs_to_stream(stream_hash, blobs))
|
||||
return d
|
||||
|
||||
def get_stream_info(self, stream_hash):
|
||||
return threads.deferToThread(self._get_stream_info, stream_hash)
|
||||
return self._get_stream_info(stream_hash)
|
||||
|
||||
def check_if_stream_exists(self, stream_hash):
|
||||
return threads.deferToThread(self._check_if_stream_exists, stream_hash)
|
||||
return self._check_if_stream_exists(stream_hash)
|
||||
|
||||
def delete_stream(self, stream_hash):
|
||||
return threads.deferToThread(self._delete_stream, stream_hash)
|
||||
return self._delete_stream(stream_hash)
|
||||
|
||||
def add_blobs_to_stream(self, stream_hash, blobs):
|
||||
|
||||
def add_blobs():
|
||||
self._add_blobs_to_stream(stream_hash, blobs, ignore_duplicate_error=True)
|
||||
|
||||
return threads.deferToThread(add_blobs)
|
||||
return self._add_blobs_to_stream(stream_hash, blobs, ignore_duplicate_error=True)
|
||||
|
||||
def get_blobs_for_stream(self, stream_hash, start_blob=None, end_blob=None, count=None, reverse=False):
|
||||
logging.info("Getting blobs for a stream. Count is %s", str(count))
|
||||
log.debug("Getting blobs for a stream. Count is %s", str(count))
|
||||
|
||||
def get_positions_of_start_and_end():
|
||||
if start_blob is not None:
|
||||
start_num = self._get_blob_num_by_hash(stream_hash, start_blob)
|
||||
d1 = self._get_blob_num_by_hash(stream_hash, start_blob)
|
||||
else:
|
||||
start_num = None
|
||||
d1 = defer.succeed(None)
|
||||
if end_blob is not None:
|
||||
end_num = self._get_blob_num_by_hash(stream_hash, end_blob)
|
||||
d2 = self._get_blob_num_by_hash(stream_hash, end_blob)
|
||||
else:
|
||||
d2 = defer.succeed(None)
|
||||
|
||||
dl = defer.DeferredList([d1, d2])
|
||||
|
||||
def get_positions(results):
|
||||
start_num = None
|
||||
end_num = None
|
||||
return start_num, end_num
|
||||
if results[0][0] is True:
|
||||
start_num = results[0][1]
|
||||
if results[1][0] is True:
|
||||
end_num = results[1][1]
|
||||
return start_num, end_num
|
||||
|
||||
dl.addCallback(get_positions)
|
||||
return dl
|
||||
|
||||
def get_blob_infos(nums):
|
||||
start_num, end_num = nums
|
||||
return threads.deferToThread(self._get_further_blob_infos, stream_hash, start_num, end_num,
|
||||
count, reverse)
|
||||
return self._get_further_blob_infos(stream_hash, start_num, end_num,
|
||||
count, reverse)
|
||||
|
||||
d = threads.deferToThread(get_positions_of_start_and_end)
|
||||
d = get_positions_of_start_and_end()
|
||||
d.addCallback(get_blob_infos)
|
||||
return d
|
||||
|
||||
def get_stream_of_blob(self, blob_hash):
|
||||
return threads.deferToThread(self._get_stream_of_blobhash, blob_hash)
|
||||
return self._get_stream_of_blobhash(blob_hash)
|
||||
|
||||
def save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
|
||||
return threads.deferToThread(self._save_sd_blob_hash_to_stream, stream_hash, sd_blob_hash)
|
||||
return self._save_sd_blob_hash_to_stream(stream_hash, sd_blob_hash)
|
||||
|
||||
def get_sd_blob_hashes_for_stream(self, stream_hash):
|
||||
return threads.deferToThread(self._get_sd_blob_hashes_for_stream, stream_hash)
|
||||
return self._get_sd_blob_hashes_for_stream(stream_hash)
|
||||
|
||||
def _open_db(self):
|
||||
self.stream_info_db = leveldb.LevelDB(os.path.join(self.db_dir, "lbryfile_info.db"))
|
||||
self.stream_blob_db = leveldb.LevelDB(os.path.join(self.db_dir, "lbryfile_blob.db"))
|
||||
self.stream_desc_db = leveldb.LevelDB(os.path.join(self.db_dir, "lbryfile_desc.db"))
|
||||
# check_same_thread=False is solely to quiet a spurious error that appears to be due
|
||||
# to a bug in twisted, where the connection is closed by a different thread than the
|
||||
# one that opened it. The individual connections in the pool are not used in multiple
|
||||
# threads.
|
||||
self.db_conn = adbapi.ConnectionPool("sqlite3", (os.path.join(self.db_dir, "lbryfile_info.db")),
|
||||
check_same_thread=False)
|
||||
|
||||
def create_tables(transaction):
|
||||
transaction.execute("create table if not exists lbry_files (" +
|
||||
" stream_hash text primary key, " +
|
||||
" key text, " +
|
||||
" stream_name text, " +
|
||||
" suggested_file_name text" +
|
||||
")")
|
||||
transaction.execute("create table if not exists lbry_file_blobs (" +
|
||||
" blob_hash text, " +
|
||||
" stream_hash text, " +
|
||||
" position integer, " +
|
||||
" iv text, " +
|
||||
" length integer, " +
|
||||
" foreign key(stream_hash) references lbry_files(stream_hash)" +
|
||||
")")
|
||||
transaction.execute("create table if not exists lbry_file_descriptors (" +
|
||||
" sd_blob_hash TEXT PRIMARY KEY, " +
|
||||
" stream_hash TEXT, " +
|
||||
" foreign key(stream_hash) references lbry_files(stream_hash)" +
|
||||
")")
|
||||
|
||||
return self.db_conn.runInteraction(create_tables)
|
||||
|
||||
@rerun_if_locked
|
||||
def _delete_stream(self, stream_hash):
|
||||
desc_batch = leveldb.WriteBatch()
|
||||
for sd_blob_hash, s_h in self.stream_desc_db.RangeIter():
|
||||
if stream_hash == s_h:
|
||||
desc_batch.Delete(sd_blob_hash)
|
||||
self.stream_desc_db.Write(desc_batch, sync=True)
|
||||
d = self.db_conn.runQuery("select stream_hash from lbry_files where stream_hash = ?", (stream_hash,))
|
||||
d.addCallback(lambda result: result[0][0] if len(result) else Failure(NoSuchStreamHashError(stream_hash)))
|
||||
|
||||
blob_batch = leveldb.WriteBatch()
|
||||
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
|
||||
b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
if stream_hash == s_h:
|
||||
blob_batch.Delete(blob_hash_stream_hash)
|
||||
self.stream_blob_db.Write(blob_batch, sync=True)
|
||||
def do_delete(transaction, s_h):
|
||||
transaction.execute("delete from lbry_files where stream_hash = ?", (s_h,))
|
||||
transaction.execute("delete from lbry_file_blobs where stream_hash = ?", (s_h,))
|
||||
transaction.execute("delete from lbry_file_descriptors where stream_hash = ?", (s_h,))
|
||||
|
||||
stream_batch = leveldb.WriteBatch()
|
||||
for s_h, stream_info in self.stream_info_db.RangeIter():
|
||||
if stream_hash == s_h:
|
||||
stream_batch.Delete(s_h)
|
||||
self.stream_info_db.Write(stream_batch, sync=True)
|
||||
d.addCallback(lambda s_h: self.db_conn.runInteraction(do_delete, s_h))
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _store_stream(self, stream_hash, name, key, suggested_file_name):
|
||||
try:
|
||||
self.stream_info_db.Get(stream_hash)
|
||||
raise DuplicateStreamHashError("Stream hash %s already exists" % stream_hash)
|
||||
except KeyError:
|
||||
pass
|
||||
self.stream_info_db.Put(stream_hash, json.dumps((key, name, suggested_file_name)), sync=True)
|
||||
d = self.db_conn.runQuery("insert into lbry_files values (?, ?, ?, ?)",
|
||||
(stream_hash, key, name, suggested_file_name))
|
||||
|
||||
def check_duplicate(err):
|
||||
if err.check(sqlite3.IntegrityError):
|
||||
raise DuplicateStreamHashError(stream_hash)
|
||||
return err
|
||||
|
||||
d.addErrback(check_duplicate)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_all_streams(self):
|
||||
return [stream_hash for stream_hash, stream_info in self.stream_info_db.RangeIter()]
|
||||
d = self.db_conn.runQuery("select stream_hash from lbry_files")
|
||||
d.addCallback(lambda results: [r[0] for r in results])
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_stream_info(self, stream_hash):
|
||||
return json.loads(self.stream_info_db.Get(stream_hash))[:3]
|
||||
d = self.db_conn.runQuery("select key, stream_name, suggested_file_name from lbry_files where stream_hash = ?",
|
||||
(stream_hash,))
|
||||
d.addCallback(lambda result: result[0] if len(result) else Failure(NoSuchStreamHashError(stream_hash)))
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _check_if_stream_exists(self, stream_hash):
|
||||
try:
|
||||
self.stream_info_db.Get(stream_hash)
|
||||
return True
|
||||
except KeyError:
|
||||
return False
|
||||
d = self.db_conn.runQuery("select stream_hash from lbry_files where stream_hash = ?", (stream_hash,))
|
||||
d.addCallback(lambda r: True if len(r) else False)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_blob_num_by_hash(self, stream_hash, blob_hash):
|
||||
blob_hash_stream_hash = json.dumps((blob_hash, stream_hash))
|
||||
return json.loads(self.stream_blob_db.Get(blob_hash_stream_hash))[0]
|
||||
d = self.db_conn.runQuery("select position from lbry_file_blobs where stream_hash = ? and blob_hash = ?",
|
||||
(stream_hash, blob_hash))
|
||||
d.addCallback(lambda r: r[0][0] if len(r) else None)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_further_blob_infos(self, stream_hash, start_num, end_num, count=None, reverse=False):
|
||||
blob_infos = []
|
||||
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
|
||||
b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
if stream_hash == s_h:
|
||||
position, iv, length = json.loads(blob_info)
|
||||
if (start_num is None) or (position > start_num):
|
||||
if (end_num is None) or (position < end_num):
|
||||
blob_infos.append((b_h, position, iv, length))
|
||||
blob_infos.sort(key=lambda i: i[1], reverse=reverse)
|
||||
params = []
|
||||
q_string = "select * from ("
|
||||
q_string += " select blob_hash, position, iv, length from lbry_file_blobs "
|
||||
q_string += " where stream_hash = ? "
|
||||
params.append(stream_hash)
|
||||
if start_num is not None:
|
||||
q_string += " and position > ? "
|
||||
params.append(start_num)
|
||||
if end_num is not None:
|
||||
q_string += " and position < ? "
|
||||
params.append(end_num)
|
||||
q_string += " order by position "
|
||||
if reverse is True:
|
||||
q_string += " DESC "
|
||||
if count is not None:
|
||||
blob_infos = blob_infos[:count]
|
||||
return blob_infos
|
||||
q_string += " limit ? "
|
||||
params.append(count)
|
||||
q_string += ") order by position"
|
||||
# Order by position is done twice so that it always returns them from lowest position to
|
||||
# greatest, but the limit by clause can select the 'count' greatest or 'count' least
|
||||
return self.db_conn.runQuery(q_string, tuple(params))
|
||||
|
||||
@rerun_if_locked
|
||||
def _add_blobs_to_stream(self, stream_hash, blob_infos, ignore_duplicate_error=False):
|
||||
batch = leveldb.WriteBatch()
|
||||
for blob_info in blob_infos:
|
||||
blob_hash_stream_hash = json.dumps((blob_info.blob_hash, stream_hash))
|
||||
try:
|
||||
self.stream_blob_db.Get(blob_hash_stream_hash)
|
||||
if ignore_duplicate_error is False:
|
||||
raise KeyError() # TODO: change this to DuplicateStreamBlobError?
|
||||
continue
|
||||
except KeyError:
|
||||
pass
|
||||
batch.Put(blob_hash_stream_hash,
|
||||
json.dumps((blob_info.blob_num,
|
||||
blob_info.iv,
|
||||
blob_info.length)))
|
||||
self.stream_blob_db.Write(batch, sync=True)
|
||||
|
||||
def add_blobs(transaction):
|
||||
for blob_info in blob_infos:
|
||||
try:
|
||||
transaction.execute("insert into lbry_file_blobs values (?, ?, ?, ?, ?)",
|
||||
(blob_info.blob_hash, stream_hash, blob_info.blob_num,
|
||||
blob_info.iv, blob_info.length))
|
||||
except sqlite3.IntegrityError:
|
||||
if ignore_duplicate_error is False:
|
||||
raise
|
||||
|
||||
return self.db_conn.runInteraction(add_blobs)
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_stream_of_blobhash(self, blob_hash):
|
||||
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
|
||||
b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
if blob_hash == b_h:
|
||||
return s_h
|
||||
return None
|
||||
d = self.db_conn.runQuery("select stream_hash from lbry_file_blobs where blob_hash = ?",
|
||||
(blob_hash,))
|
||||
d.addCallback(lambda r: r[0][0] if len(r) else None)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
|
||||
self.stream_desc_db.Put(sd_blob_hash, stream_hash)
|
||||
log.info("Saving sd blob hash %s to stream hash %s", str(sd_blob_hash), str(stream_hash))
|
||||
d = self.db_conn.runQuery("insert into lbry_file_descriptors values (?, ?)",
|
||||
(sd_blob_hash, stream_hash))
|
||||
|
||||
def ignore_duplicate(err):
|
||||
err.trap(sqlite3.IntegrityError)
|
||||
log.info("sd blob hash already known")
|
||||
|
||||
d.addErrback(ignore_duplicate)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_sd_blob_hashes_for_stream(self, stream_hash):
|
||||
return [sd_blob_hash for sd_blob_hash, s_h in self.stream_desc_db.RangeIter() if stream_hash == s_h]
|
||||
log.debug("Looking up sd blob hashes for stream hash %s", str(stream_hash))
|
||||
d = self.db_conn.runQuery("select sd_blob_hash from lbry_file_descriptors where stream_hash = ?",
|
||||
(stream_hash,))
|
||||
d.addCallback(lambda results: [r[0] for r in results])
|
||||
return d
|
||||
|
||||
|
||||
class TempLBRYFileMetadataManager(object):
|
||||
|
||||
@@ -2,15 +2,20 @@ import binascii
|
||||
import logging
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.cryptstream.CryptBlob import CryptBlobInfo
|
||||
from twisted.internet import defer
|
||||
from lbrynet.core.Error import DuplicateStreamHashError
|
||||
from twisted.internet import defer, threads
|
||||
from lbrynet.core.Error import DuplicateStreamHashError, InvalidStreamDescriptorError
|
||||
from lbrynet.core.StreamDescriptor import PlainStreamDescriptorWriter, BlobStreamDescriptorWriter
|
||||
import os
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
LBRYFileStreamType = "lbryfile"
|
||||
|
||||
|
||||
def save_sd_info(stream_info_manager, sd_info, ignore_duplicate=False):
|
||||
logging.debug("Saving info for %s", str(sd_info['stream_name']))
|
||||
log.debug("Saving info for %s", str(sd_info['stream_name']))
|
||||
hex_stream_name = sd_info['stream_name']
|
||||
key = sd_info['key']
|
||||
stream_hash = sd_info['stream_hash']
|
||||
@@ -26,7 +31,7 @@ def save_sd_info(stream_info_manager, sd_info, ignore_duplicate=False):
|
||||
blob_num = blob['blob_num']
|
||||
iv = blob['iv']
|
||||
crypt_blobs.append(CryptBlobInfo(blob_hash, blob_num, length, iv))
|
||||
logging.debug("Trying to save stream info for %s", str(hex_stream_name))
|
||||
log.debug("Trying to save stream info for %s", str(hex_stream_name))
|
||||
d = stream_info_manager.save_stream(stream_hash, hex_stream_name, key,
|
||||
suggested_file_name, crypt_blobs)
|
||||
|
||||
@@ -75,12 +80,53 @@ def get_sd_info(stream_info_manager, stream_hash, include_blobs):
|
||||
return d
|
||||
|
||||
|
||||
def publish_sd_blob(stream_info_manager, blob_manager, stream_hash):
|
||||
descriptor_writer = BlobStreamDescriptorWriter(blob_manager)
|
||||
|
||||
d = get_sd_info(stream_info_manager, stream_hash, True)
|
||||
d.addCallback(descriptor_writer.create_descriptor)
|
||||
|
||||
def add_sd_blob_to_stream(sd_blob_hash):
|
||||
d = stream_info_manager.save_sd_blob_hash_to_stream(stream_hash, sd_blob_hash)
|
||||
d.addCallback(lambda _: sd_blob_hash)
|
||||
return d
|
||||
|
||||
d.addCallback(add_sd_blob_to_stream)
|
||||
return d
|
||||
|
||||
|
||||
def create_plain_sd(stream_info_manager, stream_hash, file_name, overwrite_existing=False):
|
||||
|
||||
def _get_file_name():
|
||||
actual_file_name = file_name
|
||||
if os.path.exists(actual_file_name):
|
||||
ext_num = 1
|
||||
while os.path.exists(actual_file_name + "_" + str(ext_num)):
|
||||
ext_num += 1
|
||||
actual_file_name = actual_file_name + "_" + str(ext_num)
|
||||
return actual_file_name
|
||||
|
||||
if overwrite_existing is False:
|
||||
d = threads.deferToThread(_get_file_name())
|
||||
else:
|
||||
d = defer.succeed(file_name)
|
||||
|
||||
def do_create(file_name):
|
||||
descriptor_writer = PlainStreamDescriptorWriter(file_name)
|
||||
d = get_sd_info(stream_info_manager, stream_hash, True)
|
||||
d.addCallback(descriptor_writer.create_descriptor)
|
||||
return d
|
||||
|
||||
d.addCallback(do_create)
|
||||
return d
|
||||
|
||||
|
||||
class LBRYFileStreamDescriptorValidator(object):
|
||||
def __init__(self, raw_info):
|
||||
self.raw_info = raw_info
|
||||
|
||||
def validate(self):
|
||||
logging.debug("Trying to validate stream descriptor for %s", str(self.raw_info['stream_name']))
|
||||
log.debug("Trying to validate stream descriptor for %s", str(self.raw_info['stream_name']))
|
||||
try:
|
||||
hex_stream_name = self.raw_info['stream_name']
|
||||
key = self.raw_info['key']
|
||||
@@ -88,11 +134,10 @@ class LBRYFileStreamDescriptorValidator(object):
|
||||
stream_hash = self.raw_info['stream_hash']
|
||||
blobs = self.raw_info['blobs']
|
||||
except KeyError as e:
|
||||
raise ValueError("Invalid stream descriptor. Missing '%s'" % (e.args[0]))
|
||||
raise InvalidStreamDescriptorError("Missing '%s'" % (e.args[0]))
|
||||
for c in hex_suggested_file_name:
|
||||
if c not in '0123456789abcdef':
|
||||
raise ValueError("Invalid stream descriptor: "
|
||||
"suggested file name is not a hex-encoded string")
|
||||
raise InvalidStreamDescriptorError("Suggested file name is not a hex-encoded string")
|
||||
h = get_lbry_hash_obj()
|
||||
h.update(hex_stream_name)
|
||||
h.update(key)
|
||||
@@ -118,10 +163,11 @@ class LBRYFileStreamDescriptorValidator(object):
|
||||
for blob in blobs:
|
||||
blobs_hashsum.update(get_blob_hashsum(blob))
|
||||
if blobs[-1]['length'] != 0:
|
||||
raise ValueError("Improperly formed stream descriptor. Must end with a zero-length blob.")
|
||||
raise InvalidStreamDescriptorError("Does not end with a zero-length blob.")
|
||||
h.update(blobs_hashsum.digest())
|
||||
if h.hexdigest() != stream_hash:
|
||||
raise ValueError("Stream hash does not match stream metadata")
|
||||
raise InvalidStreamDescriptorError("Stream hash does not match stream metadata")
|
||||
log.debug("It is validated")
|
||||
return defer.succeed(True)
|
||||
|
||||
def info_to_show(self):
|
||||
@@ -130,9 +176,15 @@ class LBRYFileStreamDescriptorValidator(object):
|
||||
size_so_far = 0
|
||||
for blob_info in self.raw_info.get("blobs", []):
|
||||
size_so_far += int(blob_info['length'])
|
||||
info.append(("stream_size", str(size_so_far)))
|
||||
info.append(("stream_size", str(self.get_length_of_stream())))
|
||||
suggested_file_name = self.raw_info.get("suggested_file_name", None)
|
||||
if suggested_file_name is not None:
|
||||
suggested_file_name = binascii.unhexlify(suggested_file_name)
|
||||
info.append(("suggested_file_name", suggested_file_name))
|
||||
return info
|
||||
return info
|
||||
|
||||
def get_length_of_stream(self):
|
||||
size_so_far = 0
|
||||
for blob_info in self.raw_info.get("blobs", []):
|
||||
size_so_far += int(blob_info['length'])
|
||||
return size_so_far
|
||||
@@ -0,0 +1,2 @@
|
||||
from lbrynet.lbryfile.StreamDescriptor import get_sd_info
|
||||
from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob
|
||||
@@ -3,14 +3,20 @@ import binascii
|
||||
|
||||
from zope.interface import implements
|
||||
|
||||
from lbrynet.core.DownloadOption import DownloadOption
|
||||
from lbrynet.lbryfile.StreamDescriptor import save_sd_info
|
||||
from lbrynet.cryptstream.client.CryptStreamDownloader import CryptStreamDownloader
|
||||
from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.core.StreamDescriptor import StreamMetadata
|
||||
from lbrynet.interfaces import IStreamDownloaderFactory
|
||||
from lbrynet.lbryfile.client.LBRYFileMetadataHandler import LBRYFileMetadataHandler
|
||||
import os
|
||||
from twisted.internet import defer, threads, reactor
|
||||
from twisted.python.procutils import which
|
||||
import logging
|
||||
import traceback
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LBRYFileDownloader(CryptStreamDownloader):
|
||||
@@ -40,9 +46,34 @@ class LBRYFileDownloader(CryptStreamDownloader):
|
||||
else:
|
||||
return defer.succeed(True)
|
||||
|
||||
def stop(self):
|
||||
def delete_data(self):
|
||||
d1 = self.stream_info_manager.get_blobs_for_stream(self.stream_hash)
|
||||
|
||||
def get_blob_hashes(blob_infos):
|
||||
return [b[0] for b in blob_infos if b[0] is not None]
|
||||
|
||||
d1.addCallback(get_blob_hashes)
|
||||
d2 = self.stream_info_manager.get_sd_blob_hashes_for_stream(self.stream_hash)
|
||||
|
||||
def combine_blob_hashes(results):
|
||||
blob_hashes = []
|
||||
for success, result in results:
|
||||
if success is True:
|
||||
blob_hashes.extend(result)
|
||||
return blob_hashes
|
||||
|
||||
def delete_blobs(blob_hashes):
|
||||
self.blob_manager.delete_blobs(blob_hashes)
|
||||
return True
|
||||
|
||||
dl = defer.DeferredList([d1, d2], fireOnOneErrback=True)
|
||||
dl.addCallback(combine_blob_hashes)
|
||||
dl.addCallback(delete_blobs)
|
||||
return dl
|
||||
|
||||
def stop(self, err=None):
|
||||
d = self._close_output()
|
||||
d.addCallback(lambda _: CryptStreamDownloader.stop(self))
|
||||
d.addCallback(lambda _: CryptStreamDownloader.stop(self, err=err))
|
||||
return d
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
@@ -60,6 +91,15 @@ class LBRYFileDownloader(CryptStreamDownloader):
|
||||
pass
|
||||
|
||||
def get_total_bytes(self):
|
||||
d = self.stream_info_manager.get_blobs_for_stream(self.stream_hash)
|
||||
|
||||
def calculate_size(blobs):
|
||||
return sum([b[3] for b in blobs])
|
||||
|
||||
d.addCallback(calculate_size)
|
||||
return d
|
||||
|
||||
def get_total_bytes_cached(self):
|
||||
if self._calculated_total_bytes is None or self._calculated_total_bytes == 0:
|
||||
if self.download_manager is None:
|
||||
return 0
|
||||
@@ -94,37 +134,30 @@ class LBRYFileDownloaderFactory(object):
|
||||
self.stream_info_manager = stream_info_manager
|
||||
self.wallet = wallet
|
||||
|
||||
def get_downloader_options(self, sd_validator, payment_rate_manager):
|
||||
options = [
|
||||
DownloadOption(
|
||||
[float, None],
|
||||
"rate which will be paid for data (None means use application default)",
|
||||
"data payment rate",
|
||||
None
|
||||
),
|
||||
DownloadOption(
|
||||
[bool],
|
||||
"allow reuploading data downloaded for this file",
|
||||
"allow upload",
|
||||
True
|
||||
),
|
||||
]
|
||||
return options
|
||||
def can_download(self, sd_validator):
|
||||
return True
|
||||
|
||||
def make_downloader(self, sd_validator, options, payment_rate_manager, **kwargs):
|
||||
if options[0] is not None:
|
||||
payment_rate_manager.float(options[0])
|
||||
def make_downloader(self, metadata, options, payment_rate_manager, **kwargs):
|
||||
payment_rate_manager.min_blob_data_payment_rate = options[0]
|
||||
upload_allowed = options[1]
|
||||
|
||||
def save_source_if_blob(stream_hash):
|
||||
if metadata.metadata_source == StreamMetadata.FROM_BLOB:
|
||||
d = self.stream_info_manager.save_sd_blob_hash_to_stream(stream_hash, metadata.source_blob_hash)
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
d.addCallback(lambda _: stream_hash)
|
||||
return d
|
||||
|
||||
def create_downloader(stream_hash):
|
||||
downloader = self._make_downloader(stream_hash, payment_rate_manager, sd_validator.raw_info,
|
||||
upload_allowed)
|
||||
downloader = self._make_downloader(stream_hash, payment_rate_manager,
|
||||
metadata.validator.raw_info, upload_allowed)
|
||||
d = downloader.set_stream_info()
|
||||
d.addCallback(lambda _: downloader)
|
||||
return d
|
||||
|
||||
d = save_sd_info(self.stream_info_manager, sd_validator.raw_info)
|
||||
|
||||
d = save_sd_info(self.stream_info_manager, metadata.validator.raw_info)
|
||||
d.addCallback(save_source_if_blob)
|
||||
d.addCallback(create_downloader)
|
||||
return d
|
||||
|
||||
@@ -139,8 +172,15 @@ class LBRYFileSaver(LBRYFileDownloader):
|
||||
stream_info_manager, payment_rate_manager, wallet, upload_allowed)
|
||||
self.download_directory = download_directory
|
||||
self.file_name = file_name
|
||||
self.file_written_to = None
|
||||
self.file_handle = None
|
||||
|
||||
def __str__(self):
|
||||
if self.file_written_to is not None:
|
||||
return str(self.file_written_to)
|
||||
else:
|
||||
return str(self.file_name)
|
||||
|
||||
def set_stream_info(self):
|
||||
d = LBRYFileDownloader.set_stream_info(self)
|
||||
|
||||
@@ -154,14 +194,14 @@ class LBRYFileSaver(LBRYFileDownloader):
|
||||
d.addCallback(lambda _: set_file_name())
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
d = LBRYFileDownloader.stop(self)
|
||||
def stop(self, err=None):
|
||||
d = LBRYFileDownloader.stop(self, err=err)
|
||||
d.addCallback(lambda _: self._delete_from_info_manager())
|
||||
return d
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading, self.blob_manager, download_manager,
|
||||
delete_blob_after_finished=True)
|
||||
delete_blob_after_finished=not self.upload_allowed)
|
||||
|
||||
def _setup_output(self):
|
||||
def open_file():
|
||||
@@ -171,11 +211,28 @@ class LBRYFileSaver(LBRYFileDownloader):
|
||||
file_name = "_"
|
||||
if os.path.exists(os.path.join(self.download_directory, file_name)):
|
||||
ext_num = 1
|
||||
|
||||
def _get_file_name(ext):
|
||||
if len(file_name.split(".")):
|
||||
fn = ''.join(file_name.split(".")[:-1])
|
||||
file_ext = ''.join(file_name.split(".")[-1])
|
||||
return fn + "-" + str(ext) + "." + file_ext
|
||||
else:
|
||||
return file_name + "_" + str(ext)
|
||||
|
||||
while os.path.exists(os.path.join(self.download_directory,
|
||||
file_name + "_" + str(ext_num))):
|
||||
_get_file_name(ext_num))):
|
||||
ext_num += 1
|
||||
file_name = file_name + "_" + str(ext_num)
|
||||
self.file_handle = open(os.path.join(self.download_directory, file_name), 'wb')
|
||||
|
||||
file_name = _get_file_name(ext_num)
|
||||
try:
|
||||
self.file_handle = open(os.path.join(self.download_directory, file_name), 'wb')
|
||||
self.file_written_to = os.path.join(self.download_directory, file_name)
|
||||
except IOError:
|
||||
log.error(traceback.format_exc())
|
||||
raise ValueError("Failed to open %s. Make sure you have permission to save files to that"
|
||||
" location." % str(os.path.join(self.download_directory,
|
||||
file_name)))
|
||||
return threads.deferToThread(open_file)
|
||||
|
||||
def _close_output(self):
|
||||
@@ -212,7 +269,8 @@ class LBRYFileSaverFactory(LBRYFileDownloaderFactory):
|
||||
self.stream_info_manager, payment_rate_manager, self.wallet,
|
||||
self.download_directory, upload_allowed)
|
||||
|
||||
def get_description(self):
|
||||
@staticmethod
|
||||
def get_description():
|
||||
return "Save"
|
||||
|
||||
|
||||
@@ -224,14 +282,14 @@ class LBRYFileOpener(LBRYFileDownloader):
|
||||
self.process = None
|
||||
self.process_log = None
|
||||
|
||||
def stop(self):
|
||||
d = LBRYFileDownloader.stop(self)
|
||||
def stop(self, err=None):
|
||||
d = LBRYFileDownloader.stop(self, err=err)
|
||||
d.addCallback(lambda _: self._delete_from_info_manager())
|
||||
return d
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading, self.blob_manager, download_manager,
|
||||
delete_blob_after_finished=True)
|
||||
delete_blob_after_finished=not self.upload_allowed)
|
||||
|
||||
def _setup_output(self):
|
||||
def start_process():
|
||||
@@ -276,9 +334,21 @@ class LBRYFileOpener(LBRYFileDownloader):
|
||||
|
||||
|
||||
class LBRYFileOpenerFactory(LBRYFileDownloaderFactory):
|
||||
def can_download(self, sd_validator):
|
||||
if which('vlc'):
|
||||
return True
|
||||
elif os.name == "nt":
|
||||
paths = [r'C:\Program Files\VideoLAN\VLC\vlc.exe',
|
||||
r'C:\Program Files (x86)\VideoLAN\VLC\vlc.exe']
|
||||
for p in paths:
|
||||
if os.path.exists(p):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _make_downloader(self, stream_hash, payment_rate_manager, stream_info, upload_allowed):
|
||||
return LBRYFileOpener(stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager,
|
||||
self.stream_info_manager, payment_rate_manager, self.wallet, upload_allowed)
|
||||
|
||||
def get_description(self):
|
||||
@staticmethod
|
||||
def get_description():
|
||||
return "Stream"
|
||||
@@ -4,6 +4,9 @@ from lbrynet.cryptstream.CryptBlob import CryptBlobInfo
|
||||
from lbrynet.interfaces import IMetadataHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LBRYFileMetadataHandler(object):
|
||||
implements(IMetadataHandler)
|
||||
|
||||
@@ -31,6 +34,6 @@ class LBRYFileMetadataHandler(object):
|
||||
if blob_hash is not None:
|
||||
infos.append(CryptBlobInfo(blob_hash, blob_num, length, iv))
|
||||
else:
|
||||
logging.debug("Setting _final_blob_num to %s", str(blob_num - 1))
|
||||
log.debug("Setting _final_blob_num to %s", str(blob_num - 1))
|
||||
self._final_blob_num = blob_num - 1
|
||||
return infos
|
||||
@@ -0,0 +1,55 @@
|
||||
from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType, LBRYFileStreamDescriptorValidator
|
||||
from lbrynet.core.DownloadOption import DownloadOption, DownloadOptionChoice
|
||||
|
||||
|
||||
def add_lbry_file_to_sd_identifier(sd_identifier):
|
||||
sd_identifier.add_stream_type(LBRYFileStreamType, LBRYFileStreamDescriptorValidator, LBRYFileOptions())
|
||||
|
||||
|
||||
class LBRYFileOptions(object):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def get_downloader_options(self, sd_validator, payment_rate_manager):
|
||||
prm = payment_rate_manager
|
||||
|
||||
def get_default_data_rate_description():
|
||||
if prm.min_blob_data_payment_rate is None:
|
||||
return "Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate)
|
||||
else:
|
||||
return "%f LBC/MB" % prm.min_blob_data_payment_rate
|
||||
|
||||
rate_choices = []
|
||||
rate_choices.append(DownloadOptionChoice(prm.min_blob_data_payment_rate,
|
||||
"No change - %s" % get_default_data_rate_description(),
|
||||
"No change - %s" % get_default_data_rate_description()))
|
||||
if prm.min_blob_data_payment_rate is not None:
|
||||
rate_choices.append(DownloadOptionChoice(None,
|
||||
"Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate),
|
||||
"Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate)))
|
||||
rate_choices.append(DownloadOptionChoice(float,
|
||||
"Enter rate in LBC/MB",
|
||||
"Enter rate in LBC/MB"))
|
||||
|
||||
options = [
|
||||
DownloadOption(
|
||||
rate_choices,
|
||||
"Rate which will be paid for data",
|
||||
"data payment rate",
|
||||
prm.min_blob_data_payment_rate,
|
||||
get_default_data_rate_description()
|
||||
),
|
||||
DownloadOption(
|
||||
[
|
||||
DownloadOptionChoice(bool,
|
||||
None,
|
||||
None,
|
||||
bool_options_description=("Allow", "Disallow")),
|
||||
],
|
||||
"Allow reuploading data downloaded for this file",
|
||||
"allow upload",
|
||||
True,
|
||||
"Allow"
|
||||
),
|
||||
]
|
||||
return options
|
||||
@@ -11,7 +11,9 @@ from lbrynet import conf
|
||||
from lbrynet.lbryfile.StreamDescriptor import get_sd_info
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
from twisted.protocols.basic import FileSender
|
||||
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LBRYFileStreamCreator(CryptStreamCreator):
|
||||
@@ -30,10 +32,10 @@ class LBRYFileStreamCreator(CryptStreamCreator):
|
||||
self.blob_infos = []
|
||||
|
||||
def _blob_finished(self, blob_info):
|
||||
logging.debug("length: %s", str(blob_info.length))
|
||||
log.debug("length: %s", str(blob_info.length))
|
||||
self.blob_infos.append(blob_info)
|
||||
|
||||
def _save_lbry_file_info(self):
|
||||
def _save_stream_info(self):
|
||||
stream_info_manager = self.lbry_file_manager.stream_info_manager
|
||||
d = stream_info_manager.save_stream(self.stream_hash, binascii.hexlify(self.name),
|
||||
binascii.hexlify(self.key),
|
||||
@@ -43,8 +45,6 @@ class LBRYFileStreamCreator(CryptStreamCreator):
|
||||
|
||||
def setup(self):
|
||||
d = CryptStreamCreator.setup(self)
|
||||
d.addCallback(lambda _: self.stream_hash)
|
||||
|
||||
return d
|
||||
|
||||
def _get_blobs_hashsum(self):
|
||||
@@ -76,10 +76,7 @@ class LBRYFileStreamCreator(CryptStreamCreator):
|
||||
|
||||
def _finished(self):
|
||||
self._make_stream_hash()
|
||||
d = self._save_lbry_file_info()
|
||||
d.addCallback(lambda _: self.lbry_file_manager.change_lbry_file_status(
|
||||
self.stream_hash, ManagedLBRYFileDownloader.STATUS_FINISHED
|
||||
))
|
||||
d = self._save_stream_info()
|
||||
return d
|
||||
|
||||
|
||||
@@ -128,12 +125,13 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
|
||||
"""
|
||||
|
||||
def stop_file(creator):
|
||||
logging.debug("the file sender has triggered its deferred. stopping the stream writer")
|
||||
log.debug("the file sender has triggered its deferred. stopping the stream writer")
|
||||
return creator.stop()
|
||||
|
||||
def make_stream_desc_file(stream_hash):
|
||||
logging.debug("creating the stream descriptor file")
|
||||
descriptor_writer = PlainStreamDescriptorWriter(file_name + conf.CRYPTSD_FILE_EXTENSION)
|
||||
log.debug("creating the stream descriptor file")
|
||||
descriptor_file_path = os.path.join(session.db_dir, file_name + conf.CRYPTSD_FILE_EXTENSION)
|
||||
descriptor_writer = PlainStreamDescriptorWriter(descriptor_file_path)
|
||||
|
||||
d = get_sd_info(lbry_file_manager.stream_info_manager, stream_hash, True)
|
||||
|
||||
@@ -156,4 +154,4 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
|
||||
|
||||
d = lbry_file_creator.setup()
|
||||
d.addCallback(lambda _: start_stream())
|
||||
return d
|
||||
return d
|
||||
@@ -2,14 +2,17 @@
|
||||
Download LBRY Files from LBRYnet and save them to disk.
|
||||
"""
|
||||
|
||||
from lbrynet.core.DownloadOption import DownloadOption
|
||||
from zope.interface import implements
|
||||
from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.core.StreamDescriptor import StreamMetadata
|
||||
from lbrynet.lbryfile.client.LBRYFileDownloader import LBRYFileSaver, LBRYFileDownloader
|
||||
from lbrynet.lbryfilemanager.LBRYFileStatusReport import LBRYFileStatusReport
|
||||
from lbrynet.interfaces import IStreamDownloaderFactory
|
||||
from lbrynet.lbryfile.StreamDescriptor import save_sd_info
|
||||
from twisted.internet import defer
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ManagedLBRYFileDownloader(LBRYFileSaver):
|
||||
@@ -18,19 +21,51 @@ class ManagedLBRYFileDownloader(LBRYFileSaver):
|
||||
STATUS_STOPPED = "stopped"
|
||||
STATUS_FINISHED = "finished"
|
||||
|
||||
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager,
|
||||
def __init__(self, rowid, stream_hash, peer_finder, rate_limiter, blob_manager, stream_info_manager,
|
||||
lbry_file_manager, payment_rate_manager, wallet, download_directory, upload_allowed,
|
||||
file_name=None):
|
||||
LBRYFileSaver.__init__(self, stream_hash, peer_finder, rate_limiter, blob_manager,
|
||||
stream_info_manager, payment_rate_manager, wallet, download_directory,
|
||||
upload_allowed)
|
||||
upload_allowed, file_name)
|
||||
self.sd_hash = None
|
||||
self.txid = None
|
||||
self.uri = None
|
||||
self.claim_id = None
|
||||
self.rowid = rowid
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
self.file_name = file_name
|
||||
self.file_handle = None
|
||||
self.saving_status = False
|
||||
|
||||
def restore(self):
|
||||
d = self.lbry_file_manager.get_lbry_file_status(self.stream_hash)
|
||||
d = self.stream_info_manager._get_sd_blob_hashes_for_stream(self.stream_hash)
|
||||
|
||||
def _save_sd_hash(sd_hash):
|
||||
if len(sd_hash):
|
||||
self.sd_hash = sd_hash[0]
|
||||
d = self.wallet._get_claim_metadata_for_sd_hash(self.sd_hash)
|
||||
else:
|
||||
d = defer.succeed(None)
|
||||
|
||||
return d
|
||||
|
||||
def _save_claim_id(claim_id):
|
||||
self.claim_id = claim_id
|
||||
return defer.succeed(None)
|
||||
|
||||
def _notify_bad_claim(name, txid):
|
||||
log.error("Error loading name claim for lbry file: lbry://%s, tx %s does not contain a valid claim", name, txid)
|
||||
log.warning("lbry file for lbry://%s, tx %s has no claim, deleting it", name, txid)
|
||||
return self.lbry_file_manager.delete_lbry_file(self)
|
||||
|
||||
def _save_claim(name, txid):
|
||||
self.uri = name
|
||||
self.txid = txid
|
||||
d = self.wallet.get_claimid(name, txid)
|
||||
d.addCallbacks(_save_claim_id, lambda err: _notify_bad_claim(name, txid))
|
||||
return d
|
||||
|
||||
d.addCallback(_save_sd_hash)
|
||||
d.addCallback(lambda r: _save_claim(r[0], r[1]) if r else None)
|
||||
d.addCallback(lambda _: self.lbry_file_manager.get_lbry_file_status(self))
|
||||
|
||||
def restore_status(status):
|
||||
if status == ManagedLBRYFileDownloader.STATUS_RUNNING:
|
||||
@@ -44,12 +79,12 @@ class ManagedLBRYFileDownloader(LBRYFileSaver):
|
||||
d.addCallback(restore_status)
|
||||
return d
|
||||
|
||||
def stop(self, change_status=True):
|
||||
def stop(self, err=None, change_status=True):
|
||||
|
||||
def set_saving_status_done():
|
||||
self.saving_status = False
|
||||
|
||||
d = LBRYFileDownloader.stop(self) # LBRYFileSaver deletes metadata when it's stopped. We don't want that here.
|
||||
d = LBRYFileDownloader.stop(self, err=err) # LBRYFileSaver deletes metadata when it's stopped. We don't want that here.
|
||||
if change_status is True:
|
||||
self.saving_status = True
|
||||
d.addCallback(lambda _: self._save_status())
|
||||
@@ -88,6 +123,24 @@ class ManagedLBRYFileDownloader(LBRYFileSaver):
|
||||
|
||||
d = LBRYFileSaver._start(self)
|
||||
|
||||
d.addCallback(lambda _: self.stream_info_manager._get_sd_blob_hashes_for_stream(self.stream_hash))
|
||||
|
||||
def _save_sd_hash(sd_hash):
|
||||
if len(sd_hash):
|
||||
self.sd_hash = sd_hash[0]
|
||||
d = self.wallet._get_claim_metadata_for_sd_hash(self.sd_hash)
|
||||
else:
|
||||
d = defer.succeed(None)
|
||||
|
||||
return d
|
||||
|
||||
def _save_claim(name, txid):
|
||||
self.uri = name
|
||||
self.txid = txid
|
||||
return defer.succeed(None)
|
||||
|
||||
d.addCallback(_save_sd_hash)
|
||||
d.addCallback(lambda r: _save_claim(r[0], r[1]) if r else None)
|
||||
d.addCallback(lambda _: self._save_status())
|
||||
|
||||
return d
|
||||
@@ -105,7 +158,7 @@ class ManagedLBRYFileDownloader(LBRYFileSaver):
|
||||
s = ManagedLBRYFileDownloader.STATUS_STOPPED
|
||||
else:
|
||||
s = ManagedLBRYFileDownloader.STATUS_RUNNING
|
||||
return self.lbry_file_manager.change_lbry_file_status(self.stream_hash, s)
|
||||
return self.lbry_file_manager.change_lbry_file_status(self, s)
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading, self.blob_manager, download_manager)
|
||||
@@ -117,33 +170,32 @@ class ManagedLBRYFileDownloaderFactory(object):
|
||||
def __init__(self, lbry_file_manager):
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
|
||||
def get_downloader_options(self, sd_validator, payment_rate_manager):
|
||||
options = [
|
||||
DownloadOption(
|
||||
[float, None],
|
||||
"rate which will be paid for data (None means use application default)",
|
||||
"data payment rate",
|
||||
None
|
||||
),
|
||||
DownloadOption(
|
||||
[bool],
|
||||
"allow reuploading data downloaded for this file",
|
||||
"allow upload",
|
||||
True
|
||||
),
|
||||
]
|
||||
return options
|
||||
def can_download(self, sd_validator):
|
||||
return True
|
||||
|
||||
def make_downloader(self, sd_validator, options, payment_rate_manager):
|
||||
def make_downloader(self, metadata, options, payment_rate_manager, download_directory=None, file_name=None):
|
||||
data_rate = options[0]
|
||||
upload_allowed = options[1]
|
||||
|
||||
d = save_sd_info(self.lbry_file_manager.stream_info_manager, sd_validator.raw_info)
|
||||
def save_source_if_blob(stream_hash):
|
||||
if metadata.metadata_source == StreamMetadata.FROM_BLOB:
|
||||
d = self.lbry_file_manager.stream_info_manager.save_sd_blob_hash_to_stream(stream_hash,
|
||||
metadata.source_blob_hash)
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
d.addCallback(lambda _: stream_hash)
|
||||
return d
|
||||
|
||||
d = save_sd_info(self.lbry_file_manager.stream_info_manager, metadata.validator.raw_info)
|
||||
d.addCallback(save_source_if_blob)
|
||||
d.addCallback(lambda stream_hash: self.lbry_file_manager.add_lbry_file(stream_hash,
|
||||
payment_rate_manager,
|
||||
data_rate,
|
||||
upload_allowed))
|
||||
upload_allowed,
|
||||
download_directory=download_directory,
|
||||
file_name=file_name))
|
||||
return d
|
||||
|
||||
def get_description(self):
|
||||
@staticmethod
|
||||
def get_description():
|
||||
return "Save the file to disk"
|
||||
@@ -3,84 +3,55 @@ Keep track of which LBRY Files are downloading and store their LBRY File specifi
|
||||
"""
|
||||
|
||||
import logging
|
||||
import json
|
||||
|
||||
import leveldb
|
||||
|
||||
from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamDescriptorValidator
|
||||
import os
|
||||
|
||||
from twisted.enterprise import adbapi
|
||||
from twisted.internet import defer, task, reactor
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader
|
||||
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloaderFactory
|
||||
from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType
|
||||
from lbrynet.core.PaymentRateManager import PaymentRateManager
|
||||
from twisted.internet import threads, defer, task, reactor
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.cryptstream.client.CryptStreamDownloader import AlreadyStoppedError, CurrentlyStoppingError
|
||||
from lbrynet.core.sqlite_helpers import rerun_if_locked
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LBRYFileManager(object):
|
||||
"""
|
||||
Keeps track of currently opened LBRY Files, their options, and their LBRY File specific metadata.
|
||||
"""
|
||||
SETTING = "s"
|
||||
LBRYFILE_STATUS = "t"
|
||||
LBRYFILE_OPTIONS = "o"
|
||||
|
||||
def __init__(self, session, stream_info_manager, sd_identifier):
|
||||
def __init__(self, session, stream_info_manager, sd_identifier, download_directory=None):
|
||||
self.session = session
|
||||
self.stream_info_manager = stream_info_manager
|
||||
self.sd_identifier = sd_identifier
|
||||
self.lbry_files = []
|
||||
self.db = None
|
||||
self.download_directory = os.getcwd()
|
||||
self.sql_db = None
|
||||
if download_directory:
|
||||
self.download_directory = download_directory
|
||||
else:
|
||||
self.download_directory = os.getcwd()
|
||||
log.debug("Download directory for LBRYFileManager: %s", str(self.download_directory))
|
||||
|
||||
def setup(self):
|
||||
d = threads.deferToThread(self._open_db)
|
||||
d = self._open_db()
|
||||
d.addCallback(lambda _: self._add_to_sd_identifier())
|
||||
d.addCallback(lambda _: self._start_lbry_files())
|
||||
return d
|
||||
|
||||
def get_all_lbry_file_stream_hashes_and_options(self):
|
||||
d = threads.deferToThread(self._get_all_lbry_file_stream_hashes)
|
||||
def get_lbry_file_status(self, lbry_file):
|
||||
return self._get_lbry_file_status(lbry_file.rowid)
|
||||
|
||||
def get_options(stream_hashes):
|
||||
ds = []
|
||||
def set_lbry_file_data_payment_rate(self, lbry_file, new_rate):
|
||||
return self._set_lbry_file_payment_rate(lbry_file.rowid, new_rate)
|
||||
|
||||
def get_options_for_stream_hash(stream_hash):
|
||||
d = self.get_lbry_file_options(stream_hash)
|
||||
d.addCallback(lambda options: (stream_hash, options))
|
||||
return d
|
||||
|
||||
for stream_hash in stream_hashes:
|
||||
ds.append(get_options_for_stream_hash(stream_hash))
|
||||
dl = defer.DeferredList(ds)
|
||||
dl.addCallback(lambda results: [r[1] for r in results if r[0]])
|
||||
return dl
|
||||
|
||||
d.addCallback(get_options)
|
||||
return d
|
||||
|
||||
def get_lbry_file_status(self, stream_hash):
|
||||
return threads.deferToThread(self._get_lbry_file_status, stream_hash)
|
||||
|
||||
def save_lbry_file_options(self, stream_hash, blob_data_rate):
|
||||
return threads.deferToThread(self._save_lbry_file_options, stream_hash, blob_data_rate)
|
||||
|
||||
def get_lbry_file_options(self, stream_hash):
|
||||
return threads.deferToThread(self._get_lbry_file_options, stream_hash)
|
||||
|
||||
def delete_lbry_file_options(self, stream_hash):
|
||||
return threads.deferToThread(self._delete_lbry_file_options, stream_hash)
|
||||
|
||||
def set_lbry_file_data_payment_rate(self, stream_hash, new_rate):
|
||||
return threads.deferToThread(self._set_lbry_file_payment_rate, stream_hash, new_rate)
|
||||
|
||||
def change_lbry_file_status(self, stream_hash, status):
|
||||
logging.debug("Changing status of %s to %s", stream_hash, status)
|
||||
return threads.deferToThread(self._change_file_status, stream_hash, status)
|
||||
|
||||
def delete_lbry_file_status(self, stream_hash):
|
||||
return threads.deferToThread(self._delete_lbry_file_status, stream_hash)
|
||||
def change_lbry_file_status(self, lbry_file, status):
|
||||
log.debug("Changing status of %s to %s", lbry_file.stream_hash, status)
|
||||
return self._change_file_status(lbry_file.rowid, status)
|
||||
|
||||
def get_lbry_file_status_reports(self):
|
||||
ds = []
|
||||
@@ -98,52 +69,66 @@ class LBRYFileManager(object):
|
||||
|
||||
def _add_to_sd_identifier(self):
|
||||
downloader_factory = ManagedLBRYFileDownloaderFactory(self)
|
||||
self.sd_identifier.add_stream_info_validator(LBRYFileStreamType, LBRYFileStreamDescriptorValidator)
|
||||
self.sd_identifier.add_stream_downloader_factory(LBRYFileStreamType, downloader_factory)
|
||||
|
||||
def _start_lbry_files(self):
|
||||
|
||||
def set_options_and_restore(stream_hash, options):
|
||||
def set_options_and_restore(rowid, stream_hash, options):
|
||||
payment_rate_manager = PaymentRateManager(self.session.base_payment_rate_manager)
|
||||
d = self.add_lbry_file(stream_hash, payment_rate_manager, blob_data_rate=options[0])
|
||||
d = self.start_lbry_file(rowid, stream_hash, payment_rate_manager,
|
||||
blob_data_rate=options)
|
||||
d.addCallback(lambda downloader: downloader.restore())
|
||||
return d
|
||||
|
||||
def log_error(err):
|
||||
logging.error("An error occurred while starting a lbry file: %s", err.getErrorMessage())
|
||||
def log_error(err, rowid, stream_hash, options):
|
||||
log.error("An error occurred while starting a lbry file: %s", err.getErrorMessage())
|
||||
log.debug(rowid)
|
||||
log.debug(stream_hash)
|
||||
log.debug(options)
|
||||
|
||||
def start_lbry_files(stream_hashes_and_options):
|
||||
for stream_hash, options in stream_hashes_and_options:
|
||||
d = set_options_and_restore(stream_hash, options)
|
||||
d.addErrback(log_error)
|
||||
def start_lbry_files(lbry_files_and_options):
|
||||
for rowid, stream_hash, options in lbry_files_and_options:
|
||||
d = set_options_and_restore(rowid, stream_hash, options)
|
||||
d.addErrback(lambda err: log_error(err, rowid, stream_hash, options))
|
||||
return True
|
||||
|
||||
d = self.get_all_lbry_file_stream_hashes_and_options()
|
||||
d = self._get_all_lbry_files()
|
||||
d.addCallback(start_lbry_files)
|
||||
return d
|
||||
|
||||
def add_lbry_file(self, stream_hash, payment_rate_manager, blob_data_rate=None, upload_allowed=True):
|
||||
def start_lbry_file(self, rowid, stream_hash, payment_rate_manager, blob_data_rate=None, upload_allowed=True,
|
||||
download_directory=None, file_name=None):
|
||||
if not download_directory:
|
||||
download_directory = self.download_directory
|
||||
payment_rate_manager.min_blob_data_payment_rate = blob_data_rate
|
||||
lbry_file_downloader = ManagedLBRYFileDownloader(stream_hash, self.session.peer_finder,
|
||||
self.session.rate_limiter, self.session.blob_manager,
|
||||
lbry_file_downloader = ManagedLBRYFileDownloader(rowid, stream_hash,
|
||||
self.session.peer_finder,
|
||||
self.session.rate_limiter,
|
||||
self.session.blob_manager,
|
||||
self.stream_info_manager, self,
|
||||
payment_rate_manager, self.session.wallet,
|
||||
self.download_directory,
|
||||
upload_allowed)
|
||||
download_directory,
|
||||
upload_allowed,
|
||||
file_name=file_name)
|
||||
self.lbry_files.append(lbry_file_downloader)
|
||||
d = self.save_lbry_file_options(stream_hash, blob_data_rate)
|
||||
d.addCallback(lambda _: lbry_file_downloader.set_stream_info())
|
||||
d = lbry_file_downloader.set_stream_info()
|
||||
d.addCallback(lambda _: lbry_file_downloader)
|
||||
return d
|
||||
|
||||
def delete_lbry_file(self, stream_hash):
|
||||
def add_lbry_file(self, stream_hash, payment_rate_manager, blob_data_rate=None, upload_allowed=True,
|
||||
download_directory=None, file_name=None):
|
||||
d = self._save_lbry_file(stream_hash, blob_data_rate)
|
||||
d.addCallback(lambda rowid: self.start_lbry_file(rowid, stream_hash, payment_rate_manager,
|
||||
blob_data_rate, upload_allowed, download_directory, file_name))
|
||||
return d
|
||||
|
||||
def delete_lbry_file(self, lbry_file):
|
||||
for l in self.lbry_files:
|
||||
if l.stream_hash == stream_hash:
|
||||
if l == lbry_file:
|
||||
lbry_file = l
|
||||
break
|
||||
else:
|
||||
return defer.fail(Failure(ValueError("Could not find an LBRY file with the given stream hash, " +
|
||||
stream_hash)))
|
||||
return defer.fail(Failure(ValueError("Could not find that LBRY file")))
|
||||
|
||||
def wait_for_finished(count=2):
|
||||
if count <= 0 or lbry_file.saving_status is False:
|
||||
@@ -162,26 +147,19 @@ class LBRYFileManager(object):
|
||||
self.lbry_files.remove(lbry_file)
|
||||
|
||||
d.addCallback(lambda _: remove_from_list())
|
||||
d.addCallback(lambda _: self.delete_lbry_file_options(stream_hash))
|
||||
d.addCallback(lambda _: self.delete_lbry_file_status(stream_hash))
|
||||
d.addCallback(lambda _: self._delete_lbry_file_options(lbry_file.rowid))
|
||||
return d
|
||||
|
||||
def toggle_lbry_file_running(self, stream_hash):
|
||||
def toggle_lbry_file_running(self, lbry_file):
|
||||
"""Toggle whether a stream reader is currently running"""
|
||||
for l in self.lbry_files:
|
||||
if l.stream_hash == stream_hash:
|
||||
if l == lbry_file:
|
||||
return l.toggle_running()
|
||||
else:
|
||||
return defer.fail(Failure(ValueError("Could not find an LBRY file with the given stream hash, " +
|
||||
stream_hash)))
|
||||
|
||||
def get_stream_hash_from_name(self, lbry_file_name):
|
||||
for l in self.lbry_files:
|
||||
if l.file_name == lbry_file_name:
|
||||
return l.stream_hash
|
||||
return None
|
||||
return defer.fail(Failure(ValueError("Could not find that LBRY file")))
|
||||
|
||||
def stop(self):
|
||||
|
||||
ds = []
|
||||
|
||||
def wait_for_finished(lbry_file, count=2):
|
||||
@@ -206,50 +184,62 @@ class LBRYFileManager(object):
|
||||
dl.addCallback(lambda _: close_db())
|
||||
return dl
|
||||
|
||||
def get_count_for_stream_hash(self, stream_hash):
|
||||
return self._get_count_for_stream_hash(stream_hash)
|
||||
|
||||
######### database calls #########
|
||||
|
||||
def _open_db(self):
|
||||
self.db = leveldb.LevelDB(os.path.join(self.session.db_dir, "lbryfiles.db"))
|
||||
# check_same_thread=False is solely to quiet a spurious error that appears to be due
|
||||
# to a bug in twisted, where the connection is closed by a different thread than the
|
||||
# one that opened it. The individual connections in the pool are not used in multiple
|
||||
# threads.
|
||||
self.sql_db = adbapi.ConnectionPool("sqlite3", os.path.join(self.session.db_dir, "lbryfile_info.db"),
|
||||
check_same_thread=False)
|
||||
return self.sql_db.runQuery("create table if not exists lbry_file_options (" +
|
||||
" blob_data_rate real, " +
|
||||
" status text," +
|
||||
" stream_hash text,"
|
||||
" foreign key(stream_hash) references lbry_files(stream_hash)" +
|
||||
")")
|
||||
|
||||
def _save_payment_rate(self, rate_type, rate):
|
||||
if rate is not None:
|
||||
self.db.Put(json.dumps((self.SETTING, rate_type)), json.dumps(rate), sync=True)
|
||||
else:
|
||||
self.db.Delete(json.dumps((self.SETTING, rate_type)), sync=True)
|
||||
@rerun_if_locked
|
||||
def _save_lbry_file(self, stream_hash, data_payment_rate):
|
||||
def do_save(db_transaction):
|
||||
db_transaction.execute("insert into lbry_file_options values (?, ?, ?)",
|
||||
(data_payment_rate, ManagedLBRYFileDownloader.STATUS_STOPPED,
|
||||
stream_hash))
|
||||
return db_transaction.lastrowid
|
||||
return self.sql_db.runInteraction(do_save)
|
||||
|
||||
def _save_lbry_file_options(self, stream_hash, blob_data_rate):
|
||||
self.db.Put(json.dumps((self.LBRYFILE_OPTIONS, stream_hash)), json.dumps((blob_data_rate,)),
|
||||
sync=True)
|
||||
@rerun_if_locked
|
||||
def _delete_lbry_file_options(self, rowid):
|
||||
return self.sql_db.runQuery("delete from lbry_file_options where rowid = ?",
|
||||
(rowid,))
|
||||
|
||||
def _get_lbry_file_options(self, stream_hash):
|
||||
try:
|
||||
return json.loads(self.db.Get(json.dumps((self.LBRYFILE_OPTIONS, stream_hash))))
|
||||
except KeyError:
|
||||
return None, None
|
||||
@rerun_if_locked
|
||||
def _set_lbry_file_payment_rate(self, rowid, new_rate):
|
||||
return self.sql_db.runQuery("update lbry_file_options set blob_data_rate = ? where rowid = ?",
|
||||
(new_rate, rowid))
|
||||
|
||||
def _delete_lbry_file_options(self, stream_hash):
|
||||
self.db.Delete(json.dumps((self.LBRYFILE_OPTIONS, stream_hash)), sync=True)
|
||||
@rerun_if_locked
|
||||
def _get_all_lbry_files(self):
|
||||
d = self.sql_db.runQuery("select rowid, stream_hash, blob_data_rate from lbry_file_options")
|
||||
return d
|
||||
|
||||
def _set_lbry_file_payment_rate(self, stream_hash, new_rate):
|
||||
@rerun_if_locked
|
||||
def _change_file_status(self, rowid, new_status):
|
||||
return self.sql_db.runQuery("update lbry_file_options set status = ? where rowid = ?",
|
||||
(new_status, rowid))
|
||||
|
||||
self.db.Put(json.dumps((self.LBRYFILE_OPTIONS, stream_hash)), json.dumps((new_rate, )), sync=True)
|
||||
@rerun_if_locked
|
||||
def _get_lbry_file_status(self, rowid):
|
||||
d = self.sql_db.runQuery("select status from lbry_file_options where rowid = ?",
|
||||
(rowid,))
|
||||
d.addCallback(lambda r: r[0][0] if len(r) else ManagedLBRYFileDownloader.STATUS_STOPPED)
|
||||
return d
|
||||
|
||||
def _get_all_lbry_file_stream_hashes(self):
|
||||
hashes = []
|
||||
for k, v in self.db.RangeIter():
|
||||
key_type, stream_hash = json.loads(k)
|
||||
if key_type == self.LBRYFILE_STATUS:
|
||||
hashes.append(stream_hash)
|
||||
return hashes
|
||||
|
||||
def _change_file_status(self, stream_hash, new_status):
|
||||
self.db.Put(json.dumps((self.LBRYFILE_STATUS, stream_hash)), new_status, sync=True)
|
||||
|
||||
def _get_lbry_file_status(self, stream_hash):
|
||||
try:
|
||||
return self.db.Get(json.dumps((self.LBRYFILE_STATUS, stream_hash)))
|
||||
except KeyError:
|
||||
return ManagedLBRYFileDownloader.STATUS_STOPPED
|
||||
|
||||
def _delete_lbry_file_status(self, stream_hash):
|
||||
self.db.Delete(json.dumps((self.LBRYFILE_STATUS, stream_hash)), sync=True)
|
||||
@rerun_if_locked
|
||||
def _get_count_for_stream_hash(self, stream_hash):
|
||||
return self.sql_db.runQuery("select count(*) from lbry_file_options where stream_hash = ?",
|
||||
(stream_hash,))
|
||||
@@ -1,3 +1,6 @@
|
||||
# pylint: skip-file
|
||||
# This file is not maintained, but might be used in the future
|
||||
#
|
||||
import logging
|
||||
import sys
|
||||
from lbrynet.lbrylive.LiveStreamCreator import StdOutLiveStreamCreator
|
||||
@@ -15,7 +18,8 @@ from twisted.internet import defer, task
|
||||
|
||||
class LBRYStdinUploader():
|
||||
"""This class reads from standard in, creates a stream, and makes it available on the network."""
|
||||
def __init__(self, peer_port, dht_node_port, known_dht_nodes):
|
||||
def __init__(self, peer_port, dht_node_port, known_dht_nodes,
|
||||
stream_info_manager_class=DBLiveStreamMetadataManager, blob_manager_class=TempBlobManager):
|
||||
"""
|
||||
@param peer_port: the network port on which to listen for peers
|
||||
|
||||
@@ -25,8 +29,8 @@ class LBRYStdinUploader():
|
||||
"""
|
||||
self.peer_port = peer_port
|
||||
self.lbry_server_port = None
|
||||
self.session = LBRYSession(blob_manager_class=TempBlobManager,
|
||||
stream_info_manager_class=DBLiveStreamMetadataManager,
|
||||
self.session = LBRYSession(blob_manager_class=blob_manager_class,
|
||||
stream_info_manager_class=stream_info_manager_class,
|
||||
dht_node_class=Node, dht_node_port=dht_node_port,
|
||||
known_dht_nodes=known_dht_nodes, peer_port=self.peer_port,
|
||||
use_upnp=False)
|
||||
@@ -114,4 +118,4 @@ def launch_stdin_uploader():
|
||||
d.addCallback(lambda _: start_stdin_uploader())
|
||||
d.addCallback(lambda _: shut_down())
|
||||
reactor.addSystemEventTrigger('before', 'shutdown', uploader.shut_down)
|
||||
reactor.run()
|
||||
reactor.run()
|
||||
@@ -1,7 +1,10 @@
|
||||
# pylint: skip-file
|
||||
# This file is not maintained, but might be used in the future
|
||||
#
|
||||
import logging
|
||||
import sys
|
||||
|
||||
from lbrynet.lbrynet_console.plugins.LBRYLive.LBRYLiveStreamDownloader import LBRYLiveStreamDownloader
|
||||
from lbrynet.lbrylive.client.LiveStreamDownloader import LBRYLiveStreamDownloader
|
||||
from lbrynet.core.BlobManager import TempBlobManager
|
||||
from lbrynet.core.Session import LBRYSession
|
||||
from lbrynet.core.client.StandaloneBlobDownloader import StandaloneBlobDownloader
|
||||
@@ -15,15 +18,17 @@ from twisted.internet import task
|
||||
|
||||
class LBRYStdoutDownloader():
|
||||
"""This class downloads a live stream from the network and outputs it to standard out."""
|
||||
def __init__(self, dht_node_port, known_dht_nodes):
|
||||
def __init__(self, dht_node_port, known_dht_nodes,
|
||||
stream_info_manager_class=DBLiveStreamMetadataManager, blob_manager_class=TempBlobManager):
|
||||
"""
|
||||
@param dht_node_port: the network port on which to listen for DHT node requests
|
||||
|
||||
@param known_dht_nodes: a list of (ip_address, dht_port) which will be used to join the DHT network
|
||||
|
||||
"""
|
||||
self.session = LBRYSession(blob_manager_class=TempBlobManager,
|
||||
stream_info_manager_class=DBLiveStreamMetadataManager,
|
||||
|
||||
self.session = LBRYSession(blob_manager_class=blob_manager_class,
|
||||
stream_info_manager_class=stream_info_manager_class,
|
||||
dht_node_class=Node, dht_node_port=dht_node_port, known_dht_nodes=known_dht_nodes,
|
||||
use_upnp=False)
|
||||
self.payment_rate_manager = BaseLiveStreamPaymentRateManager()
|
||||
@@ -93,4 +98,4 @@ def launch_stdout_downloader():
|
||||
d.addErrback(print_error)
|
||||
d.addCallback(lambda _: shut_down())
|
||||
reactor.addSystemEventTrigger('before', 'shutdown', downloader.shut_down)
|
||||
reactor.run()
|
||||
reactor.run()
|
||||
@@ -1,20 +1,20 @@
|
||||
from lbrynet.core.StreamDescriptor import BlobStreamDescriptorWriter
|
||||
from lbrynet.lbrylive.StreamDescriptor import get_sd_info, LiveStreamType, LBRYLiveStreamDescriptorValidator
|
||||
from lbrynet.lbrylive.StreamDescriptor import get_sd_info
|
||||
from lbrynet.cryptstream.CryptStreamCreator import CryptStreamCreator
|
||||
from lbrynet.lbrylive.LiveBlob import LiveStreamBlobMaker
|
||||
from lbrynet.lbrylive.PaymentRateManager import BaseLiveStreamPaymentRateManager
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj, get_pub_key, sign_with_pass_phrase
|
||||
from Crypto import Random
|
||||
import binascii
|
||||
import logging
|
||||
from lbrynet.conf import CRYPTSD_FILE_EXTENSION
|
||||
from lbrynet.conf import MIN_BLOB_INFO_PAYMENT_RATE
|
||||
from lbrynet.lbrylive.client.LiveStreamDownloader import FullLiveStreamDownloaderFactory
|
||||
from twisted.internet import interfaces, defer
|
||||
from twisted.protocols.basic import FileSender
|
||||
from zope.interface import implements
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LiveStreamCreator(CryptStreamCreator):
|
||||
def __init__(self, blob_manager, stream_info_manager, name=None, key=None, iv_generator=None,
|
||||
delete_after_num=None, secret_pass_phrase=None):
|
||||
@@ -33,8 +33,8 @@ class LiveStreamCreator(CryptStreamCreator):
|
||||
return d
|
||||
|
||||
def _blob_finished(self, blob_info):
|
||||
logging.debug("In blob_finished")
|
||||
logging.debug("length: %s", str(blob_info.length))
|
||||
log.debug("In blob_finished")
|
||||
log.debug("length: %s", str(blob_info.length))
|
||||
sig_hash = get_lbry_hash_obj()
|
||||
sig_hash.update(self.stream_hash)
|
||||
if blob_info.length != 0:
|
||||
@@ -51,11 +51,11 @@ class LiveStreamCreator(CryptStreamCreator):
|
||||
d = self.stream_info_manager.add_blobs_to_stream(self.stream_hash, [blob_info])
|
||||
|
||||
def log_add_error(err):
|
||||
logging.error("An error occurred adding a blob info to the stream info manager: %s", err.getErrorMessage())
|
||||
log.error("An error occurred adding a blob info to the stream info manager: %s", err.getErrorMessage())
|
||||
return err
|
||||
|
||||
d.addErrback(log_add_error)
|
||||
logging.debug("returning from blob_finished")
|
||||
log.debug("returning from blob_finished")
|
||||
return d
|
||||
|
||||
def setup(self):
|
||||
@@ -173,17 +173,4 @@ class StdinStreamProducer(object):
|
||||
self.consumer.write(data)
|
||||
|
||||
def childConnectionLost(self, fd, reason):
|
||||
self.stopProducing()
|
||||
|
||||
|
||||
def add_live_stream_to_sd_identifier(session, stream_info_manager, sd_identifier):
|
||||
downloader_factory = FullLiveStreamDownloaderFactory(session.peer_finder,
|
||||
session.rate_limiter,
|
||||
session.blob_manager,
|
||||
stream_info_manager,
|
||||
session.wallet,
|
||||
BaseLiveStreamPaymentRateManager(
|
||||
MIN_BLOB_INFO_PAYMENT_RATE
|
||||
))
|
||||
sd_identifier.add_stream_info_validator(LiveStreamType, LBRYLiveStreamDescriptorValidator)
|
||||
sd_identifier.add_stream_downloader_factory(LiveStreamType, downloader_factory)
|
||||
self.stopProducing()
|
||||
@@ -1,11 +1,16 @@
|
||||
import time
|
||||
import logging
|
||||
import leveldb
|
||||
import json
|
||||
from twisted.enterprise import adbapi
|
||||
import os
|
||||
from twisted.internet import threads, defer
|
||||
import sqlite3
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.core.server.DHTHashAnnouncer import DHTHashSupplier
|
||||
from lbrynet.core.Error import DuplicateStreamHashError
|
||||
from lbrynet.core.Error import DuplicateStreamHashError, NoSuchStreamHashError
|
||||
from lbrynet.core.sqlite_helpers import rerun_if_locked
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DBLiveStreamMetadataManager(DHTHashSupplier):
|
||||
@@ -14,26 +19,22 @@ class DBLiveStreamMetadataManager(DHTHashSupplier):
|
||||
def __init__(self, db_dir, hash_announcer):
|
||||
DHTHashSupplier.__init__(self, hash_announcer)
|
||||
self.db_dir = db_dir
|
||||
self.stream_info_db = None
|
||||
self.stream_blob_db = None
|
||||
self.stream_desc_db = None
|
||||
self.db_conn = None
|
||||
|
||||
def setup(self):
|
||||
return threads.deferToThread(self._open_db)
|
||||
return self._open_db()
|
||||
|
||||
def stop(self):
|
||||
self.stream_info_db = None
|
||||
self.stream_blob_db = None
|
||||
self.stream_desc_db = None
|
||||
self.db_conn = None
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_all_streams(self):
|
||||
return threads.deferToThread(self._get_all_streams)
|
||||
return self._get_all_streams()
|
||||
|
||||
def save_stream(self, stream_hash, pub_key, file_name, key, blobs):
|
||||
next_announce_time = time.time() + self.hash_reannounce_time
|
||||
d = threads.deferToThread(self._store_stream, stream_hash, pub_key, file_name, key,
|
||||
next_announce_time=next_announce_time)
|
||||
d = self._store_stream(stream_hash, pub_key, file_name, key,
|
||||
next_announce_time=next_announce_time)
|
||||
|
||||
def save_blobs():
|
||||
return self.add_blobs_to_stream(stream_hash, blobs)
|
||||
@@ -48,169 +49,229 @@ class DBLiveStreamMetadataManager(DHTHashSupplier):
|
||||
return d
|
||||
|
||||
def get_stream_info(self, stream_hash):
|
||||
return threads.deferToThread(self._get_stream_info, stream_hash)
|
||||
return self._get_stream_info(stream_hash)
|
||||
|
||||
def check_if_stream_exists(self, stream_hash):
|
||||
return threads.deferToThread(self._check_if_stream_exists, stream_hash)
|
||||
return self._check_if_stream_exists(stream_hash)
|
||||
|
||||
def delete_stream(self, stream_hash):
|
||||
return threads.deferToThread(self._delete_stream, stream_hash)
|
||||
return self._delete_stream(stream_hash)
|
||||
|
||||
def add_blobs_to_stream(self, stream_hash, blobs):
|
||||
|
||||
def add_blobs():
|
||||
self._add_blobs_to_stream(stream_hash, blobs, ignore_duplicate_error=True)
|
||||
|
||||
return threads.deferToThread(add_blobs)
|
||||
return self._add_blobs_to_stream(stream_hash, blobs, ignore_duplicate_error=True)
|
||||
|
||||
def get_blobs_for_stream(self, stream_hash, start_blob=None, end_blob=None, count=None, reverse=False):
|
||||
logging.info("Getting blobs for a stream. Count is %s", str(count))
|
||||
log.info("Getting blobs for a stream. Count is %s", str(count))
|
||||
|
||||
def get_positions_of_start_and_end():
|
||||
if start_blob is not None:
|
||||
start_num = self._get_blob_num_by_hash(stream_hash, start_blob)
|
||||
d1 = self._get_blob_num_by_hash(stream_hash, start_blob)
|
||||
else:
|
||||
start_num = None
|
||||
d1 = defer.succeed(None)
|
||||
if end_blob is not None:
|
||||
end_num = self._get_blob_num_by_hash(stream_hash, end_blob)
|
||||
d2 = self._get_blob_num_by_hash(stream_hash, end_blob)
|
||||
else:
|
||||
d2 = defer.succeed(None)
|
||||
|
||||
dl = defer.DeferredList([d1, d2])
|
||||
|
||||
def get_positions(results):
|
||||
start_num = None
|
||||
end_num = None
|
||||
return start_num, end_num
|
||||
if results[0][0] is True:
|
||||
start_num = results[0][1]
|
||||
if results[1][0] is True:
|
||||
end_num = results[1][1]
|
||||
return start_num, end_num
|
||||
|
||||
dl.addCallback(get_positions)
|
||||
return dl
|
||||
|
||||
def get_blob_infos(nums):
|
||||
start_num, end_num = nums
|
||||
return threads.deferToThread(self._get_further_blob_infos, stream_hash, start_num, end_num,
|
||||
count, reverse)
|
||||
return self._get_further_blob_infos(stream_hash, start_num, end_num,
|
||||
count, reverse)
|
||||
|
||||
d = threads.deferToThread(get_positions_of_start_and_end)
|
||||
d = get_positions_of_start_and_end()
|
||||
d.addCallback(get_blob_infos)
|
||||
return d
|
||||
|
||||
def get_stream_of_blob(self, blob_hash):
|
||||
return threads.deferToThread(self._get_stream_of_blobhash, blob_hash)
|
||||
return self._get_stream_of_blobhash(blob_hash)
|
||||
|
||||
def save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
|
||||
return threads.deferToThread(self._save_sd_blob_hash_to_stream, stream_hash, sd_blob_hash)
|
||||
return self._save_sd_blob_hash_to_stream(stream_hash, sd_blob_hash)
|
||||
|
||||
def get_sd_blob_hashes_for_stream(self, stream_hash):
|
||||
return threads.deferToThread(self._get_sd_blob_hashes_for_stream, stream_hash)
|
||||
return self._get_sd_blob_hashes_for_stream(stream_hash)
|
||||
|
||||
def hashes_to_announce(self):
|
||||
next_announce_time = time.time() + self.hash_reannounce_time
|
||||
return threads.deferToThread(self._get_streams_to_announce, next_announce_time)
|
||||
return self._get_streams_to_announce(next_announce_time)
|
||||
|
||||
######### database calls #########
|
||||
|
||||
def _open_db(self):
|
||||
self.stream_info_db = leveldb.LevelDB(os.path.join(self.db_dir, "stream_info.db"))
|
||||
self.stream_blob_db = leveldb.LevelDB(os.path.join(self.db_dir, "stream_blob.db"))
|
||||
self.stream_desc_db = leveldb.LevelDB(os.path.join(self.db_dir, "stream_desc.db"))
|
||||
# check_same_thread=False is solely to quiet a spurious error that appears to be due
|
||||
# to a bug in twisted, where the connection is closed by a different thread than the
|
||||
# one that opened it. The individual connections in the pool are not used in multiple
|
||||
# threads.
|
||||
self.db_conn = adbapi.ConnectionPool('sqlite3', os.path.join(self.db_dir, "live_stream.db"),
|
||||
check_same_thread=False)
|
||||
|
||||
def create_tables(transaction):
|
||||
transaction.execute("create table if not exists live_streams (" +
|
||||
" stream_hash text primary key, " +
|
||||
" public_key text, " +
|
||||
" key text, " +
|
||||
" stream_name text, " +
|
||||
" next_announce_time real" +
|
||||
")")
|
||||
transaction.execute("create table if not exists live_stream_blobs (" +
|
||||
" blob_hash text, " +
|
||||
" stream_hash text, " +
|
||||
" position integer, " +
|
||||
" revision integer, " +
|
||||
" iv text, " +
|
||||
" length integer, " +
|
||||
" signature text, " +
|
||||
" foreign key(stream_hash) references live_streams(stream_hash)" +
|
||||
")")
|
||||
transaction.execute("create table if not exists live_stream_descriptors (" +
|
||||
" sd_blob_hash TEXT PRIMARY KEY, " +
|
||||
" stream_hash TEXT, " +
|
||||
" foreign key(stream_hash) references live_streams(stream_hash)" +
|
||||
")")
|
||||
|
||||
return self.db_conn.runInteraction(create_tables)
|
||||
|
||||
@rerun_if_locked
|
||||
def _delete_stream(self, stream_hash):
|
||||
desc_batch = leveldb.WriteBatch()
|
||||
for sd_blob_hash, s_h in self.stream_desc_db.RangeIter():
|
||||
if stream_hash == s_h:
|
||||
desc_batch.Delete(sd_blob_hash)
|
||||
self.stream_desc_db.Write(desc_batch, sync=True)
|
||||
|
||||
blob_batch = leveldb.WriteBatch()
|
||||
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
|
||||
b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
if stream_hash == s_h:
|
||||
blob_batch.Delete(blob_hash_stream_hash)
|
||||
self.stream_blob_db.Write(blob_batch, sync=True)
|
||||
d = self.db_conn.runQuery("select stream_hash from live_streams where stream_hash = ?", (stream_hash,))
|
||||
d.addCallback(lambda result: result[0][0] if len(result) else Failure(NoSuchStreamHashError(stream_hash)))
|
||||
|
||||
stream_batch = leveldb.WriteBatch()
|
||||
for s_h, stream_info in self.stream_info_db.RangeIter():
|
||||
if stream_hash == s_h:
|
||||
stream_batch.Delete(s_h)
|
||||
self.stream_info_db.Write(stream_batch, sync=True)
|
||||
def do_delete(transaction, s_h):
|
||||
transaction.execute("delete from live_streams where stream_hash = ?", (s_h,))
|
||||
transaction.execute("delete from live_stream_blobs where stream_hash = ?", (s_h,))
|
||||
transaction.execute("delete from live_stream_descriptors where stream_hash = ?", (s_h,))
|
||||
|
||||
d.addCallback(lambda s_h: self.db_conn.runInteraction(do_delete, s_h))
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _store_stream(self, stream_hash, public_key, name, key, next_announce_time=None):
|
||||
try:
|
||||
self.stream_info_db.Get(stream_hash)
|
||||
raise DuplicateStreamHashError("Stream hash %s already exists" % stream_hash)
|
||||
except KeyError:
|
||||
pass
|
||||
self.stream_info_db.Put(stream_hash, json.dumps((public_key, key, name, next_announce_time)), sync=True)
|
||||
d = self.db_conn.runQuery("insert into live_streams values (?, ?, ?, ?, ?)",
|
||||
(stream_hash, public_key, key, name, next_announce_time))
|
||||
|
||||
def check_duplicate(err):
|
||||
if err.check(sqlite3.IntegrityError):
|
||||
raise DuplicateStreamHashError(stream_hash)
|
||||
return err
|
||||
|
||||
d.addErrback(check_duplicate)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_all_streams(self):
|
||||
return [stream_hash for stream_hash, stream_info in self.stream_info_db.RangeIter()]
|
||||
d = self.db_conn.runQuery("select stream_hash from live_streams")
|
||||
d.addCallback(lambda results: [r[0] for r in results])
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_stream_info(self, stream_hash):
|
||||
return json.loads(self.stream_info_db.Get(stream_hash))[:3]
|
||||
d = self.db_conn.runQuery("select public_key, key, stream_name from live_streams where stream_hash = ?",
|
||||
(stream_hash,))
|
||||
d.addCallback(lambda result: result[0] if len(result) else Failure(NoSuchStreamHashError(stream_hash)))
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _check_if_stream_exists(self, stream_hash):
|
||||
try:
|
||||
self.stream_info_db.Get(stream_hash)
|
||||
return True
|
||||
except KeyError:
|
||||
return False
|
||||
d = self.db_conn.runQuery("select stream_hash from live_streams where stream_hash = ?", (stream_hash,))
|
||||
d.addCallback(lambda r: True if len(r) else False)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_streams_to_announce(self, next_announce_time):
|
||||
# TODO: See if the following would be better for handling announce times:
|
||||
# TODO: Have a separate db for them, and read the whole thing into memory
|
||||
# TODO: on startup, and then write changes to db when they happen
|
||||
stream_hashes = []
|
||||
batch = leveldb.WriteBatch()
|
||||
current_time = time.time()
|
||||
for stream_hash, stream_info in self.stream_info_db.RangeIter():
|
||||
public_key, key, name, announce_time = json.loads(stream_info)
|
||||
if announce_time < current_time:
|
||||
batch.Put(stream_hash, json.dumps((public_key, key, name, next_announce_time)))
|
||||
stream_hashes.append(stream_hash)
|
||||
self.stream_info_db.Write(batch, sync=True)
|
||||
return stream_hashes
|
||||
|
||||
def get_and_update(transaction):
|
||||
timestamp = time.time()
|
||||
r = transaction.execute("select stream_hash from live_streams where" +
|
||||
" (next_announce_time is null or next_announce_time < ?) " +
|
||||
" and stream_hash is not null", (timestamp, ))
|
||||
s_hs = [s_h for s_h, in r.fetchall()]
|
||||
transaction.execute("update live_streams set next_announce_time = ? where " +
|
||||
" (next_announce_time is null or next_announce_time < ?)",
|
||||
(next_announce_time, timestamp))
|
||||
return s_hs
|
||||
|
||||
return self.db_conn.runInteraction(get_and_update)
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_blob_num_by_hash(self, stream_hash, blob_hash):
|
||||
blob_hash_stream_hash = json.dumps((blob_hash, stream_hash))
|
||||
return json.loads(self.stream_blob_db.Get(blob_hash_stream_hash))[0]
|
||||
d = self.db_conn.runQuery("select position from live_stream_blobs where stream_hash = ? and blob_hash = ?",
|
||||
(stream_hash, blob_hash))
|
||||
d.addCallback(lambda r: r[0][0] if len(r) else None)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_further_blob_infos(self, stream_hash, start_num, end_num, count=None, reverse=False):
|
||||
blob_infos = []
|
||||
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
|
||||
b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
if stream_hash == s_h:
|
||||
position, revision, iv, length, signature = json.loads(blob_info)
|
||||
if (start_num is None) or (position > start_num):
|
||||
if (end_num is None) or (position < end_num):
|
||||
blob_infos.append((b_h, position, revision, iv, length, signature))
|
||||
blob_infos.sort(key=lambda i: i[1], reverse=reverse)
|
||||
params = []
|
||||
q_string = "select * from ("
|
||||
q_string += " select blob_hash, position, revision, iv, length, signature from live_stream_blobs "
|
||||
q_string += " where stream_hash = ? "
|
||||
params.append(stream_hash)
|
||||
if start_num is not None:
|
||||
q_string += " and position > ? "
|
||||
params.append(start_num)
|
||||
if end_num is not None:
|
||||
q_string += " and position < ? "
|
||||
params.append(end_num)
|
||||
q_string += " order by position "
|
||||
if reverse is True:
|
||||
q_string += " DESC "
|
||||
if count is not None:
|
||||
blob_infos = blob_infos[:count]
|
||||
return blob_infos
|
||||
q_string += " limit ? "
|
||||
params.append(count)
|
||||
q_string += ") order by position"
|
||||
# Order by position is done twice so that it always returns them from lowest position to
|
||||
# greatest, but the limit by clause can select the 'count' greatest or 'count' least
|
||||
return self.db_conn.runQuery(q_string, tuple(params))
|
||||
|
||||
@rerun_if_locked
|
||||
def _add_blobs_to_stream(self, stream_hash, blob_infos, ignore_duplicate_error=False):
|
||||
batch = leveldb.WriteBatch()
|
||||
for blob_info in blob_infos:
|
||||
blob_hash_stream_hash = json.dumps((blob_info.blob_hash, stream_hash))
|
||||
try:
|
||||
self.stream_blob_db.Get(blob_hash_stream_hash)
|
||||
if ignore_duplicate_error is False:
|
||||
raise KeyError() # TODO: change this to DuplicateStreamBlobError?
|
||||
continue
|
||||
except KeyError:
|
||||
pass
|
||||
batch.Put(blob_hash_stream_hash,
|
||||
json.dumps((blob_info.blob_num,
|
||||
blob_info.revision,
|
||||
blob_info.iv,
|
||||
blob_info.length,
|
||||
blob_info.signature)))
|
||||
self.stream_blob_db.Write(batch, sync=True)
|
||||
|
||||
def add_blobs(transaction):
|
||||
for blob_info in blob_infos:
|
||||
try:
|
||||
transaction.execute("insert into live_stream_blobs values (?, ?, ?, ?, ?, ?, ?)",
|
||||
(blob_info.blob_hash, stream_hash, blob_info.blob_num,
|
||||
blob_info.revision, blob_info.iv, blob_info.length,
|
||||
blob_info.signature))
|
||||
except sqlite3.IntegrityError:
|
||||
if ignore_duplicate_error is False:
|
||||
raise
|
||||
|
||||
return self.db_conn.runInteraction(add_blobs)
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_stream_of_blobhash(self, blob_hash):
|
||||
for blob_hash_stream_hash, blob_info in self.stream_blob_db.RangeIter():
|
||||
b_h, s_h = json.loads(blob_hash_stream_hash)
|
||||
if blob_hash == b_h:
|
||||
return s_h
|
||||
return None
|
||||
d = self.db_conn.runQuery("select stream_hash from live_stream_blobs where blob_hash = ?",
|
||||
(blob_hash,))
|
||||
d.addCallback(lambda r: r[0][0] if len(r) else None)
|
||||
return d
|
||||
|
||||
@rerun_if_locked
|
||||
def _save_sd_blob_hash_to_stream(self, stream_hash, sd_blob_hash):
|
||||
self.stream_desc_db.Put(sd_blob_hash, stream_hash)
|
||||
return self.db_conn.runQuery("insert into live_stream_descriptors values (?, ?)",
|
||||
(sd_blob_hash, stream_hash))
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_sd_blob_hashes_for_stream(self, stream_hash):
|
||||
return [sd_blob_hash for sd_blob_hash, s_h in self.stream_desc_db.RangeIter() if stream_hash == s_h]
|
||||
d = self.db_conn.runQuery("select sd_blob_hash from live_stream_descriptors where stream_hash = ?",
|
||||
(stream_hash,))
|
||||
d.addCallback(lambda results: [r[0] for r in results])
|
||||
return d
|
||||
|
||||
|
||||
class TempLiveStreamMetadataManager(DHTHashSupplier):
|
||||
|
||||
@@ -2,17 +2,20 @@ import binascii
|
||||
import logging
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj, verify_signature
|
||||
from twisted.internet import defer, threads
|
||||
from lbrynet.core.Error import DuplicateStreamHashError
|
||||
from lbrynet.core.Error import DuplicateStreamHashError, InvalidStreamDescriptorError
|
||||
from lbrynet.lbrylive.LiveBlob import LiveBlobInfo
|
||||
from lbrynet.interfaces import IStreamDescriptorValidator
|
||||
from zope.interface import implements
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
LiveStreamType = "lbrylive"
|
||||
|
||||
|
||||
def save_sd_info(stream_info_manager, sd_info, ignore_duplicate=False):
|
||||
logging.debug("Saving info for %s", str(sd_info['stream_name']))
|
||||
log.debug("Saving info for %s", str(sd_info['stream_name']))
|
||||
hex_stream_name = sd_info['stream_name']
|
||||
public_key = sd_info['public_key']
|
||||
key = sd_info['key']
|
||||
@@ -30,7 +33,7 @@ def save_sd_info(stream_info_manager, sd_info, ignore_duplicate=False):
|
||||
iv = blob['iv']
|
||||
signature = blob['signature']
|
||||
crypt_blobs.append(LiveBlobInfo(blob_hash, blob_num, length, iv, revision, signature))
|
||||
logging.debug("Trying to save stream info for %s", str(hex_stream_name))
|
||||
log.debug("Trying to save stream info for %s", str(hex_stream_name))
|
||||
d = stream_info_manager.save_stream(stream_hash, public_key, hex_stream_name,
|
||||
key, crypt_blobs)
|
||||
|
||||
@@ -88,7 +91,7 @@ class LBRYLiveStreamDescriptorValidator(object):
|
||||
self.raw_info = raw_info
|
||||
|
||||
def validate(self):
|
||||
logging.debug("Trying to validate stream descriptor for %s", str(self.raw_info['stream_name']))
|
||||
log.debug("Trying to validate stream descriptor for %s", str(self.raw_info['stream_name']))
|
||||
hex_stream_name = self.raw_info['stream_name']
|
||||
public_key = self.raw_info['public_key']
|
||||
key = self.raw_info['key']
|
||||
@@ -98,7 +101,7 @@ class LBRYLiveStreamDescriptorValidator(object):
|
||||
h.update(public_key)
|
||||
h.update(key)
|
||||
if h.hexdigest() != stream_hash:
|
||||
raise ValueError("Stream hash does not match stream metadata")
|
||||
raise InvalidStreamDescriptorError("Stream hash does not match stream metadata")
|
||||
blobs = self.raw_info['blobs']
|
||||
|
||||
def check_blob_signatures():
|
||||
@@ -121,11 +124,14 @@ class LBRYLiveStreamDescriptorValidator(object):
|
||||
hashsum.update(iv)
|
||||
hashsum.update(str(length))
|
||||
if not verify_signature(hashsum.digest(), signature, public_key):
|
||||
raise ValueError("Invalid signature in stream descriptor")
|
||||
raise InvalidStreamDescriptorError("Invalid signature in stream descriptor")
|
||||
|
||||
return threads.deferToThread(check_blob_signatures)
|
||||
|
||||
def info_to_show(self):
|
||||
info = []
|
||||
info.append(("stream_name", binascii.unhexlify(self.raw_info.get("stream_name"))))
|
||||
return info
|
||||
return info
|
||||
|
||||
def get_length_of_stream(self):
|
||||
return None
|
||||
@@ -1,5 +1,5 @@
|
||||
import binascii
|
||||
from lbrynet.core.DownloadOption import DownloadOption
|
||||
from lbrynet.core.StreamDescriptor import StreamMetadata
|
||||
from lbrynet.cryptstream.client.CryptStreamDownloader import CryptStreamDownloader
|
||||
from zope.interface import implements
|
||||
from lbrynet.lbrylive.client.LiveStreamMetadataHandler import LiveStreamMetadataHandler
|
||||
@@ -9,6 +9,7 @@ from lbrynet.lbrylive.StreamDescriptor import save_sd_info
|
||||
from lbrynet.lbrylive.PaymentRateManager import LiveStreamPaymentRateManager
|
||||
from twisted.internet import defer, threads # , process
|
||||
from lbrynet.interfaces import IStreamDownloaderFactory
|
||||
from lbrynet.lbrylive.StreamDescriptor import LiveStreamType
|
||||
|
||||
|
||||
class LiveStreamDownloader(CryptStreamDownloader):
|
||||
@@ -85,9 +86,9 @@ class FullLiveStreamDownloader(LiveStreamDownloader):
|
||||
d.addCallback(lambda _: set_file_name_if_unset())
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
def stop(self, err=None):
|
||||
d = self._close_file()
|
||||
d.addBoth(lambda _: LiveStreamDownloader.stop(self))
|
||||
d.addBoth(lambda _: LiveStreamDownloader.stop(self, err))
|
||||
return d
|
||||
|
||||
def _start(self):
|
||||
@@ -138,34 +139,24 @@ class FullLiveStreamDownloaderFactory(object):
|
||||
self.wallet = wallet
|
||||
self.default_payment_rate_manager = default_payment_rate_manager
|
||||
|
||||
def get_downloader_options(self, sd_validator, payment_rate_manager):
|
||||
options = [
|
||||
DownloadOption(
|
||||
[float, None],
|
||||
"rate which will be paid for data (None means use application default)",
|
||||
"data payment rate",
|
||||
None
|
||||
),
|
||||
DownloadOption(
|
||||
[float, None],
|
||||
"rate which will be paid for metadata (None means use application default)",
|
||||
"metadata payment rate",
|
||||
None
|
||||
),
|
||||
DownloadOption(
|
||||
[bool],
|
||||
"allow reuploading data downloaded for this file",
|
||||
"allow upload",
|
||||
True
|
||||
),
|
||||
]
|
||||
return options
|
||||
def can_download(self, sd_validator):
|
||||
return True
|
||||
|
||||
def make_downloader(self, sd_validator, options, payment_rate_manager):
|
||||
def make_downloader(self, metadata, options, payment_rate_manager):
|
||||
# TODO: check options for payment rate manager parameters
|
||||
payment_rate_manager = LiveStreamPaymentRateManager(self.default_payment_rate_manager,
|
||||
payment_rate_manager)
|
||||
d = save_sd_info(self.stream_info_manager, sd_validator.raw_info)
|
||||
|
||||
def save_source_if_blob(stream_hash):
|
||||
if metadata.metadata_source == StreamMetadata.FROM_BLOB:
|
||||
d = self.stream_info_manager.save_sd_blob_hash_to_stream(stream_hash, metadata.source_blob_hash)
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
d.addCallback(lambda _: stream_hash)
|
||||
return d
|
||||
|
||||
d = save_sd_info(self.stream_info_manager, metadata.validator.raw_info)
|
||||
d.addCallback(save_source_if_blob)
|
||||
|
||||
def create_downloader(stream_hash):
|
||||
stream_downloader = FullLiveStreamDownloader(stream_hash, self.peer_finder, self.rate_limiter,
|
||||
@@ -177,4 +168,15 @@ class FullLiveStreamDownloaderFactory(object):
|
||||
return d
|
||||
|
||||
d.addCallback(create_downloader)
|
||||
return d
|
||||
return d
|
||||
|
||||
|
||||
def add_full_live_stream_downloader_to_sd_identifier(session, stream_info_manager, sd_identifier,
|
||||
base_live_stream_payment_rate_manager):
|
||||
downloader_factory = FullLiveStreamDownloaderFactory(session.peer_finder,
|
||||
session.rate_limiter,
|
||||
session.blob_manager,
|
||||
stream_info_manager,
|
||||
session.wallet,
|
||||
base_live_stream_payment_rate_manager)
|
||||
sd_identifier.add_stream_downloader_factory(LiveStreamType, downloader_factory)
|
||||
@@ -9,7 +9,10 @@ from lbrynet.lbrylive.LiveBlob import LiveBlobInfo
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj, verify_signature
|
||||
from lbrynet.interfaces import IRequestCreator, IMetadataHandler
|
||||
from lbrynet.core.Error import InsufficientFundsError, InvalidResponseError, RequestCanceledError
|
||||
from lbrynet.core.Error import NoResponseError
|
||||
from lbrynet.core.Error import NoResponseError, ConnectionClosedBeforeResponseError
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LiveStreamMetadataHandler(object):
|
||||
@@ -79,7 +82,7 @@ class LiveStreamMetadataHandler(object):
|
||||
r = None
|
||||
if self._finished_discovery() is False:
|
||||
r = self.stream_hash
|
||||
logging.debug("Info finder peer search response for stream %s: %s", str(self.stream_hash), str(r))
|
||||
log.debug("Info finder peer search response for stream %s: %s", str(self.stream_hash), str(r))
|
||||
return defer.succeed(r)
|
||||
|
||||
def _find_peers_for_hash(self, h):
|
||||
@@ -101,7 +104,7 @@ class LiveStreamMetadataHandler(object):
|
||||
if blob_hash is not None:
|
||||
infos.append(LiveBlobInfo(blob_hash, blob_num, length, iv, revision, signature))
|
||||
else:
|
||||
logging.debug("Setting _final_blob_num to %s", str(blob_num - 1))
|
||||
log.debug("Setting _final_blob_num to %s", str(blob_num - 1))
|
||||
self._final_blob_num = blob_num - 1
|
||||
return infos
|
||||
|
||||
@@ -134,7 +137,7 @@ class LiveStreamMetadataHandler(object):
|
||||
further_blobs_request['count'] = count
|
||||
else:
|
||||
further_blobs_request['count'] = MAX_BLOB_INFOS_TO_REQUEST
|
||||
logging.debug("Requesting %s blob infos from %s", str(further_blobs_request['count']), str(peer))
|
||||
log.debug("Requesting %s blob infos from %s", str(further_blobs_request['count']), str(peer))
|
||||
r_dict = {'further_blobs': further_blobs_request}
|
||||
response_identifier = 'further_blobs'
|
||||
request = ClientPaidRequest(r_dict, response_identifier, further_blobs_request['count'])
|
||||
@@ -142,7 +145,7 @@ class LiveStreamMetadataHandler(object):
|
||||
return None
|
||||
|
||||
def _get_discovery_params(self):
|
||||
logging.debug("In _get_discovery_params")
|
||||
log.debug("In _get_discovery_params")
|
||||
stream_position = self.download_manager.stream_position()
|
||||
blobs = self.download_manager.blobs
|
||||
if blobs:
|
||||
@@ -153,7 +156,7 @@ class LiveStreamMetadataHandler(object):
|
||||
if final_blob_num is not None:
|
||||
last_blob_num = final_blob_num
|
||||
if self.download_whole is False:
|
||||
logging.debug("download_whole is False")
|
||||
log.debug("download_whole is False")
|
||||
if final_blob_num is not None:
|
||||
for i in xrange(stream_position, final_blob_num + 1):
|
||||
if not i in blobs:
|
||||
@@ -171,7 +174,7 @@ class LiveStreamMetadataHandler(object):
|
||||
return self.stream_hash, blobs[last_blob_num].blob_hash, 'end', -1 * self.max_before_skip_ahead
|
||||
else:
|
||||
return self.stream_hash, None, 'end', -1 * self.max_before_skip_ahead
|
||||
logging.debug("download_whole is True")
|
||||
log.debug("download_whole is True")
|
||||
beginning = None
|
||||
end = None
|
||||
for i in xrange(stream_position, last_blob_num + 1):
|
||||
@@ -187,17 +190,17 @@ class LiveStreamMetadataHandler(object):
|
||||
break
|
||||
if beginning is None:
|
||||
if final_blob_num is not None:
|
||||
logging.debug("Discovery is finished. stream_position: %s, last_blob_num + 1: %s", str(stream_position),
|
||||
str(last_blob_num + 1))
|
||||
log.debug("Discovery is finished. stream_position: %s, last_blob_num + 1: %s", str(stream_position),
|
||||
str(last_blob_num + 1))
|
||||
return None
|
||||
else:
|
||||
logging.debug("Discovery is not finished. final blob num is unknown.")
|
||||
log.debug("Discovery is not finished. final blob num is unknown.")
|
||||
if last_blob_num != -1:
|
||||
return self.stream_hash, blobs[last_blob_num].blob_hash, None, None
|
||||
else:
|
||||
return self.stream_hash, 'beginning', None, None
|
||||
else:
|
||||
logging.info("Discovery is not finished. Not all blobs are known.")
|
||||
log.info("Discovery is not finished. Not all blobs are known.")
|
||||
return self.stream_hash, beginning, end, None
|
||||
|
||||
def _price_settled(self, protocol):
|
||||
@@ -244,7 +247,7 @@ class LiveStreamMetadataHandler(object):
|
||||
if response == "RATE_ACCEPTED":
|
||||
return True
|
||||
else:
|
||||
logging.info("Rate offer has been rejected by %s", str(peer))
|
||||
log.info("Rate offer has been rejected by %s", str(peer))
|
||||
del self._protocol_prices[protocol]
|
||||
self._price_disagreements.append(peer)
|
||||
return True
|
||||
@@ -263,8 +266,8 @@ class LiveStreamMetadataHandler(object):
|
||||
if not 'blob_infos' in response:
|
||||
return InvalidResponseError("Missing the required field 'blob_infos'")
|
||||
raw_blob_infos = response['blob_infos']
|
||||
logging.info("Handling %s further blobs from %s", str(len(raw_blob_infos)), str(peer))
|
||||
logging.debug("blobs: %s", str(raw_blob_infos))
|
||||
log.info("Handling %s further blobs from %s", str(len(raw_blob_infos)), str(peer))
|
||||
log.debug("blobs: %s", str(raw_blob_infos))
|
||||
for raw_blob_info in raw_blob_infos:
|
||||
length = raw_blob_info['length']
|
||||
if length != 0:
|
||||
@@ -276,10 +279,10 @@ class LiveStreamMetadataHandler(object):
|
||||
iv = raw_blob_info['iv']
|
||||
signature = raw_blob_info['signature']
|
||||
blob_info = LiveBlobInfo(blob_hash, num, length, iv, revision, signature)
|
||||
logging.debug("Learned about a potential blob: %s", str(blob_hash))
|
||||
log.debug("Learned about a potential blob: %s", str(blob_hash))
|
||||
if self._verify_blob(blob_info):
|
||||
if blob_hash is None:
|
||||
logging.info("Setting _final_blob_num to %s", str(num - 1))
|
||||
log.info("Setting _final_blob_num to %s", str(num - 1))
|
||||
self._final_blob_num = num - 1
|
||||
else:
|
||||
blob_infos.append(blob_info)
|
||||
@@ -289,7 +292,7 @@ class LiveStreamMetadataHandler(object):
|
||||
|
||||
def add_blobs_to_download_manager():
|
||||
blob_nums = [b.blob_num for b in blob_infos]
|
||||
logging.info("Adding the following blob nums to the download manager: %s", str(blob_nums))
|
||||
log.info("Adding the following blob nums to the download manager: %s", str(blob_nums))
|
||||
self.download_manager.add_blobs_to_download(blob_infos)
|
||||
|
||||
d.addCallback(lambda _: add_blobs_to_download_manager())
|
||||
@@ -308,12 +311,12 @@ class LiveStreamMetadataHandler(object):
|
||||
return d
|
||||
|
||||
def _verify_blob(self, blob):
|
||||
logging.debug("Got an unverified blob to check:")
|
||||
logging.debug("blob_hash: %s", blob.blob_hash)
|
||||
logging.debug("blob_num: %s", str(blob.blob_num))
|
||||
logging.debug("revision: %s", str(blob.revision))
|
||||
logging.debug("iv: %s", blob.iv)
|
||||
logging.debug("length: %s", str(blob.length))
|
||||
log.debug("Got an unverified blob to check:")
|
||||
log.debug("blob_hash: %s", blob.blob_hash)
|
||||
log.debug("blob_num: %s", str(blob.blob_num))
|
||||
log.debug("revision: %s", str(blob.revision))
|
||||
log.debug("iv: %s", blob.iv)
|
||||
log.debug("length: %s", str(blob.length))
|
||||
hashsum = get_lbry_hash_obj()
|
||||
hashsum.update(self.stream_hash)
|
||||
if blob.length != 0:
|
||||
@@ -322,12 +325,12 @@ class LiveStreamMetadataHandler(object):
|
||||
hashsum.update(str(blob.revision))
|
||||
hashsum.update(blob.iv)
|
||||
hashsum.update(str(blob.length))
|
||||
logging.debug("hexdigest to be verified: %s", hashsum.hexdigest())
|
||||
log.debug("hexdigest to be verified: %s", hashsum.hexdigest())
|
||||
if verify_signature(hashsum.digest(), blob.signature, self.stream_pub_key):
|
||||
logging.debug("Blob info is valid")
|
||||
log.debug("Blob info is valid")
|
||||
return True
|
||||
else:
|
||||
logging.debug("The blob info is invalid")
|
||||
log.debug("The blob info is invalid")
|
||||
return False
|
||||
|
||||
def _request_failed(self, reason, peer):
|
||||
@@ -335,8 +338,9 @@ class LiveStreamMetadataHandler(object):
|
||||
return
|
||||
if reason.check(NoResponseError):
|
||||
self._incompatible_peers.append(peer)
|
||||
return
|
||||
logging.warning("Crypt stream info finder: a request failed. Reason: %s", reason.getErrorMessage())
|
||||
log.warning("Crypt stream info finder: a request failed. Reason: %s", reason.getErrorMessage())
|
||||
self._update_local_score(peer, -5.0)
|
||||
peer.update_score(-10.0)
|
||||
if reason.check(ConnectionClosedBeforeResponseError):
|
||||
return
|
||||
return reason
|
||||
@@ -0,0 +1,73 @@
|
||||
from lbrynet.lbrylive.StreamDescriptor import LiveStreamType, LBRYLiveStreamDescriptorValidator
|
||||
from lbrynet.core.DownloadOption import DownloadOption, DownloadOptionChoice
|
||||
|
||||
|
||||
def add_live_stream_to_sd_identifier(sd_identifier, base_live_stream_payment_rate_manager):
|
||||
sd_identifier.add_stream_type(LiveStreamType, LBRYLiveStreamDescriptorValidator,
|
||||
LiveStreamOptions(base_live_stream_payment_rate_manager))
|
||||
|
||||
|
||||
class LiveStreamOptions(object):
|
||||
def __init__(self, base_live_stream_payment_rate_manager):
|
||||
self.base_live_stream_prm = base_live_stream_payment_rate_manager
|
||||
|
||||
def get_downloader_options(self, sd_validator, payment_rate_manager):
|
||||
prm = payment_rate_manager
|
||||
|
||||
def get_default_data_rate_description():
|
||||
if prm.min_blob_data_payment_rate is None:
|
||||
return "Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate)
|
||||
else:
|
||||
return "%f LBC/MB" % prm.min_blob_data_payment_rate
|
||||
|
||||
options = [
|
||||
DownloadOption(
|
||||
[
|
||||
DownloadOptionChoice(None,
|
||||
"No change",
|
||||
"No change"),
|
||||
DownloadOptionChoice(None,
|
||||
"Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate),
|
||||
"Default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate)),
|
||||
DownloadOptionChoice(float,
|
||||
"Rate in LBC/MB",
|
||||
"Rate in LBC/MB")
|
||||
],
|
||||
"rate which will be paid for data",
|
||||
"data payment rate",
|
||||
prm.min_blob_data_payment_rate,
|
||||
get_default_data_rate_description()
|
||||
),
|
||||
DownloadOption(
|
||||
[
|
||||
DownloadOptionChoice(None,
|
||||
"No change",
|
||||
"No change"),
|
||||
DownloadOptionChoice(None,
|
||||
"Application default (%s LBC/MB)" % str(self.base_live_stream_prm.min_live_blob_info_payment_rate),
|
||||
"Default (%s LBC/MB)" % str(self.base_live_stream_prm.min_live_blob_info_payment_rate)),
|
||||
DownloadOptionChoice(float,
|
||||
"Rate in LBC/MB",
|
||||
"Rate in LBC/MB")
|
||||
],
|
||||
"rate which will be paid for metadata",
|
||||
"metadata payment rate",
|
||||
None,
|
||||
"Application default (%s LBC/MB)" % str(self.base_live_stream_prm.min_live_blob_info_payment_rate)
|
||||
),
|
||||
DownloadOption(
|
||||
[
|
||||
DownloadOptionChoice(True,
|
||||
"Allow reuploading data downloaded for this file",
|
||||
"Allow reuploading"),
|
||||
DownloadOptionChoice(False,
|
||||
"Disallow reuploading data downloaded for this file",
|
||||
"Disallow reuploading")
|
||||
],
|
||||
"allow reuploading data downloaded for this file",
|
||||
"allow upload",
|
||||
True,
|
||||
"Allow"
|
||||
),
|
||||
]
|
||||
return options
|
||||
@@ -3,6 +3,9 @@ from lbrynet.core.client.StreamProgressManager import StreamProgressManager
|
||||
from twisted.internet import defer
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LiveStreamProgressManager(StreamProgressManager):
|
||||
def __init__(self, finished_callback, blob_manager, download_manager, delete_blob_after_finished=False,
|
||||
download_whole=True, max_before_skip_ahead=5):
|
||||
@@ -52,9 +55,9 @@ class LiveStreamProgressManager(StreamProgressManager):
|
||||
return
|
||||
|
||||
blobs = self.download_manager.blobs
|
||||
logging.info("In _output_loop. last_blob_outputted: %s", str(self.last_blob_outputted))
|
||||
log.info("In _output_loop. last_blob_outputted: %s", str(self.last_blob_outputted))
|
||||
if blobs:
|
||||
logging.debug("Newest blob number: %s", str(max(blobs.iterkeys())))
|
||||
log.debug("Newest blob number: %s", str(max(blobs.iterkeys())))
|
||||
if self.outputting_d is None:
|
||||
self.outputting_d = defer.Deferred()
|
||||
|
||||
@@ -71,15 +74,15 @@ class LiveStreamProgressManager(StreamProgressManager):
|
||||
reactor.callLater(0, self._output_loop)
|
||||
|
||||
if current_blob_num in blobs and blobs[current_blob_num].is_validated():
|
||||
logging.info("Outputting blob %s", str(current_blob_num))
|
||||
log.info("Outputting blob %s", str(current_blob_num))
|
||||
self.provided_blob_nums.append(current_blob_num)
|
||||
d = self.download_manager.handle_blob(current_blob_num)
|
||||
d.addCallback(lambda _: finished_outputting_blob())
|
||||
d.addCallback(lambda _: self._finished_with_blob(current_blob_num))
|
||||
elif blobs and max(blobs.iterkeys()) > self.last_blob_outputted + self.max_before_skip_ahead - 1:
|
||||
self.last_blob_outputted += 1
|
||||
logging.info("Skipping blob number %s due to knowing about blob number %s",
|
||||
str(self.last_blob_outputted), str(max(blobs.iterkeys())))
|
||||
log.info("Skipping blob number %s due to knowing about blob number %s",
|
||||
str(self.last_blob_outputted), str(max(blobs.iterkeys())))
|
||||
self._finished_with_blob(current_blob_num)
|
||||
reactor.callLater(0, self._output_loop)
|
||||
else:
|
||||
|
||||
@@ -4,6 +4,9 @@ from zope.interface import implements
|
||||
from lbrynet.interfaces import IQueryHandlerFactory, IQueryHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CryptBlobInfoQueryHandlerFactory(object):
|
||||
implements(IQueryHandlerFactory)
|
||||
|
||||
@@ -54,7 +57,7 @@ class CryptBlobInfoQueryHandler(object):
|
||||
|
||||
if self.query_identifiers[1] in queries:
|
||||
further_blobs_request = queries[self.query_identifiers[1]]
|
||||
logging.debug("Received the client's request for additional blob information")
|
||||
log.debug("Received the client's request for additional blob information")
|
||||
|
||||
if self.blob_info_payment_rate is None:
|
||||
response['further_blobs'] = {'error': 'RATE_UNSET'}
|
||||
@@ -62,7 +65,7 @@ class CryptBlobInfoQueryHandler(object):
|
||||
|
||||
def count_and_charge(blob_infos):
|
||||
if len(blob_infos) != 0:
|
||||
logging.info("Responding with %s infos", str(len(blob_infos)))
|
||||
log.info("Responding with %s infos", str(len(blob_infos)))
|
||||
expected_payment = 1.0 * len(blob_infos) * self.blob_info_payment_rate / 1000.0
|
||||
self.wallet.add_expected_payment(self.peer, expected_payment)
|
||||
self.peer.update_stats('uploaded_crypt_blob_infos', len(blob_infos))
|
||||
|
||||
@@ -1,60 +1,176 @@
|
||||
from twisted.protocols import basic
|
||||
from twisted.internet import defer
|
||||
import logging
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ConsoleControl(basic.LineReceiver):
|
||||
from os import linesep as delimiter
|
||||
|
||||
def __init__(self, control_handlers):
|
||||
self.control_handlers = {}
|
||||
self.categories = {}
|
||||
categories = set([category for category, handler in control_handlers])
|
||||
prompt_number = 0
|
||||
for category in categories:
|
||||
self.categories[prompt_number] = category
|
||||
for handler in [handler for cat, handler in control_handlers if cat == category]:
|
||||
self.control_handlers[prompt_number] = handler
|
||||
prompt_number += 1
|
||||
def __init__(self):
|
||||
self.connected = False
|
||||
self.buffer = []
|
||||
self.command_handlers = {}
|
||||
self.current_handler = None
|
||||
|
||||
def start(self, command_handlers):
|
||||
self.command_handlers = {h.command: h for h in command_handlers}
|
||||
self.current_handler = None
|
||||
self.send_initial_prompt()
|
||||
return defer.succeed(True)
|
||||
|
||||
def connectionMade(self):
|
||||
self.connected = True
|
||||
if self.buffer:
|
||||
self.send(self.buffer)
|
||||
self.buffer = []
|
||||
|
||||
def send_initial_prompt(self):
|
||||
self.sendLine("")
|
||||
self.sendLine("You should have received 1000 LBC the first time you ran\n"
|
||||
"this program. If you did not, let us know! But first give\n"
|
||||
"them a moment to show up. The block time is currently 30\n"
|
||||
"seconds so they should show up within a couple of minutes.\n\n"
|
||||
"Welcome to lbrynet-console!")
|
||||
self.sendLine("")
|
||||
self.sendLine("Enter a command. Try 'get wonderfullife' or 'help' to see more options.")
|
||||
self.show_prompt()
|
||||
|
||||
def send(self, s):
|
||||
self.transport.write(s)
|
||||
|
||||
def write(self, s):
|
||||
if self.connected is False:
|
||||
self.buffer.append(s)
|
||||
else:
|
||||
self.send(s)
|
||||
|
||||
def flush(self):
|
||||
if self.connected is True and self.buffer:
|
||||
self.send(self.buffer)
|
||||
self.buffer = []
|
||||
|
||||
def show_prompt(self):
|
||||
self.send("> ")
|
||||
|
||||
def show_quick_help(self):
|
||||
self.sendLine("Available commands:")
|
||||
self.sendLine("")
|
||||
showed_help_all = False
|
||||
sorted_handlers = sorted(self.command_handlers.items(), key=lambda x: x[0])
|
||||
sorted_handlers = sorted(sorted_handlers, key=lambda x: x[1].priority, reverse=True)
|
||||
for command, handler in sorted_handlers:
|
||||
if handler.priority > 0:
|
||||
if showed_help_all is False and handler.priority < 10:
|
||||
self.sendLine("help-all - Show the full list of available commands")
|
||||
showed_help_all = True
|
||||
self.sendLine(command + " - " + handler.short_help)
|
||||
self.sendLine("")
|
||||
self.sendLine("For more information about any command type 'help <command>'")
|
||||
|
||||
def show_full_help(self):
|
||||
self.sendLine("Available commands:")
|
||||
self.sendLine("")
|
||||
for command, handler in sorted(self.command_handlers.items(), key=lambda x: x[0]):
|
||||
self.sendLine(command + " - " + handler.short_help)
|
||||
self.sendLine("")
|
||||
self.sendLine("For more information about any command type 'help <command>'")
|
||||
|
||||
def handler_done(self):
|
||||
self.current_handler = None
|
||||
self.show_prompt()
|
||||
|
||||
def handler_failed(self, err):
|
||||
self.current_handler = None
|
||||
self.sendLine("An error occurred:")
|
||||
self.sendLine(err.getTraceback())
|
||||
self.show_prompt()
|
||||
|
||||
def lineReceived(self, line):
|
||||
|
||||
def show_response(response):
|
||||
if response is not None:
|
||||
self.sendLine(response)
|
||||
|
||||
def show_error(err):
|
||||
self.sendLine(err.getTraceback())
|
||||
|
||||
if not self.command_handlers:
|
||||
return
|
||||
if self.current_handler is None:
|
||||
words = line.split()
|
||||
if len(words) == 0:
|
||||
self.show_prompt()
|
||||
return
|
||||
command, args = words[0], words[1:]
|
||||
if command == "help":
|
||||
if len(args) == 0:
|
||||
self.show_quick_help()
|
||||
self.show_prompt()
|
||||
return
|
||||
if args[0] in self.command_handlers:
|
||||
self.sendLine(self.command_handlers[args[0]].full_help)
|
||||
self.show_prompt()
|
||||
return
|
||||
if args[0] == "help-all":
|
||||
self.sendLine("Show the full list of available commands!")
|
||||
self.show_prompt()
|
||||
return
|
||||
self.sendLine("Can't help you with '%s'. Sorry!" % args[0])
|
||||
self.show_prompt()
|
||||
return
|
||||
elif command == "help-all":
|
||||
self.show_full_help()
|
||||
self.show_prompt()
|
||||
return
|
||||
if command in self.command_handlers:
|
||||
command_handler = self.command_handlers[command]
|
||||
else:
|
||||
candidates = [k for k in self.command_handlers.keys() if k.startswith(command)]
|
||||
if len(candidates) == 0:
|
||||
self.sendLine("Unknown command. Type 'help' for a list of commands.")
|
||||
self.show_prompt()
|
||||
return
|
||||
if len(candidates) >= 2:
|
||||
l = "Ambiguous command. Matches: "
|
||||
for candidate in candidates:
|
||||
l += candidate
|
||||
l += ", "
|
||||
l = l[:-2]
|
||||
l += l
|
||||
self.sendLine(l)
|
||||
self.show_prompt()
|
||||
return
|
||||
else:
|
||||
command_handler = self.command_handlers[candidates[0]]
|
||||
try:
|
||||
num = int(line)
|
||||
except ValueError:
|
||||
num = None
|
||||
if num in self.control_handlers:
|
||||
self.current_handler = self.control_handlers[num].get_handler()
|
||||
line = None
|
||||
if self.current_handler is not None:
|
||||
try:
|
||||
r = self.current_handler.handle_line(line)
|
||||
done, ds = r[0], [d for d in r[1:] if d is not None]
|
||||
except Exception as e:
|
||||
done = True
|
||||
ds = [defer.fail(e)]
|
||||
if done is True:
|
||||
self.current_handler = command_handler.get_handler(self)
|
||||
except:
|
||||
self.current_handler = None
|
||||
map(lambda d: d.addCallbacks(show_response, show_error), ds)
|
||||
if self.current_handler is None:
|
||||
self.show_prompt()
|
||||
|
||||
def show_prompt(self):
|
||||
self.sendLine("Options:")
|
||||
for num, handler in self.control_handlers.iteritems():
|
||||
if num in self.categories:
|
||||
self.sendLine("")
|
||||
self.sendLine(self.categories[num])
|
||||
self.sendLine("")
|
||||
self.sendLine("[" + str(num) + "] " + handler.get_prompt_description())
|
||||
import traceback
|
||||
self.sendLine(traceback.format_exc())
|
||||
log.error(traceback.format_exc())
|
||||
self.show_prompt()
|
||||
return
|
||||
try:
|
||||
self.current_handler.start(*args)
|
||||
except TypeError:
|
||||
self.current_handler = None
|
||||
self.sendLine("Invalid arguments. Type 'help <command>' for the argument list.")
|
||||
import traceback
|
||||
log.error(traceback.format_exc())
|
||||
self.show_prompt()
|
||||
return
|
||||
except:
|
||||
self.current_handler = None
|
||||
import traceback
|
||||
self.sendLine(traceback.format_exc())
|
||||
log.error(traceback.format_exc())
|
||||
self.show_prompt()
|
||||
return
|
||||
self.current_handler.finished_deferred.addCallbacks(lambda _: self.handler_done(),
|
||||
self.handler_failed)
|
||||
else:
|
||||
try:
|
||||
self.current_handler.handle_line(line)
|
||||
except Exception as e:
|
||||
self.current_handler = None
|
||||
import traceback
|
||||
self.sendLine(traceback.format_exc())
|
||||
log.error(traceback.format_exc())
|
||||
self.show_prompt()
|
||||
return
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,13 +1,22 @@
|
||||
import logging
|
||||
from lbrynet.core.Session import LBRYSession
|
||||
import os.path
|
||||
import argparse
|
||||
import requests
|
||||
import locale
|
||||
import sys
|
||||
import webbrowser
|
||||
|
||||
if sys.platform == "darwin":
|
||||
from appdirs import user_data_dir
|
||||
from yapsy.PluginManager import PluginManager
|
||||
from twisted.internet import defer, threads, stdio, task
|
||||
from twisted.internet import defer, threads, stdio, task, error
|
||||
from jsonrpc.proxy import JSONRPCProxy
|
||||
|
||||
from lbrynet.core.Session import LBRYSession
|
||||
from lbrynet.lbrynet_console.ConsoleControl import ConsoleControl
|
||||
from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings
|
||||
from lbrynet.lbryfilemanager.LBRYFileManager import LBRYFileManager
|
||||
from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE # , MIN_BLOB_INFO_PAYMENT_RATE
|
||||
from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, API_CONNECTION_STRING # , MIN_BLOB_INFO_PAYMENT_RATE
|
||||
from lbrynet.core.utils import generate_id
|
||||
from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier
|
||||
from lbrynet.core.PaymentRateManager import PaymentRateManager
|
||||
@@ -15,11 +24,12 @@ from lbrynet.core.server.BlobAvailabilityHandler import BlobAvailabilityHandlerF
|
||||
from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory
|
||||
from lbrynet.core.server.ServerProtocol import ServerProtocolFactory
|
||||
from lbrynet.core.PTCWallet import PTCWallet
|
||||
from lbrynet.lbryfile.client.LBRYFileOptions import add_lbry_file_to_sd_identifier
|
||||
from lbrynet.lbryfile.client.LBRYFileDownloader import LBRYFileOpenerFactory
|
||||
from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType
|
||||
from lbrynet.lbryfile.LBRYFileMetadataManager import DBLBRYFileMetadataManager, TempLBRYFileMetadataManager
|
||||
#from lbrynet.lbrylive.PaymentRateManager import LiveStreamPaymentRateManager
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ApplicationStatusFactory, GetWalletBalancesFactory, ShutDownFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ImmediateAnnounceAllBlobsFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import LBRYFileStatusFactory, DeleteLBRYFileChooserFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ToggleLBRYFileRunningChooserFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ModifyApplicationDefaultsFactory
|
||||
@@ -28,17 +38,23 @@ from lbrynet.lbrynet_console.ControlHandlers import ShowPublishedSDHashesChooser
|
||||
from lbrynet.lbrynet_console.ControlHandlers import CreatePlainStreamDescriptorChooserFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ShowLBRYFileStreamHashChooserFactory, AddStreamFromHashFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import AddStreamFromSDFactory, AddStreamFromLBRYcrdNameFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ClaimNameFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ClaimNameFactory, GetNewWalletAddressFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ShowServerStatusFactory, ModifyServerSettingsFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ModifyLBRYFileOptionsChooserFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import PeerStatsAndSettingsChooserFactory
|
||||
from lbrynet.core.LBRYcrdWallet import LBRYcrdWallet
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ModifyLBRYFileOptionsChooserFactory, StatusFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import PeerStatsAndSettingsChooserFactory, PublishFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import BlockchainStatusFactory
|
||||
from lbrynet.core.LBRYWallet import LBRYcrdWallet, LBRYumWallet
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
alert = logging.getLogger("lbryalert." + __name__)
|
||||
|
||||
|
||||
class LBRYConsole():
|
||||
"""A class which can upload and download file streams to and from the network"""
|
||||
def __init__(self, peer_port, dht_node_port, known_dht_nodes, control_class, wallet_type, lbrycrd_rpc_port,
|
||||
use_upnp, conf_dir, data_dir):
|
||||
def __init__(self, peer_port, dht_node_port, known_dht_nodes, fake_wallet,
|
||||
lbrycrd_conf, lbrycrd_dir, use_upnp, data_dir, created_data_dir,
|
||||
lbrycrdd_path):
|
||||
"""
|
||||
@param peer_port: the network port on which to listen for peers
|
||||
|
||||
@@ -49,34 +65,49 @@ class LBRYConsole():
|
||||
self.peer_port = peer_port
|
||||
self.dht_node_port = dht_node_port
|
||||
self.known_dht_nodes = known_dht_nodes
|
||||
self.wallet_type = wallet_type
|
||||
self.wallet_rpc_port = lbrycrd_rpc_port
|
||||
self.fake_wallet = fake_wallet
|
||||
self.lbrycrd_conf = lbrycrd_conf
|
||||
self.lbrycrd_dir = lbrycrd_dir
|
||||
if not self.lbrycrd_dir:
|
||||
if sys.platform == "darwin":
|
||||
self.lbrycrd_dir = os.path.join(os.path.expanduser("~"), "Library/Application Support/lbrycrd")
|
||||
else:
|
||||
self.lbrycrd_dir = os.path.join(os.path.expanduser("~"), ".lbrycrd")
|
||||
if not self.lbrycrd_conf:
|
||||
self.lbrycrd_conf = os.path.join(self.lbrycrd_dir, "lbrycrd.conf")
|
||||
self.lbrycrdd_path = lbrycrdd_path
|
||||
self.use_upnp = use_upnp
|
||||
self.lbry_server_port = None
|
||||
self.control_class = control_class
|
||||
self.session = None
|
||||
self.lbry_file_metadata_manager = None
|
||||
self.lbry_file_manager = None
|
||||
self.conf_dir = conf_dir
|
||||
self.data_dir = data_dir
|
||||
self.db_dir = data_dir
|
||||
self.current_db_revision = 1
|
||||
self.blobfile_dir = os.path.join(self.db_dir, "blobfiles")
|
||||
self.created_data_dir = created_data_dir
|
||||
self.plugin_manager = PluginManager()
|
||||
self.plugin_manager.setPluginPlaces([
|
||||
os.path.join(self.conf_dir, "plugins"),
|
||||
os.path.join(self.db_dir, "plugins"),
|
||||
os.path.join(os.path.dirname(__file__), "plugins"),
|
||||
])
|
||||
self.control_handlers = []
|
||||
self.command_handlers = []
|
||||
self.query_handlers = {}
|
||||
|
||||
self.settings = LBRYSettings(self.conf_dir)
|
||||
self.settings = LBRYSettings(self.db_dir)
|
||||
self.blob_request_payment_rate_manager = None
|
||||
self.lbryid = None
|
||||
self.sd_identifier = StreamDescriptorIdentifier()
|
||||
self.plugin_objects = []
|
||||
self.db_migration_revisions = None
|
||||
|
||||
def start(self):
|
||||
"""Initialize the session and restore everything to its saved state"""
|
||||
d = threads.deferToThread(self._create_directory)
|
||||
d = self._setup_controller()
|
||||
d.addCallback(lambda _: threads.deferToThread(self._setup_data_directory))
|
||||
d.addCallback(lambda _: self._check_db_migration())
|
||||
d.addCallback(lambda _: self._get_settings())
|
||||
d.addCallback(lambda _: self._get_session())
|
||||
d.addCallback(lambda _: add_lbry_file_to_sd_identifier(self.sd_identifier))
|
||||
d.addCallback(lambda _: self._setup_lbry_file_manager())
|
||||
d.addCallback(lambda _: self._setup_lbry_file_opener())
|
||||
d.addCallback(lambda _: self._setup_control_handlers())
|
||||
@@ -84,17 +115,30 @@ class LBRYConsole():
|
||||
d.addCallback(lambda _: self._load_plugins())
|
||||
d.addCallback(lambda _: self._setup_server())
|
||||
d.addCallback(lambda _: self._start_controller())
|
||||
d.addErrback(self._show_start_error)
|
||||
return d
|
||||
|
||||
def _show_start_error(self, error):
|
||||
print error.getTraceback()
|
||||
log.error("An error occurred during start up: %s", error.getTraceback())
|
||||
return error
|
||||
|
||||
def _show_shutdown_error(self, error):
|
||||
print error.getErrorMessage()
|
||||
log.error("An error occurred during shutdown: %s", error.getTraceback())
|
||||
return error
|
||||
|
||||
def shut_down(self):
|
||||
"""Stop the session, all currently running streams, and stop the server"""
|
||||
d = self.session.shut_down()
|
||||
d.addCallback(lambda _: self._shut_down())
|
||||
d = self._shut_down()
|
||||
if self.session is not None:
|
||||
d.addCallback(lambda _: self.session.shut_down())
|
||||
d.addErrback(self._show_shutdown_error)
|
||||
return d
|
||||
|
||||
def add_control_handlers(self, control_handlers):
|
||||
for control_handler in control_handlers:
|
||||
self.control_handlers.append(control_handler)
|
||||
self.command_handlers.append(control_handler)
|
||||
|
||||
def add_query_handlers(self, query_handlers):
|
||||
|
||||
@@ -111,13 +155,40 @@ class LBRYConsole():
|
||||
dl.addCallback(_set_query_handlers)
|
||||
return dl
|
||||
|
||||
def _create_directory(self):
|
||||
if not os.path.exists(self.conf_dir):
|
||||
os.makedirs(self.conf_dir)
|
||||
logging.debug("Created the configuration directory: %s", str(self.conf_dir))
|
||||
if not os.path.exists(self.data_dir):
|
||||
os.makedirs(self.data_dir)
|
||||
logging.debug("Created the data directory: %s", str(self.data_dir))
|
||||
def _setup_data_directory(self):
|
||||
alert.info("Loading databases...")
|
||||
if self.created_data_dir:
|
||||
db_revision = open(os.path.join(self.db_dir, "db_revision"), mode='w')
|
||||
db_revision.write(str(self.current_db_revision))
|
||||
db_revision.close()
|
||||
log.debug("Created the db revision file: %s", str(os.path.join(self.db_dir, "db_revision")))
|
||||
if not os.path.exists(self.blobfile_dir):
|
||||
os.mkdir(self.blobfile_dir)
|
||||
log.debug("Created the blobfile directory: %s", str(self.blobfile_dir))
|
||||
|
||||
def _check_db_migration(self):
|
||||
old_revision = 1
|
||||
db_revision_file = os.path.join(self.db_dir, "db_revision")
|
||||
if os.path.exists(db_revision_file):
|
||||
old_revision = int(open(db_revision_file).read().strip())
|
||||
if old_revision < self.current_db_revision:
|
||||
from lbrynet.db_migrator import dbmigrator
|
||||
print "Upgrading your databases..."
|
||||
d = threads.deferToThread(dbmigrator.migrate_db, self.db_dir, old_revision, self.current_db_revision)
|
||||
|
||||
def print_success(old_dirs):
|
||||
success_string = "Finished upgrading the databases. It is now safe to delete the"
|
||||
success_string += " following directories, if you feel like it. It won't make any"
|
||||
success_string += " difference.\nAnyway here they are: "
|
||||
for i, old_dir in enumerate(old_dirs):
|
||||
success_string += old_dir
|
||||
if i + 1 < len(old_dir):
|
||||
success_string += ", "
|
||||
print success_string
|
||||
|
||||
d.addCallback(print_success)
|
||||
return d
|
||||
return defer.succeed(True)
|
||||
|
||||
def _get_settings(self):
|
||||
d = self.settings.start()
|
||||
@@ -137,28 +208,96 @@ class LBRYConsole():
|
||||
return d
|
||||
|
||||
def _get_session(self):
|
||||
d = self.settings.get_default_data_payment_rate()
|
||||
def get_default_data_rate():
|
||||
d = self.settings.get_default_data_payment_rate()
|
||||
d.addCallback(lambda rate: {"default_data_payment_rate": rate if rate is not None else MIN_BLOB_DATA_PAYMENT_RATE})
|
||||
return d
|
||||
|
||||
def create_session(default_data_payment_rate):
|
||||
if default_data_payment_rate is None:
|
||||
default_data_payment_rate = MIN_BLOB_DATA_PAYMENT_RATE
|
||||
if self.wallet_type == "lbrycrd":
|
||||
wallet = LBRYcrdWallet("rpcuser", "rpcpassword", "127.0.0.1", self.wallet_rpc_port)
|
||||
def get_wallet():
|
||||
if self.fake_wallet:
|
||||
d = defer.succeed(PTCWallet(self.db_dir))
|
||||
elif self.lbrycrdd_path is not None:
|
||||
d = defer.succeed(LBRYcrdWallet(self.db_dir, wallet_dir=self.lbrycrd_dir,
|
||||
wallet_conf=self.lbrycrd_conf,
|
||||
lbrycrdd_path=self.lbrycrdd_path))
|
||||
else:
|
||||
wallet = PTCWallet(self.conf_dir)
|
||||
self.session = LBRYSession(default_data_payment_rate, db_dir=self.conf_dir, lbryid=self.lbryid,
|
||||
blob_dir=self.data_dir, dht_node_port=self.dht_node_port,
|
||||
d = defer.succeed(LBRYumWallet(self.db_dir))
|
||||
d.addCallback(lambda wallet: {"wallet": wallet})
|
||||
return d
|
||||
|
||||
d1 = get_default_data_rate()
|
||||
d2 = get_wallet()
|
||||
|
||||
def combine_results(results):
|
||||
r = {}
|
||||
for success, result in results:
|
||||
if success is True:
|
||||
r.update(result)
|
||||
return r
|
||||
|
||||
def create_session(results):
|
||||
|
||||
alert.info("Databases loaded.")
|
||||
|
||||
self.session = LBRYSession(results['default_data_payment_rate'], db_dir=self.db_dir, lbryid=self.lbryid,
|
||||
blob_dir=self.blobfile_dir, dht_node_port=self.dht_node_port,
|
||||
known_dht_nodes=self.known_dht_nodes, peer_port=self.peer_port,
|
||||
use_upnp=self.use_upnp, wallet=wallet)
|
||||
use_upnp=self.use_upnp, wallet=results['wallet'])
|
||||
|
||||
d.addCallback(create_session)
|
||||
dl = defer.DeferredList([d1, d2], fireOnOneErrback=True)
|
||||
|
||||
d.addCallback(lambda _: self.session.setup())
|
||||
dl.addCallback(combine_results)
|
||||
|
||||
dl.addCallback(create_session)
|
||||
|
||||
dl.addCallback(lambda _: self.session.setup())
|
||||
|
||||
dl.addCallback(lambda _: self.check_first_run())
|
||||
|
||||
dl.addCallback(self._show_first_run_result)
|
||||
|
||||
return dl
|
||||
|
||||
def check_first_run(self):
|
||||
d = self.session.wallet.is_first_run()
|
||||
d.addCallback(lambda is_first_run: self._do_first_run() if is_first_run else 0.0)
|
||||
return d
|
||||
|
||||
def _do_first_run(self):
|
||||
d = self.session.wallet.get_new_address()
|
||||
|
||||
def send_request(url, data):
|
||||
r = requests.post(url, json=data)
|
||||
if r.status_code == 200:
|
||||
return r.json()['credits_sent']
|
||||
return 0.0
|
||||
|
||||
def log_error(err):
|
||||
log.warning("unable to request free credits. %s", err.getErrorMessage())
|
||||
return 0.0
|
||||
|
||||
def request_credits(address):
|
||||
url = "http://credreq.lbry.io/requestcredits"
|
||||
data = {"address": address}
|
||||
d = threads.deferToThread(send_request, url, data)
|
||||
d.addErrback(log_error)
|
||||
return d
|
||||
|
||||
d.addCallback(request_credits)
|
||||
return d
|
||||
|
||||
@staticmethod
|
||||
def _show_first_run_result(credits_received):
|
||||
if credits_received != 0.0:
|
||||
points_string = locale.format_string("%.2f LBC", (round(credits_received, 2),),
|
||||
grouping=True)
|
||||
alert.info("\n\nThank you for testing the alpha version of LBRY!\n\n"
|
||||
"You have been given %s for free because we love you.\n"
|
||||
"Please give them a few minutes to show up while you\n"
|
||||
"catch up with our blockchain.\n", points_string)
|
||||
|
||||
def _setup_lbry_file_manager(self):
|
||||
self.lbry_file_metadata_manager = DBLBRYFileMetadataManager(self.conf_dir)
|
||||
self.lbry_file_metadata_manager = DBLBRYFileMetadataManager(self.db_dir)
|
||||
d = self.lbry_file_metadata_manager.setup()
|
||||
|
||||
def set_lbry_file_manager():
|
||||
@@ -179,59 +318,46 @@ class LBRYConsole():
|
||||
|
||||
def _setup_control_handlers(self):
|
||||
handlers = [
|
||||
('General',
|
||||
ApplicationStatusFactory(self.session.rate_limiter, self.session.dht_node)),
|
||||
('General',
|
||||
GetWalletBalancesFactory(self.session.wallet)),
|
||||
('General',
|
||||
ModifyApplicationDefaultsFactory(self)),
|
||||
('General',
|
||||
ShutDownFactory(self)),
|
||||
('General',
|
||||
PeerStatsAndSettingsChooserFactory(self.session.peer_manager)),
|
||||
('lbryfile',
|
||||
LBRYFileStatusFactory(self.lbry_file_manager)),
|
||||
('Stream Downloading',
|
||||
AddStreamFromSDFactory(self.sd_identifier, self.session.base_payment_rate_manager)),
|
||||
('lbryfile',
|
||||
DeleteLBRYFileChooserFactory(self.lbry_file_metadata_manager, self.session.blob_manager,
|
||||
self.lbry_file_manager)),
|
||||
('lbryfile',
|
||||
ToggleLBRYFileRunningChooserFactory(self.lbry_file_manager)),
|
||||
('lbryfile',
|
||||
CreateLBRYFileFactory(self.session, self.lbry_file_manager)),
|
||||
('lbryfile',
|
||||
PublishStreamDescriptorChooserFactory(self.lbry_file_metadata_manager,
|
||||
self.session.blob_manager,
|
||||
self.lbry_file_manager)),
|
||||
('lbryfile',
|
||||
ShowPublishedSDHashesChooserFactory(self.lbry_file_metadata_manager,
|
||||
self.lbry_file_manager)),
|
||||
('lbryfile',
|
||||
CreatePlainStreamDescriptorChooserFactory(self.lbry_file_manager)),
|
||||
('lbryfile',
|
||||
ShowLBRYFileStreamHashChooserFactory(self.lbry_file_manager)),
|
||||
('lbryfile',
|
||||
ModifyLBRYFileOptionsChooserFactory(self.lbry_file_manager)),
|
||||
('Stream Downloading',
|
||||
AddStreamFromHashFactory(self.sd_identifier, self.session))
|
||||
ApplicationStatusFactory(self.session.rate_limiter, self.session.dht_node),
|
||||
GetWalletBalancesFactory(self.session.wallet),
|
||||
ModifyApplicationDefaultsFactory(self),
|
||||
ShutDownFactory(self),
|
||||
PeerStatsAndSettingsChooserFactory(self.session.peer_manager),
|
||||
LBRYFileStatusFactory(self.lbry_file_manager),
|
||||
AddStreamFromSDFactory(self.sd_identifier, self.session.base_payment_rate_manager,
|
||||
self.session.wallet),
|
||||
DeleteLBRYFileChooserFactory(self.lbry_file_metadata_manager, self.session.blob_manager,
|
||||
self.lbry_file_manager),
|
||||
ToggleLBRYFileRunningChooserFactory(self.lbry_file_manager),
|
||||
CreateLBRYFileFactory(self.session, self.lbry_file_manager),
|
||||
PublishStreamDescriptorChooserFactory(self.lbry_file_metadata_manager,
|
||||
self.session.blob_manager),
|
||||
ShowPublishedSDHashesChooserFactory(self.lbry_file_metadata_manager,
|
||||
self.lbry_file_manager),
|
||||
CreatePlainStreamDescriptorChooserFactory(self.lbry_file_manager),
|
||||
ShowLBRYFileStreamHashChooserFactory(self.lbry_file_manager),
|
||||
ModifyLBRYFileOptionsChooserFactory(self.lbry_file_manager),
|
||||
AddStreamFromHashFactory(self.sd_identifier, self.session, self.session.wallet),
|
||||
StatusFactory(self, self.session.rate_limiter, self.lbry_file_manager,
|
||||
self.session.blob_manager, self.session.wallet if not self.fake_wallet else None),
|
||||
ImmediateAnnounceAllBlobsFactory(self.session.blob_manager)
|
||||
]
|
||||
self.add_control_handlers(handlers)
|
||||
if self.wallet_type == 'lbrycrd':
|
||||
if not self.fake_wallet:
|
||||
lbrycrd_handlers = [
|
||||
('Stream Downloading',
|
||||
AddStreamFromLBRYcrdNameFactory(self.sd_identifier, self.session,
|
||||
self.session.wallet)),
|
||||
('General',
|
||||
ClaimNameFactory(self.session.wallet)),
|
||||
AddStreamFromLBRYcrdNameFactory(self.sd_identifier, self.session,
|
||||
self.session.wallet),
|
||||
ClaimNameFactory(self.session.wallet, self.lbry_file_manager,
|
||||
self.session.blob_manager),
|
||||
GetNewWalletAddressFactory(self.session.wallet),
|
||||
PublishFactory(self.session, self.lbry_file_manager, self.session.wallet),
|
||||
BlockchainStatusFactory(self.session.wallet)
|
||||
]
|
||||
self.add_control_handlers(lbrycrd_handlers)
|
||||
if self.peer_port is not None:
|
||||
server_handlers = [
|
||||
('Server',
|
||||
ShowServerStatusFactory(self)),
|
||||
('Server',
|
||||
ModifyServerSettingsFactory(self)),
|
||||
ShowServerStatusFactory(self),
|
||||
ModifyServerSettingsFactory(self),
|
||||
]
|
||||
self.add_control_handlers(server_handlers)
|
||||
|
||||
@@ -265,12 +391,19 @@ class LBRYConsole():
|
||||
def setup_plugins():
|
||||
ds = []
|
||||
for plugin in self.plugin_manager.getAllPlugins():
|
||||
self.plugin_objects.append(plugin.plugin_object)
|
||||
ds.append(plugin.plugin_object.setup(self))
|
||||
return defer.DeferredList(ds)
|
||||
|
||||
d.addCallback(lambda _: setup_plugins())
|
||||
return d
|
||||
|
||||
def _stop_plugins(self):
|
||||
ds = []
|
||||
for plugin_object in self.plugin_objects:
|
||||
ds.append(defer.maybeDeferred(plugin_object.stop))
|
||||
return defer.DeferredList(ds)
|
||||
|
||||
def _setup_server(self):
|
||||
|
||||
def restore_running_status(running):
|
||||
@@ -290,7 +423,12 @@ class LBRYConsole():
|
||||
self.query_handlers,
|
||||
self.session.peer_manager)
|
||||
from twisted.internet import reactor
|
||||
self.lbry_server_port = reactor.listenTCP(self.peer_port, server_factory)
|
||||
try:
|
||||
self.lbry_server_port = reactor.listenTCP(self.peer_port, server_factory)
|
||||
except error.CannotListenError as e:
|
||||
import traceback
|
||||
log.error("Couldn't bind to port %d. %s", self.peer_port, traceback.format_exc())
|
||||
raise ValueError("%s lbrynet may already be running on your computer.", str(e))
|
||||
return defer.succeed(True)
|
||||
|
||||
def stop_server(self):
|
||||
@@ -300,26 +438,35 @@ class LBRYConsole():
|
||||
else:
|
||||
return defer.succeed(True)
|
||||
|
||||
def _start_controller(self):
|
||||
self.control_class(self.control_handlers)
|
||||
def _setup_controller(self):
|
||||
self.controller = ConsoleControl()
|
||||
stdio.StandardIO(self.controller)
|
||||
logger = logging.getLogger()
|
||||
formatter = logging.Formatter("%(message)s")
|
||||
alert_handler = logging.StreamHandler(self.controller)
|
||||
alert_handler.setFormatter(formatter)
|
||||
alert_handler.addFilter(logging.Filter("lbryalert"))
|
||||
alert_handler.setLevel(logging.DEBUG)
|
||||
logger.addHandler(alert_handler)
|
||||
return defer.succeed(True)
|
||||
|
||||
def _start_controller(self):
|
||||
return self.controller.start(self.command_handlers)
|
||||
|
||||
def _shut_down(self):
|
||||
self.plugin_manager = None
|
||||
d1 = self.lbry_file_metadata_manager.stop()
|
||||
d1.addCallback(lambda _: self.lbry_file_manager.stop())
|
||||
d2 = self.stop_server()
|
||||
dl = defer.DeferredList([d1, d2])
|
||||
ds = []
|
||||
if self.lbry_file_metadata_manager is not None:
|
||||
d = self.lbry_file_metadata_manager.stop()
|
||||
d.addCallback(lambda _: self.lbry_file_manager.stop())
|
||||
ds.append(d)
|
||||
ds.append(self.stop_server())
|
||||
ds.append(self._stop_plugins())
|
||||
dl = defer.DeferredList(ds)
|
||||
return dl
|
||||
|
||||
|
||||
class StdIOControl():
|
||||
def __init__(self, control_handlers):
|
||||
stdio.StandardIO(ConsoleControl(control_handlers))
|
||||
|
||||
|
||||
def launch_lbry_console():
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
parser = argparse.ArgumentParser(description="Launch a lbrynet console")
|
||||
@@ -335,12 +482,9 @@ def launch_lbry_console():
|
||||
parser.add_argument("--dht_node_port",
|
||||
help="The port on which the console will listen for DHT connections.",
|
||||
type=int, default=4444)
|
||||
parser.add_argument("--wallet_type",
|
||||
help="Either 'lbrycrd' or 'ptc'.",
|
||||
type=str, default="lbrycrd")
|
||||
parser.add_argument("--lbrycrd_wallet_rpc_port",
|
||||
help="The rpc port on which the LBRYcrd wallet is listening",
|
||||
type=int, default=8332)
|
||||
parser.add_argument("--fake_wallet",
|
||||
help="Testing purposes only. Use a non-blockchain wallet.",
|
||||
action="store_true")
|
||||
parser.add_argument("--no_dht_bootstrap",
|
||||
help="Don't try to connect to the DHT",
|
||||
action="store_true")
|
||||
@@ -352,16 +496,22 @@ def launch_lbry_console():
|
||||
help="The port of a known DHT node, to be used to bootstrap into the DHT. Must "
|
||||
"be used with --dht_bootstrap_host",
|
||||
type=int, default=4000)
|
||||
parser.add_argument("--use_upnp",
|
||||
help="Try to use UPnP to enable incoming connections through the firewall",
|
||||
parser.add_argument("--disable_upnp",
|
||||
help="Don't try to use UPnP to enable incoming connections through the firewall",
|
||||
action="store_true")
|
||||
parser.add_argument("--conf_dir",
|
||||
help=("The full path to the directory in which to store configuration "
|
||||
"options and user added plugins. Default: ~/.lbrynet"),
|
||||
type=str)
|
||||
parser.add_argument("--data_dir",
|
||||
help=("The full path to the directory in which to store data chunks "
|
||||
"downloaded from lbrynet. Default: <conf_dir>/blobfiles"),
|
||||
help=("The full path to the directory in which lbrynet data and metadata will be stored. "
|
||||
"Default: ~/.lbrynet on linux, ~/Library/Application Support/lbrynet on OS X"),
|
||||
type=str)
|
||||
parser.add_argument("--lbrycrdd_path",
|
||||
help="The path to lbrycrdd, which will be launched if it isn't running. If"
|
||||
"this option is chosen, lbrycrdd will be used as the interface to the"
|
||||
"blockchain. By default, a lightweight interface is used.")
|
||||
parser.add_argument("--lbrycrd_wallet_dir",
|
||||
help="The directory in which lbrycrd data will stored. Used if lbrycrdd is "
|
||||
"launched by this application.")
|
||||
parser.add_argument("--lbrycrd_wallet_conf",
|
||||
help="The configuration file for the LBRYcrd wallet. Default: ~/.lbrycrd/lbrycrd.conf",
|
||||
type=str)
|
||||
|
||||
args = parser.parse_args()
|
||||
@@ -381,27 +531,49 @@ def launch_lbry_console():
|
||||
else:
|
||||
dht_node_port = args.dht_node_port
|
||||
|
||||
if not args.conf_dir:
|
||||
conf_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
|
||||
else:
|
||||
conf_dir = args.conf_dir
|
||||
if not os.path.exists(conf_dir):
|
||||
os.mkdir(conf_dir)
|
||||
created_data_dir = False
|
||||
if not args.data_dir:
|
||||
data_dir = os.path.join(conf_dir, "blobfiles")
|
||||
if sys.platform == "darwin":
|
||||
data_dir = user_data_dir("LBRY")
|
||||
else:
|
||||
data_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
|
||||
else:
|
||||
data_dir = args.data_dir
|
||||
if not os.path.exists(data_dir):
|
||||
os.mkdir(data_dir)
|
||||
created_data_dir = True
|
||||
|
||||
log_format = "(%(asctime)s)[%(filename)s:%(lineno)s] %(funcName)s(): %(message)s"
|
||||
logging.basicConfig(level=logging.DEBUG, filename=os.path.join(conf_dir, "console.log"),
|
||||
format=log_format)
|
||||
daemon = JSONRPCProxy.from_url(API_CONNECTION_STRING)
|
||||
try:
|
||||
daemon.is_running()
|
||||
log.info("Attempt to start lbrynet-console while lbrynet-daemon is running")
|
||||
print "lbrynet-daemon is running, you must turn it off before using lbrynet-console"
|
||||
print "If you're running the app, quit before starting lbrynet-console"
|
||||
print "If you're running lbrynet-daemon in a terminal, run 'stop-lbrynet-daemon' to turn it off"
|
||||
|
||||
console = LBRYConsole(peer_port, dht_node_port, bootstrap_nodes, StdIOControl, wallet_type=args.wallet_type,
|
||||
lbrycrd_rpc_port=args.lbrycrd_wallet_rpc_port, use_upnp=args.use_upnp,
|
||||
conf_dir=conf_dir, data_dir=data_dir)
|
||||
webbrowser.open("http://localhost:5279")
|
||||
|
||||
d = task.deferLater(reactor, 0, console.start)
|
||||
reactor.addSystemEventTrigger('before', 'shutdown', console.shut_down)
|
||||
reactor.run()
|
||||
except:
|
||||
log_format = "(%(asctime)s)[%(filename)s:%(lineno)s] %(funcName)s(): %(message)s"
|
||||
formatter = logging.Formatter(log_format)
|
||||
|
||||
logger = logging.getLogger()
|
||||
logger.setLevel(logging.DEBUG)
|
||||
file_handler = logging.FileHandler(os.path.join(data_dir, "console.log"))
|
||||
file_handler.setFormatter(formatter)
|
||||
file_handler.addFilter(logging.Filter("lbrynet"))
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
|
||||
console = LBRYConsole(peer_port, dht_node_port, bootstrap_nodes, fake_wallet=args.fake_wallet,
|
||||
lbrycrd_conf=args.lbrycrd_wallet_conf, lbrycrd_dir=args.lbrycrd_wallet_dir,
|
||||
use_upnp=not args.disable_upnp, data_dir=data_dir,
|
||||
created_data_dir=created_data_dir, lbrycrdd_path=args.lbrycrdd_path)
|
||||
|
||||
d = task.deferLater(reactor, 0, console.start)
|
||||
d.addErrback(lambda _: reactor.stop())
|
||||
reactor.addSystemEventTrigger('before', 'shutdown', console.shut_down)
|
||||
reactor.run()
|
||||
|
||||
if __name__ == "__main__":
|
||||
launch_lbry_console()
|
||||
@@ -7,4 +7,7 @@ class LBRYPlugin(IPlugin):
|
||||
IPlugin.__init__(self)
|
||||
|
||||
def setup(self, lbry_console):
|
||||
raise NotImplementedError
|
||||
|
||||
def stop(self):
|
||||
raise NotImplementedError
|
||||
@@ -1,60 +1,66 @@
|
||||
import binascii
|
||||
import functools
|
||||
import json
|
||||
import leveldb
|
||||
import logging
|
||||
import os
|
||||
|
||||
from twisted.internet import threads, defer
|
||||
import unqlite
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run_in_thread(fn):
|
||||
@functools.wraps(fn)
|
||||
def wrapped(*args, **kwargs):
|
||||
return threads.deferToThread(fn, *args, **kwargs)
|
||||
return wrapped
|
||||
|
||||
|
||||
class LBRYSettings(object):
|
||||
NAME = "settings.db"
|
||||
def __init__(self, db_dir):
|
||||
self.db_dir = db_dir
|
||||
self.db = None
|
||||
|
||||
def start(self):
|
||||
return threads.deferToThread(self._open_db)
|
||||
return self._open_db()
|
||||
|
||||
def stop(self):
|
||||
self.db.close()
|
||||
self.db = None
|
||||
return defer.succeed(True)
|
||||
|
||||
def _open_db(self):
|
||||
logging.debug("Opening %s as the settings database", str(os.path.join(self.db_dir, "settings.db")))
|
||||
self.db = leveldb.LevelDB(os.path.join(self.db_dir, "settings.db"))
|
||||
filename = os.path.join(self.db_dir, self.NAME)
|
||||
log.debug("Opening %s as the settings database", filename)
|
||||
self.db = unqlite.UnQLite(filename)
|
||||
return defer.succeed(True)
|
||||
|
||||
@run_in_thread
|
||||
def save_lbryid(self, lbryid):
|
||||
self.db['lbryid'] = binascii.hexlify(lbryid)
|
||||
self.db.commit()
|
||||
|
||||
def save_lbryid():
|
||||
self.db.Put("lbryid", binascii.hexlify(lbryid), sync=True)
|
||||
|
||||
return threads.deferToThread(save_lbryid)
|
||||
|
||||
@run_in_thread
|
||||
def get_lbryid(self):
|
||||
if 'lbryid' in self.db:
|
||||
return binascii.unhexlify(self.db['lbryid'])
|
||||
else:
|
||||
return None
|
||||
|
||||
def get_lbryid():
|
||||
try:
|
||||
return binascii.unhexlify(self.db.Get("lbryid"))
|
||||
except KeyError:
|
||||
return None
|
||||
|
||||
return threads.deferToThread(get_lbryid)
|
||||
|
||||
@run_in_thread
|
||||
def get_server_running_status(self):
|
||||
if 'server_running' in self.db:
|
||||
return json.loads(self.db['server_running'])
|
||||
else:
|
||||
return True
|
||||
|
||||
def get_status():
|
||||
try:
|
||||
return json.loads(self.db.Get("server_running"))
|
||||
except KeyError:
|
||||
return True
|
||||
|
||||
return threads.deferToThread(get_status)
|
||||
|
||||
@run_in_thread
|
||||
def save_server_running_status(self, running):
|
||||
|
||||
def save_status():
|
||||
self.db.Put("server_running", json.dumps(running), sync=True)
|
||||
|
||||
return threads.deferToThread(save_status)
|
||||
self.db['server_running'] = json.dumps(running)
|
||||
self.db.commit()
|
||||
|
||||
def get_default_data_payment_rate(self):
|
||||
return self._get_payment_rate("default_data_payment_rate")
|
||||
@@ -74,35 +80,27 @@ class LBRYSettings(object):
|
||||
def save_server_crypt_info_payment_rate(self, rate):
|
||||
return self._save_payment_rate("server_crypt_info_payment_rate", rate)
|
||||
|
||||
@run_in_thread
|
||||
def _get_payment_rate(self, rate_type):
|
||||
if rate_type in self.db:
|
||||
return json.loads(self.db[rate_type])
|
||||
else:
|
||||
return None
|
||||
|
||||
def get_rate():
|
||||
try:
|
||||
return json.loads(self.db.Get(rate_type))
|
||||
except KeyError:
|
||||
return None
|
||||
|
||||
return threads.deferToThread(get_rate)
|
||||
|
||||
@run_in_thread
|
||||
def _save_payment_rate(self, rate_type, rate):
|
||||
if rate is not None:
|
||||
self.db[rate_type] = json.dumps(rate)
|
||||
elif rate_type in self.db:
|
||||
del self.db[rate_type]
|
||||
self.db.commit()
|
||||
|
||||
def save_rate():
|
||||
if rate is not None:
|
||||
self.db.Put(rate_type, json.dumps(rate), sync=True)
|
||||
else:
|
||||
self.db.Delete(rate_type, sync=True)
|
||||
|
||||
return threads.deferToThread(save_rate)
|
||||
|
||||
@run_in_thread
|
||||
def get_query_handler_status(self, query_identifier):
|
||||
|
||||
def get_status():
|
||||
try:
|
||||
return json.loads(self.db.Get(json.dumps(('q_h', query_identifier))))
|
||||
except KeyError:
|
||||
return True
|
||||
|
||||
return threads.deferToThread(get_status)
|
||||
if json.dumps(('q_h', query_identifier)) in self.db:
|
||||
return json.loads(self.db[(json.dumps(('q_h', query_identifier)))])
|
||||
else:
|
||||
return True
|
||||
|
||||
def enable_query_handler(self, query_identifier):
|
||||
return self._set_query_handler_status(query_identifier, True)
|
||||
@@ -110,7 +108,7 @@ class LBRYSettings(object):
|
||||
def disable_query_handler(self, query_identifier):
|
||||
return self._set_query_handler_status(query_identifier, False)
|
||||
|
||||
@run_in_thread
|
||||
def _set_query_handler_status(self, query_identifier, status):
|
||||
def set_status():
|
||||
self.db.Put(json.dumps(('q_h', query_identifier)), json.dumps(status), sync=True)
|
||||
return threads.deferToThread(set_status)
|
||||
self.db[json.dumps(('q_h', query_identifier))] = json.dumps(status)
|
||||
self.db.commit()
|
||||
@@ -10,5 +10,15 @@ class IControlHandlerFactory(Interface):
|
||||
|
||||
|
||||
class IControlHandler(Interface):
|
||||
def handle_line(self, line):
|
||||
pass
|
||||
|
||||
|
||||
class ICommandHandlerFactory(Interface):
|
||||
def get_handler(self):
|
||||
pass
|
||||
|
||||
|
||||
class ICommandHandler(Interface):
|
||||
def handle_line(self, line):
|
||||
pass
|
||||
@@ -1,25 +1,43 @@
|
||||
from twisted.internet import threads, defer
|
||||
from twisted.internet import defer
|
||||
from ValuableBlobInfo import ValuableBlobInfo
|
||||
from db_keys import BLOB_INFO_TYPE
|
||||
import json
|
||||
import leveldb
|
||||
import os
|
||||
import sqlite3
|
||||
from twisted.enterprise import adbapi
|
||||
from lbrynet.core.sqlite_helpers import rerun_if_locked
|
||||
|
||||
|
||||
class BlindInfoManager(object):
|
||||
|
||||
def __init__(self, db, peer_manager):
|
||||
self.db = db
|
||||
def __init__(self, db_dir, peer_manager):
|
||||
self.db_dir = db_dir
|
||||
self.db_conn = None
|
||||
self.peer_manager = peer_manager
|
||||
|
||||
def setup(self):
|
||||
return defer.succeed(True)
|
||||
# check_same_thread=False is solely to quiet a spurious error that appears to be due
|
||||
# to a bug in twisted, where the connection is closed by a different thread than the
|
||||
# one that opened it. The individual connections in the pool are not used in multiple
|
||||
# threads.
|
||||
self.db_conn = adbapi.ConnectionPool('sqlite3', os.path.join(self.db_dir, "blind_info.db"),
|
||||
check_same_thread=False)
|
||||
|
||||
def set_up_table(transaction):
|
||||
transaction.execute("create table if not exists valuable_blobs (" +
|
||||
" blob_hash text primary key, " +
|
||||
" blob_length integer, " +
|
||||
" reference text, " +
|
||||
" peer_host text, " +
|
||||
" peer_port integer, " +
|
||||
" peer_score text" +
|
||||
")")
|
||||
return self.db_conn.runInteraction(set_up_table)
|
||||
|
||||
def stop(self):
|
||||
self.db = None
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_all_blob_infos(self):
|
||||
d = threads.deferToThread(self._get_all_blob_infos)
|
||||
d = self._get_all_blob_infos()
|
||||
|
||||
def make_blob_infos(blob_data):
|
||||
blob_infos = []
|
||||
@@ -42,21 +60,19 @@ class BlindInfoManager(object):
|
||||
peer_port = blob_info.peer.port
|
||||
peer_score = blob_info.peer_score
|
||||
blobs.append((blob_hash, length, reference, peer_host, peer_port, peer_score))
|
||||
return threads.deferToThread(self._save_blob_infos, blobs)
|
||||
return self._save_blob_infos(blobs)
|
||||
|
||||
@rerun_if_locked
|
||||
def _get_all_blob_infos(self):
|
||||
blob_infos = []
|
||||
for key, blob_info in self.db.RangeIter():
|
||||
key_type, blob_hash = json.loads(key)
|
||||
if key_type == BLOB_INFO_TYPE:
|
||||
blob_infos.append([blob_hash] + json.loads(blob_info))
|
||||
return blob_infos
|
||||
return self.db_conn.runQuery("select * from valuable_blobs")
|
||||
|
||||
@rerun_if_locked
|
||||
def _save_blob_infos(self, blobs):
|
||||
batch = leveldb.WriteBatch()
|
||||
for blob in blobs:
|
||||
try:
|
||||
self.db.Get(json.dumps((BLOB_INFO_TYPE, blob[0])))
|
||||
except KeyError:
|
||||
batch.Put(json.dumps((BLOB_INFO_TYPE, blob[0])), json.dumps(blob[1:]))
|
||||
self.db.Write(batch, sync=True)
|
||||
def save_infos(transaction):
|
||||
for blob in blobs:
|
||||
try:
|
||||
transaction.execute("insert into valuable_blobs values (?, ?, ?, ?, ?, ?)",
|
||||
blob)
|
||||
except sqlite3.IntegrityError:
|
||||
pass
|
||||
return self.db_conn.runInteraction(save_infos)
|
||||
@@ -2,7 +2,7 @@ from zope.interface import implements
|
||||
from lbrynet.interfaces import IMetadataHandler, IRequestCreator
|
||||
from lbrynet.core.client.ClientRequest import ClientRequest, ClientPaidRequest
|
||||
from lbrynet.core.Error import InsufficientFundsError, InvalidResponseError, RequestCanceledError
|
||||
from lbrynet.core.Error import NoResponseError
|
||||
from lbrynet.core.Error import NoResponseError, ConnectionClosedBeforeResponseError
|
||||
from ValuableBlobInfo import ValuableBlobInfo
|
||||
import datetime
|
||||
import logging
|
||||
@@ -12,6 +12,9 @@ from twisted.python.failure import Failure
|
||||
from collections import defaultdict
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlindMetadataHandler(object):
|
||||
implements(IMetadataHandler, IRequestCreator)
|
||||
|
||||
@@ -195,7 +198,7 @@ class BlindMetadataHandler(object):
|
||||
if not 'valuable_blob_hashes' in response:
|
||||
return InvalidResponseError("Missing the required field 'valuable_blob_hashes'")
|
||||
hashes = response['valuable_blob_hashes']
|
||||
logging.info("Handling %s valuable blob hashes from %s", str(len(hashes)), str(peer))
|
||||
log.info("Handling %s valuable blob hashes from %s", str(len(hashes)), str(peer))
|
||||
expire_time = datetime.datetime.now() + datetime.timedelta(minutes=10)
|
||||
reference = None
|
||||
unique_hashes = set()
|
||||
@@ -230,8 +233,8 @@ class BlindMetadataHandler(object):
|
||||
if not 'blob_lengths' in response:
|
||||
return InvalidResponseError("Missing the required field 'blob_lengths'")
|
||||
raw_blob_lengths = response['blob_lengths']
|
||||
logging.info("Handling %s blob lengths from %s", str(len(raw_blob_lengths)), str(peer))
|
||||
logging.debug("blobs: %s", str(raw_blob_lengths))
|
||||
log.info("Handling %s blob lengths from %s", str(len(raw_blob_lengths)), str(peer))
|
||||
log.debug("blobs: %s", str(raw_blob_lengths))
|
||||
infos = []
|
||||
unique_hashes = set()
|
||||
for blob_hash, length in raw_blob_lengths:
|
||||
@@ -287,11 +290,12 @@ class BlindMetadataHandler(object):
|
||||
return
|
||||
if reason.check(NoResponseError):
|
||||
self._incompatible_peers.append(peer)
|
||||
return
|
||||
logging.warning("Valuable blob info requester: a request of type %s has failed. Reason: %s",
|
||||
str(request_type), str(reason.getErrorMessage()))
|
||||
log.warning("Valuable blob info requester: a request of type %s has failed. Reason: %s",
|
||||
str(request_type), str(reason.getErrorMessage()))
|
||||
self._update_local_score(peer, -10.0)
|
||||
peer.update_score(-5.0)
|
||||
if reason.check(ConnectionClosedBeforeResponseError):
|
||||
return
|
||||
return reason
|
||||
|
||||
def _search_for_peers(self):
|
||||
|
||||
@@ -109,5 +109,5 @@ class BlindRepeater(object):
|
||||
def _update_status(self, stopped=True):
|
||||
self.stopped = stopped
|
||||
|
||||
def insufficient_funds(self):
|
||||
def insufficient_funds(self, err):
|
||||
return self.stop()
|
||||
@@ -1,111 +1,121 @@
|
||||
from lbrynet.lbrynet_console.ControlHandlers import ControlHandler, ControlHandlerFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import RecursiveControlHandler, ModifyPaymentRate
|
||||
from lbrynet.lbrynet_console.ControlHandlers import CommandHandler, CommandHandlerFactory
|
||||
from lbrynet.lbrynet_console.ControlHandlers import RecursiveCommandHandler, ModifyPaymentRate
|
||||
from twisted.internet import defer
|
||||
|
||||
|
||||
class StartRepeater(ControlHandler):
|
||||
class StartRepeater(CommandHandler):
|
||||
prompt_description = "Start the blind repeater"
|
||||
|
||||
def __init__(self, repeater, settings):
|
||||
def __init__(self, console, repeater, settings):
|
||||
CommandHandler.__init__(self, console)
|
||||
self.repeater = repeater
|
||||
self.settings = settings
|
||||
|
||||
def handle_line(self, line):
|
||||
assert line is None, "Start repeater should not be passed any arguments"
|
||||
def start(self):
|
||||
#assert line is None, "Start repeater should not be passed any arguments"
|
||||
d = self.settings.save_repeater_status(running=True)
|
||||
d.addCallback(lambda _: self.repeater.start())
|
||||
d.addCallback(lambda _: "Started the repeater")
|
||||
return True, d
|
||||
d.addCallback(lambda _: self.console.sendLine("Started the repeater"))
|
||||
d.chainDeferred(self.finished_deferred)
|
||||
|
||||
|
||||
class StartRepeaterFactory(ControlHandlerFactory):
|
||||
class StartRepeaterFactory(CommandHandlerFactory):
|
||||
control_handler_class = StartRepeater
|
||||
|
||||
|
||||
class StopRepeater(ControlHandler):
|
||||
class StopRepeater(CommandHandler):
|
||||
prompt_description = "Stop the blind repeater"
|
||||
|
||||
def __init__(self, repeater, settings):
|
||||
def __init__(self, console, repeater, settings):
|
||||
CommandHandler.__init__(self, console)
|
||||
self.repeater = repeater
|
||||
self.settings = settings
|
||||
|
||||
def handle_line(self, line):
|
||||
assert line is None, "Stop repeater should not be passed any arguments"
|
||||
def start(self):
|
||||
#assert line is None, "Stop repeater should not be passed any arguments"
|
||||
d = self.settings.save_repeater_status(running=False)
|
||||
d.addCallback(lambda _: self.repeater.stop())
|
||||
d.addCallback(lambda _: "Stopped the repeater")
|
||||
return True, d
|
||||
d.addCallback(lambda _: self.console.sendLine("Stopped the repeater"))
|
||||
d.chainDeferred(self.finished_deferred)
|
||||
|
||||
|
||||
class StopRepeaterFactory(ControlHandlerFactory):
|
||||
class StopRepeaterFactory(CommandHandlerFactory):
|
||||
control_handler_class = StopRepeater
|
||||
|
||||
|
||||
class UpdateMaxSpace(ControlHandler):
|
||||
class UpdateMaxSpace(CommandHandler):
|
||||
prompt_description = "Set the maximum space to be used by the blind repeater"
|
||||
line_prompt = "Maximum space (in bytes):"
|
||||
|
||||
def __init__(self, repeater, settings):
|
||||
def __init__(self, console, repeater, settings):
|
||||
CommandHandler.__init__(self, console)
|
||||
self.repeater = repeater
|
||||
self.settings = settings
|
||||
|
||||
def start(self):
|
||||
self.console.sendLine(self.line_prompt)
|
||||
|
||||
def handle_line(self, line):
|
||||
if line is None:
|
||||
return False, defer.succeed(self.line_prompt)
|
||||
return True, self._set_max_space(line)
|
||||
d = self._set_max_space(line)
|
||||
d.chainDeferred(self.finished_deferred)
|
||||
|
||||
def _set_max_space(self, line):
|
||||
max_space = int(line)
|
||||
d = self.settings.save_max_space(max_space)
|
||||
d.addCallback(lambda _: self.repeater.set_max_space(max_space))
|
||||
d.addCallback(lambda _: "Set the maximum space to " + str(max_space) + " bytes")
|
||||
d.addCallback(lambda _: self.console.sendLine("Set the maximum space to " + str(max_space) + " bytes"))
|
||||
return d
|
||||
|
||||
|
||||
class UpdateMaxSpaceFactory(ControlHandlerFactory):
|
||||
class UpdateMaxSpaceFactory(CommandHandlerFactory):
|
||||
control_handler_class = UpdateMaxSpace
|
||||
|
||||
|
||||
class AddApprovedPeer(ControlHandler):
|
||||
class AddApprovedPeer(CommandHandler):
|
||||
prompt_description = "Add a peer to the approved list of peers to check for valuable blob hashes"
|
||||
host_prompt = "Peer host in dotted quad (e.g. 127.0.0.1)"
|
||||
port_prompt = "Peer port (e.g. 4444)"
|
||||
|
||||
def __init__(self, repeater, peer_manager, settings):
|
||||
def __init__(self, console, repeater, peer_manager, settings):
|
||||
CommandHandler.__init__(self, console)
|
||||
self.repeater = repeater
|
||||
self.peer_manager = peer_manager
|
||||
self.settings = settings
|
||||
self.host_to_add = None
|
||||
|
||||
def start(self):
|
||||
self.console.sendLine(self.host_prompt)
|
||||
|
||||
def handle_line(self, line):
|
||||
if line is None:
|
||||
return False, defer.succeed(self.host_prompt)
|
||||
elif self.host_to_add is None:
|
||||
#if line is None:
|
||||
# return False, defer.succeed(self.host_prompt)
|
||||
if self.host_to_add is None:
|
||||
self.host_to_add = line
|
||||
return False, defer.succeed(self.port_prompt)
|
||||
self.console.sendLine(self.port_prompt)
|
||||
else:
|
||||
self.host_to_add, host = None, self.host_to_add
|
||||
return True, self._add_peer(host, line)
|
||||
d = self._add_peer(host, line)
|
||||
d.chainDeferred(self.finished_deferred)
|
||||
|
||||
def _add_peer(self, host, port):
|
||||
peer = self.peer_manager.get_peer(host, int(port))
|
||||
d = self.settings.save_approved_peer(host, int(port))
|
||||
d.addCallback(lambda _: self.repeater.add_approved_peer(peer))
|
||||
d.addCallback(lambda _: "Successfully added peer")
|
||||
d.addCallback(lambda _: self.console.sendLine("Successfully added peer"))
|
||||
return d
|
||||
|
||||
|
||||
class AddApprovedPeerFactory(ControlHandlerFactory):
|
||||
class AddApprovedPeerFactory(CommandHandlerFactory):
|
||||
control_handler_class = AddApprovedPeer
|
||||
|
||||
|
||||
class ApprovedPeerChooser(RecursiveControlHandler):
|
||||
class ApprovedPeerChooser(RecursiveCommandHandler):
|
||||
|
||||
def __init__(self, repeater, factory_class, *args, **kwargs):
|
||||
def __init__(self, console, repeater, factory_class, *args, **kwargs):
|
||||
self.repeater = repeater
|
||||
self.factory_class = factory_class
|
||||
self.args = args
|
||||
RecursiveControlHandler.__init__(self, **kwargs)
|
||||
RecursiveCommandHandler.__init__(self, console, **kwargs)
|
||||
|
||||
def _get_control_handler_factories(self):
|
||||
control_handler_factories = []
|
||||
@@ -114,7 +124,7 @@ class ApprovedPeerChooser(RecursiveControlHandler):
|
||||
return control_handler_factories
|
||||
|
||||
|
||||
class ApprovedPeerChooserFactory(ControlHandlerFactory):
|
||||
class ApprovedPeerChooserFactory(CommandHandlerFactory):
|
||||
def get_prompt_description(self):
|
||||
peer = self.args[0]
|
||||
return str(peer)
|
||||
@@ -123,30 +133,32 @@ class ApprovedPeerChooserFactory(ControlHandlerFactory):
|
||||
class DeleteApprovedPeerChooser(ApprovedPeerChooser):
|
||||
prompt_description = "Remove a peer from the approved list of peers to check for valuable blob hashes"
|
||||
|
||||
def __init__(self, repeater, settings):
|
||||
ApprovedPeerChooser.__init__(self, repeater, DeleteApprovedPeerFactory, repeater, settings,
|
||||
exit_after_one_done=True)
|
||||
def __init__(self, console, repeater, settings):
|
||||
ApprovedPeerChooser.__init__(self, console, repeater, DeleteApprovedPeerFactory, repeater,
|
||||
settings, exit_after_one_done=True)
|
||||
|
||||
|
||||
class DeleteApprovedPeerChooserFactory(ControlHandlerFactory):
|
||||
class DeleteApprovedPeerChooserFactory(CommandHandlerFactory):
|
||||
control_handler_class = DeleteApprovedPeerChooser
|
||||
|
||||
|
||||
class DeleteApprovedPeer(ControlHandler):
|
||||
class DeleteApprovedPeer(CommandHandler):
|
||||
prompt_description = "Remove a peer from the approved list of peers to check for valuable blob hashes"
|
||||
|
||||
def __init__(self, peer, repeater, settings):
|
||||
def __init__(self, console, peer, repeater, settings):
|
||||
CommandHandler.__init__(self, console)
|
||||
self.repeater = repeater
|
||||
self.settings = settings
|
||||
self.peer_to_remove = peer
|
||||
|
||||
def handle_line(self, line):
|
||||
return True, self._remove_peer()
|
||||
def start(self):
|
||||
d = self._remove_peer()
|
||||
d.chainDeferred(self.finished_deferred)
|
||||
|
||||
def _remove_peer(self):
|
||||
d = self.settings.remove_approved_peer(self.peer_to_remove.host, int(self.peer_to_remove.port))
|
||||
d.addCallback(lambda _: self.repeater.remove_approved_peer(self.peer_to_remove))
|
||||
d.addCallback(lambda _: "Successfully removed peer")
|
||||
d.addCallback(lambda _: self.console.sendLine("Successfully removed peer"))
|
||||
return d
|
||||
|
||||
|
||||
@@ -154,38 +166,43 @@ class DeleteApprovedPeerFactory(ApprovedPeerChooserFactory):
|
||||
control_handler_class = DeleteApprovedPeer
|
||||
|
||||
|
||||
class ShowApprovedPeers(ControlHandler):
|
||||
class ShowApprovedPeers(CommandHandler):
|
||||
prompt_description = "Show the list of peers approved to be checked for valuable blob hashes"
|
||||
|
||||
def __init__(self, repeater):
|
||||
def __init__(self, console, repeater):
|
||||
CommandHandler.__init__(self, console)
|
||||
self.repeater = repeater
|
||||
|
||||
def handle_line(self, line):
|
||||
assert line is None, "Show approved peers should not be passed any arguments"
|
||||
return True, self._show_peers()
|
||||
def start(self):
|
||||
#assert line is None, "Show approved peers should not be passed any arguments"
|
||||
d = self._show_peers()
|
||||
d.chainDeferred(self.finished_deferred)
|
||||
|
||||
def _show_peers(self):
|
||||
peer_string = "Approved peers:\n"
|
||||
for peer in self.repeater.approved_peers:
|
||||
peer_string += str(peer) + "\n"
|
||||
return defer.succeed(peer_string)
|
||||
self.console.sendLine(peer_string)
|
||||
return defer.succeed(None)
|
||||
|
||||
|
||||
class ShowApprovedPeersFactory(ControlHandlerFactory):
|
||||
class ShowApprovedPeersFactory(CommandHandlerFactory):
|
||||
control_handler_class = ShowApprovedPeers
|
||||
|
||||
|
||||
class RepeaterStatus(ControlHandler):
|
||||
class RepeaterStatus(CommandHandler):
|
||||
prompt_description = "Show the repeater's status"
|
||||
|
||||
def __init__(self, repeater):
|
||||
def __init__(self, console, repeater):
|
||||
CommandHandler.__init__(self, console)
|
||||
self.repeater = repeater
|
||||
|
||||
def handle_line(self, line):
|
||||
assert line is None, "Show repeater status should not be passed any arguments"
|
||||
return True, defer.maybeDeferred(self._get_status)
|
||||
def start(self):
|
||||
#assert line is None, "Show repeater status should not be passed any arguments"
|
||||
self._show_status()
|
||||
self.finished_deferred.callback(None)
|
||||
|
||||
def _get_status(self):
|
||||
def _show_status(self):
|
||||
status_string = "Repeater status: " + self.repeater.status() + "\n"
|
||||
|
||||
if self.repeater.stopped is False:
|
||||
@@ -197,25 +214,25 @@ class RepeaterStatus(ControlHandler):
|
||||
|
||||
status_string += "Maximum space: " + str(max_space) + " bytes\n"
|
||||
status_string += "Space used: " + str(space_used) + " bytes\n"
|
||||
return defer.succeed(status_string)
|
||||
self.console.sendLine(status_string)
|
||||
|
||||
|
||||
class RepeaterStatusFactory(ControlHandlerFactory):
|
||||
class RepeaterStatusFactory(CommandHandlerFactory):
|
||||
control_handler_class = RepeaterStatus
|
||||
|
||||
|
||||
class ModifyDataPaymentRate(ModifyPaymentRate):
|
||||
prompt_description = "Modify Blind Repeater data payment rate"
|
||||
|
||||
def __init__(self, repeater, settings):
|
||||
ModifyPaymentRate.__init__(self)
|
||||
def __init__(self, console, repeater, settings):
|
||||
ModifyPaymentRate.__init__(self, console)
|
||||
self._prompt_choices['unset'] = (self._unset, "Use the application default data rate")
|
||||
self.payment_rate_manager = repeater.payment_rate_manager
|
||||
self.settings = settings
|
||||
|
||||
def _unset(self):
|
||||
self._set_rate(None)
|
||||
return True, defer.succeed("Using the application default data rate")
|
||||
return defer.succeed("Using the application default data rate")
|
||||
|
||||
def _set_rate(self, rate):
|
||||
|
||||
@@ -237,15 +254,15 @@ class ModifyDataPaymentRate(ModifyPaymentRate):
|
||||
return status
|
||||
|
||||
|
||||
class ModifyDataPaymentRateFactory(ControlHandlerFactory):
|
||||
class ModifyDataPaymentRateFactory(CommandHandlerFactory):
|
||||
control_handler_class = ModifyDataPaymentRate
|
||||
|
||||
|
||||
class ModifyInfoPaymentRate(ModifyPaymentRate):
|
||||
prompt_description = "Modify Blind Repeater valuable info payment rate"
|
||||
|
||||
def __init__(self, repeater, settings):
|
||||
ModifyPaymentRate.__init__(self)
|
||||
def __init__(self, console, repeater, settings):
|
||||
ModifyPaymentRate.__init__(self, console)
|
||||
self.payment_rate_manager = repeater.payment_rate_manager
|
||||
self.settings = settings
|
||||
|
||||
@@ -264,15 +281,15 @@ class ModifyInfoPaymentRate(ModifyPaymentRate):
|
||||
return status
|
||||
|
||||
|
||||
class ModifyInfoPaymentRateFactory(ControlHandlerFactory):
|
||||
class ModifyInfoPaymentRateFactory(CommandHandlerFactory):
|
||||
control_handler_class = ModifyInfoPaymentRate
|
||||
|
||||
|
||||
class ModifyHashPaymentRate(ModifyPaymentRate):
|
||||
prompt_description = "Modify Blind Repeater valuable hash payment rate"
|
||||
|
||||
def __init__(self, repeater, settings):
|
||||
ModifyPaymentRate.__init__(self)
|
||||
def __init__(self, console, repeater, settings):
|
||||
ModifyPaymentRate.__init__(self, console)
|
||||
self.payment_rate_manager = repeater.payment_rate_manager
|
||||
self.settings = settings
|
||||
|
||||
@@ -291,18 +308,18 @@ class ModifyHashPaymentRate(ModifyPaymentRate):
|
||||
return status
|
||||
|
||||
|
||||
class ModifyHashPaymentRateFactory(ControlHandlerFactory):
|
||||
class ModifyHashPaymentRateFactory(CommandHandlerFactory):
|
||||
control_handler_class = ModifyHashPaymentRate
|
||||
|
||||
|
||||
class ModifyRepeaterOptions(RecursiveControlHandler):
|
||||
class ModifyRepeaterOptions(RecursiveCommandHandler):
|
||||
prompt_description = "Modify Blind Repeater options"
|
||||
|
||||
def __init__(self, repeater, lbry_session, settings):
|
||||
def __init__(self, console, repeater, lbry_session, settings):
|
||||
self.repeater = repeater
|
||||
self.lbry_session = lbry_session
|
||||
self.settings = settings
|
||||
RecursiveControlHandler.__init__(self)
|
||||
RecursiveCommandHandler.__init__(self, console)
|
||||
|
||||
def _get_control_handler_factories(self):
|
||||
return [ModifyDataPaymentRateFactory(self.repeater, self.settings),
|
||||
@@ -314,5 +331,5 @@ class ModifyRepeaterOptions(RecursiveControlHandler):
|
||||
]
|
||||
|
||||
|
||||
class ModifyRepeaterOptionsFactory(ControlHandlerFactory):
|
||||
class ModifyRepeaterOptionsFactory(CommandHandlerFactory):
|
||||
control_handler_class = ModifyRepeaterOptions
|
||||
@@ -1,67 +1,79 @@
|
||||
from db_keys import SETTING_TYPE, PEER_TYPE
|
||||
from twisted.internet import threads
|
||||
from twisted.internet import threads, defer
|
||||
import json
|
||||
|
||||
import unqlite
|
||||
import os
|
||||
from twisted.enterprise import adbapi
|
||||
from lbrynet.core.sqlite_helpers import rerun_if_locked
|
||||
|
||||
class BlindRepeaterSettings(object):
|
||||
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
def __init__(self, db_dir):
|
||||
self.db_dir = db_dir
|
||||
self.unq_db = None
|
||||
self.sql_db = None
|
||||
|
||||
def setup(self):
|
||||
self.unq_db = unqlite.UnQLite(os.path.join(self.db_dir, "blind_settings.db"))
|
||||
# check_same_thread=False is solely to quiet a spurious error that appears to be due
|
||||
# to a bug in twisted, where the connection is closed by a different thread than the
|
||||
# one that opened it. The individual connections in the pool are not used in multiple
|
||||
# threads.
|
||||
self.sql_db = adbapi.ConnectionPool('sqlite3', os.path.join(self.db_dir, "blind_peers.db"),
|
||||
check_same_thread=False)
|
||||
|
||||
return self.sql_db.runQuery("create table if not exists approved_peers (" +
|
||||
" ip_address text, " +
|
||||
" port integer" +
|
||||
")")
|
||||
|
||||
def stop(self):
|
||||
self.unq_db = None
|
||||
self.sql_db = None
|
||||
return defer.succeed(True)
|
||||
|
||||
def save_repeater_status(self, running):
|
||||
def save_status():
|
||||
self.db.Put(json.dumps((SETTING_TYPE, "running")), json.dumps(running), sync=True)
|
||||
self.unq_db["running"] = json.dumps(running)
|
||||
|
||||
return threads.deferToThread(save_status)
|
||||
|
||||
def get_repeater_saved_status(self):
|
||||
def get_status():
|
||||
try:
|
||||
return json.loads(self.db.Get(json.dumps((SETTING_TYPE, "running"))))
|
||||
except KeyError:
|
||||
if "running" in self.unq_db:
|
||||
return json.loads(self.unq_db['running'])
|
||||
else:
|
||||
return False
|
||||
|
||||
return threads.deferToThread(get_status)
|
||||
|
||||
def save_max_space(self, max_space):
|
||||
def save_space():
|
||||
self.db.Put(json.dumps((SETTING_TYPE, "max_space")), str(max_space), sync=True)
|
||||
self.unq_db['max_space'] = json.dumps(max_space)
|
||||
|
||||
return threads.deferToThread(save_space)
|
||||
|
||||
def get_saved_max_space(self):
|
||||
def get_space():
|
||||
try:
|
||||
return int(self.db.Get(json.dumps((SETTING_TYPE, "max_space"))))
|
||||
except KeyError:
|
||||
if 'max_space' in self.unq_db:
|
||||
return json.loads(self.unq_db['max_space'])
|
||||
else:
|
||||
return 0
|
||||
|
||||
return threads.deferToThread(get_space)
|
||||
|
||||
@rerun_if_locked
|
||||
def save_approved_peer(self, host, port):
|
||||
def add_peer():
|
||||
peer_string = json.dumps((PEER_TYPE, (host, port)))
|
||||
self.db.Put(peer_string, "", sync=True)
|
||||
|
||||
return threads.deferToThread(add_peer)
|
||||
return self.sql_db.runQuery("insert into approved_peers values (?, ?)",
|
||||
(host, port))
|
||||
|
||||
@rerun_if_locked
|
||||
def remove_approved_peer(self, host, port):
|
||||
def remove_peer():
|
||||
peer_string = json.dumps((PEER_TYPE, (host, port)))
|
||||
self.db.Delete(peer_string, sync=True)
|
||||
|
||||
return threads.deferToThread(remove_peer)
|
||||
return self.sql_db.runQuery("delete from approved_peers where ip_address = ? and port = ?",
|
||||
(host, port))
|
||||
|
||||
@rerun_if_locked
|
||||
def get_approved_peers(self):
|
||||
def get_peers():
|
||||
peers = []
|
||||
for k, v in self.db.RangeIter():
|
||||
key_type, peer_info = json.loads(k)
|
||||
if key_type == PEER_TYPE:
|
||||
peers.append(peer_info)
|
||||
return peers
|
||||
|
||||
return threads.deferToThread(get_peers)
|
||||
return self.sql_db.runQuery("select * from approved_peers")
|
||||
|
||||
def get_data_payment_rate(self):
|
||||
return threads.deferToThread(self._get_rate, "data_payment_rate")
|
||||
@@ -82,13 +94,13 @@ class BlindRepeaterSettings(object):
|
||||
return threads.deferToThread(self._save_rate, "valuable_hash_rate", rate)
|
||||
|
||||
def _get_rate(self, rate_type):
|
||||
try:
|
||||
return json.loads(self.db.Get(json.dumps((SETTING_TYPE, rate_type))))
|
||||
except KeyError:
|
||||
if rate_type in self.unq_db:
|
||||
return json.loads(self.unq_db[rate_type])
|
||||
else:
|
||||
return None
|
||||
|
||||
def _save_rate(self, rate_type, rate):
|
||||
if rate is not None:
|
||||
self.db.Put(json.dumps((SETTING_TYPE, rate_type)), json.dumps(rate), sync=True)
|
||||
else:
|
||||
self.db.Delete(json.dumps((SETTING_TYPE, rate_type)), sync=True)
|
||||
self.unq_db[rate_type] = json.dumps(rate)
|
||||
elif rate_type in self.unq_db:
|
||||
del self.unq_db[rate_type]
|
||||
@@ -4,6 +4,9 @@ from twisted.internet import defer
|
||||
import logging
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ValuableQueryHandler(object):
|
||||
implements(IQueryHandler)
|
||||
|
||||
@@ -98,8 +101,8 @@ class ValuableBlobHashQueryHandler(ValuableQueryHandler):
|
||||
for blob_hash, count in valuable_hashes:
|
||||
hashes_and_scores.append((blob_hash, 1.0 * count / 10.0))
|
||||
if len(hashes_and_scores) != 0:
|
||||
logging.info("Responding to a valuable blob hashes request with %s blob hashes: %s",
|
||||
str(len(hashes_and_scores)))
|
||||
log.info("Responding to a valuable blob hashes request with %s blob hashes",
|
||||
str(len(hashes_and_scores)))
|
||||
expected_payment = 1.0 * len(hashes_and_scores) * self.valuable_blob_hash_payment_rate / 1000.0
|
||||
self.wallet.add_expected_payment(self.peer, expected_payment)
|
||||
self.peer.update_stats('uploaded_valuable_blob_hashes', len(hashes_and_scores))
|
||||
@@ -190,10 +193,10 @@ class ValuableBlobLengthQueryHandler(ValuableQueryHandler):
|
||||
if success is True:
|
||||
lengths.append(response_pair)
|
||||
if len(lengths) > 0:
|
||||
logging.info("Responding with %s blob lengths: %s", str(len(lengths)))
|
||||
log.info("Responding with %s blob lengths", str(len(lengths)))
|
||||
expected_payment = 1.0 * len(lengths) * self.blob_length_payment_rate / 1000.0
|
||||
self.wallet.add_expected_payment(self.peer, expected_payment)
|
||||
self.peer.update_stats('uploaded_valuable_blob_infos', len(lengths))
|
||||
return {'blob_length': {'blob_lengths': lengths}}
|
||||
|
||||
dl.addCallback(make_response)
|
||||
dl.addCallback(make_response)
|
||||
@@ -1,7 +1,5 @@
|
||||
import leveldb
|
||||
import os
|
||||
from lbrynet.lbrynet_console import LBRYPlugin
|
||||
from twisted.internet import defer, threads
|
||||
from twisted.internet import defer
|
||||
from lbrynet.conf import MIN_VALUABLE_BLOB_HASH_PAYMENT_RATE, MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE
|
||||
from BlindRepeater import BlindRepeater
|
||||
from BlindInfoManager import BlindInfoManager
|
||||
@@ -19,6 +17,7 @@ class BlindRepeaterPlugin(LBRYPlugin.LBRYPlugin):
|
||||
|
||||
def __init__(self):
|
||||
LBRYPlugin.LBRYPlugin.__init__(self)
|
||||
self.enabled = False
|
||||
self.blind_info_manager = None
|
||||
self.valuable_blob_length_query_handler = None
|
||||
self.valuable_blob_hash_query_handler = None
|
||||
@@ -26,14 +25,14 @@ class BlindRepeaterPlugin(LBRYPlugin.LBRYPlugin):
|
||||
self.control_handlers = None
|
||||
self.payment_rate_manager = None
|
||||
self.settings = None
|
||||
self.db = None
|
||||
|
||||
def setup(self, lbry_console):
|
||||
if not self.enabled:
|
||||
return defer.succeed(True)
|
||||
lbry_session = lbry_console.session
|
||||
d = threads.deferToThread(self._setup_db, lbry_session.db_dir)
|
||||
d.addCallback(lambda _: self._setup_settings())
|
||||
d = self._setup_settings(lbry_session.db_dir)
|
||||
d.addCallback(lambda _: self._get_payment_rate_manager(lbry_session.base_payment_rate_manager))
|
||||
d.addCallback(lambda _: self._setup_blind_info_manager(lbry_session.peer_manager))
|
||||
d.addCallback(lambda _: self._setup_blind_info_manager(lbry_session.peer_manager, lbry_session.db_dir))
|
||||
d.addCallback(lambda _: self._setup_blind_repeater(lbry_session))
|
||||
d.addCallback(lambda _: self._setup_valuable_blob_query_handler(lbry_session))
|
||||
d.addCallback(lambda _: self._create_control_handlers(lbry_session))
|
||||
@@ -41,11 +40,14 @@ class BlindRepeaterPlugin(LBRYPlugin.LBRYPlugin):
|
||||
d.addCallback(lambda _: self._add_to_lbry_console(lbry_console))
|
||||
return d
|
||||
|
||||
def _setup_db(self, db_dir):
|
||||
self.db = leveldb.LevelDB(os.path.join(db_dir, "valuable_blobs.db"))
|
||||
def stop(self):
|
||||
if self.settings is not None:
|
||||
return self.settings.stop()
|
||||
return defer.succeed(True)
|
||||
|
||||
def _setup_settings(self):
|
||||
self.settings = BlindRepeaterSettings(self.db)
|
||||
def _setup_settings(self, db_dir):
|
||||
self.settings = BlindRepeaterSettings(db_dir)
|
||||
return self.settings.setup()
|
||||
|
||||
def _get_payment_rate_manager(self, default_payment_rate_manager):
|
||||
d1 = self.settings.get_data_payment_rate()
|
||||
@@ -67,8 +69,8 @@ class BlindRepeaterPlugin(LBRYPlugin.LBRYPlugin):
|
||||
dl.addCallback(get_payment_rate_manager)
|
||||
return dl
|
||||
|
||||
def _setup_blind_info_manager(self, peer_manager):
|
||||
self.blind_info_manager = BlindInfoManager(self.db, peer_manager)
|
||||
def _setup_blind_info_manager(self, peer_manager, db_dir):
|
||||
self.blind_info_manager = BlindInfoManager(db_dir, peer_manager)
|
||||
return self.blind_info_manager.setup()
|
||||
|
||||
def _setup_valuable_blob_query_handler(self, lbry_session):
|
||||
@@ -125,4 +127,4 @@ class BlindRepeaterPlugin(LBRYPlugin.LBRYPlugin):
|
||||
def _add_to_lbry_console(self, lbry_console):
|
||||
lbry_console.add_control_handlers(self.control_handlers)
|
||||
lbry_console.add_query_handlers([self.valuable_blob_length_query_handler,
|
||||
self.valuable_blob_hash_query_handler])
|
||||
self.valuable_blob_hash_query_handler])
|
||||
@@ -1,3 +0,0 @@
|
||||
BLOB_INFO_TYPE = 'b'
|
||||
SETTING_TYPE = 's'
|
||||
PEER_TYPE = 'p'
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,88 @@
|
||||
import sys
|
||||
import json
|
||||
import argparse
|
||||
|
||||
from lbrynet.conf import API_CONNECTION_STRING
|
||||
from jsonrpc.proxy import JSONRPCProxy
|
||||
|
||||
help_msg = "Usage: lbrynet-cli method json-args\n" \
|
||||
+ "Examples: " \
|
||||
+ "lbrynet-cli resolve_name '{\"name\": \"what\"}'\n" \
|
||||
+ "lbrynet-cli get_balance\n" \
|
||||
+ "lbrynet-cli help '{\"function\": \"resolve_name\"}'\n" \
|
||||
+ "\n******lbrynet-cli functions******\n"
|
||||
|
||||
|
||||
def guess_type(x):
|
||||
if '.' in x:
|
||||
try:
|
||||
return float(x)
|
||||
except ValueError:
|
||||
# not a float
|
||||
pass
|
||||
try:
|
||||
return int(x)
|
||||
except ValueError:
|
||||
return x
|
||||
|
||||
|
||||
def get_params_from_kwargs(params):
|
||||
params_for_return = {}
|
||||
for i in params:
|
||||
eq_pos = i.index('=')
|
||||
k, v = i[:eq_pos], i[eq_pos+1:]
|
||||
params_for_return[k] = guess_type(v)
|
||||
return params_for_return
|
||||
|
||||
|
||||
def main():
|
||||
api = JSONRPCProxy.from_url(API_CONNECTION_STRING)
|
||||
|
||||
try:
|
||||
s = api.is_running()
|
||||
except:
|
||||
print "lbrynet-daemon isn't running"
|
||||
sys.exit(1)
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('method', nargs=1)
|
||||
parser.add_argument('params', nargs=argparse.REMAINDER, default=None)
|
||||
args = parser.parse_args()
|
||||
|
||||
meth = args.method[0]
|
||||
params = {}
|
||||
|
||||
if args.params:
|
||||
if len(args.params) > 1:
|
||||
params = get_params_from_kwargs(args.params)
|
||||
elif len(args.params) == 1:
|
||||
try:
|
||||
params = json.loads(args.params[0])
|
||||
except ValueError:
|
||||
params = get_params_from_kwargs(args.params)
|
||||
|
||||
msg = help_msg
|
||||
for f in api.help():
|
||||
msg += f + "\n"
|
||||
|
||||
if meth in ['--help', '-h', 'help']:
|
||||
print msg
|
||||
sys.exit(1)
|
||||
|
||||
if meth in api.help():
|
||||
try:
|
||||
if params:
|
||||
r = api.call(meth, params)
|
||||
else:
|
||||
r = api.call(meth)
|
||||
print json.dumps(r, sort_keys=True)
|
||||
except:
|
||||
print "Something went wrong, here's the usage for %s:" % meth
|
||||
print api.help({'function': meth})
|
||||
else:
|
||||
print "Unknown function"
|
||||
print msg
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,130 @@
|
||||
import argparse
|
||||
import logging
|
||||
import logging.handlers
|
||||
import os
|
||||
import webbrowser
|
||||
import sys
|
||||
import socket
|
||||
from appdirs import user_data_dir
|
||||
|
||||
from twisted.web import server
|
||||
from twisted.internet import reactor, defer
|
||||
from jsonrpc.proxy import JSONRPCProxy
|
||||
|
||||
from lbrynet.core import log_support
|
||||
from lbrynet.lbrynet_daemon.LBRYDaemonServer import LBRYDaemonServer, LBRYDaemonRequest
|
||||
from lbrynet.conf import API_CONNECTION_STRING, API_INTERFACE, API_PORT, \
|
||||
UI_ADDRESS, DEFAULT_UI_BRANCH, LOG_FILE_NAME
|
||||
|
||||
# TODO: stop it!
|
||||
if sys.platform != "darwin":
|
||||
log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
|
||||
else:
|
||||
log_dir = user_data_dir("LBRY")
|
||||
|
||||
if not os.path.isdir(log_dir):
|
||||
os.mkdir(log_dir)
|
||||
|
||||
lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME)
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
REMOTE_SERVER = "www.google.com"
|
||||
|
||||
if getattr(sys, 'frozen', False) and os.name == "nt":
|
||||
os.environ["REQUESTS_CA_BUNDLE"] = os.path.join(os.path.dirname(sys.executable), "cacert.pem")
|
||||
|
||||
|
||||
def test_internet_connection():
|
||||
try:
|
||||
host = socket.gethostbyname(REMOTE_SERVER)
|
||||
s = socket.create_connection((host, 80), 2)
|
||||
return True
|
||||
except:
|
||||
return False
|
||||
|
||||
|
||||
def stop():
|
||||
def _disp_shutdown():
|
||||
print "Shutting down lbrynet-daemon from command line"
|
||||
log.info("Shutting down lbrynet-daemon from command line")
|
||||
|
||||
def _disp_not_running():
|
||||
print "Attempt to shut down lbrynet-daemon from command line when daemon isn't running"
|
||||
log.info("Attempt to shut down lbrynet-daemon from command line when daemon isn't running")
|
||||
|
||||
d = defer.Deferred(None)
|
||||
d.addCallback(lambda _: JSONRPCProxy.from_url(API_CONNECTION_STRING).stop())
|
||||
d.addCallbacks(lambda _: _disp_shutdown(), lambda _: _disp_not_running())
|
||||
d.callback(None)
|
||||
|
||||
|
||||
def start():
|
||||
parser = argparse.ArgumentParser(description="Launch lbrynet-daemon")
|
||||
parser.add_argument("--wallet",
|
||||
help="lbrycrd or lbryum, default lbryum",
|
||||
type=str,
|
||||
default='')
|
||||
parser.add_argument("--ui",
|
||||
help="path to custom UI folder",
|
||||
default=None)
|
||||
parser.add_argument("--branch",
|
||||
help="Branch of lbry-web-ui repo to use, defaults on master")
|
||||
parser.add_argument('--no-launch', dest='launchui', action="store_false")
|
||||
parser.add_argument('--log-to-console', dest='logtoconsole', action="store_true")
|
||||
parser.add_argument('--quiet', dest='quiet', action="store_true")
|
||||
parser.set_defaults(branch=False, launchui=True, logtoconsole=False, quiet=False)
|
||||
args = parser.parse_args()
|
||||
|
||||
log_support.configure_file_handler(lbrynet_log)
|
||||
log_support.configure_loggly_handler()
|
||||
if args.logtoconsole:
|
||||
log_support.configure_console(level='DEBUG')
|
||||
log_support.disable_third_party_loggers()
|
||||
log_support.disable_noisy_loggers()
|
||||
|
||||
try:
|
||||
JSONRPCProxy.from_url(API_CONNECTION_STRING).is_running()
|
||||
log.info("lbrynet-daemon is already running")
|
||||
if not args.logtoconsole:
|
||||
print "lbrynet-daemon is already running"
|
||||
if args.launchui:
|
||||
webbrowser.open(UI_ADDRESS)
|
||||
return
|
||||
except:
|
||||
pass
|
||||
|
||||
log.info("Starting lbrynet-daemon from command line")
|
||||
|
||||
if not args.logtoconsole and not args.quiet:
|
||||
print "Starting lbrynet-daemon from command line"
|
||||
print "To view activity, view the log file here: " + lbrynet_log
|
||||
print "Web UI is available at http://%s:%i" % (API_INTERFACE, API_PORT)
|
||||
print "JSONRPC API is available at " + API_CONNECTION_STRING
|
||||
print "To quit press ctrl-c or call 'stop' via the API"
|
||||
|
||||
if test_internet_connection():
|
||||
lbry = LBRYDaemonServer()
|
||||
|
||||
d = lbry.start(branch=args.branch if args.branch else DEFAULT_UI_BRANCH,
|
||||
user_specified=args.ui,
|
||||
wallet=args.wallet,
|
||||
branch_specified=True if args.branch else False)
|
||||
if args.launchui:
|
||||
d.addCallback(lambda _: webbrowser.open(UI_ADDRESS))
|
||||
|
||||
lbrynet_server = server.Site(lbry.root)
|
||||
lbrynet_server.requestFactory = LBRYDaemonRequest
|
||||
reactor.listenTCP(API_PORT, lbrynet_server, interface=API_INTERFACE)
|
||||
reactor.run()
|
||||
|
||||
if not args.logtoconsole and not args.quiet:
|
||||
print "\nClosing lbrynet-daemon"
|
||||
else:
|
||||
log.info("Not connected to internet, unable to start")
|
||||
if not args.logtoconsole:
|
||||
print "Not connected to internet, unable to start"
|
||||
return
|
||||
|
||||
if __name__ == "__main__":
|
||||
start()
|
||||
@@ -0,0 +1,416 @@
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import json
|
||||
import sys
|
||||
import mimetypes
|
||||
import mimetools
|
||||
import tempfile
|
||||
import time
|
||||
import cgi
|
||||
|
||||
from appdirs import user_data_dir
|
||||
from twisted.web import server, static, resource
|
||||
from twisted.internet import defer, interfaces, error, reactor, threads
|
||||
|
||||
from zope.interface import implements
|
||||
|
||||
from lbrynet.lbrynet_daemon.LBRYDaemon import LBRYDaemon
|
||||
from lbrynet.conf import API_ADDRESS, UI_ADDRESS, DEFAULT_UI_BRANCH, LOG_FILE_NAME
|
||||
|
||||
|
||||
# TODO: omg, this code is essentially duplicated in LBRYDaemon
|
||||
if sys.platform != "darwin":
|
||||
data_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
|
||||
else:
|
||||
data_dir = user_data_dir("LBRY")
|
||||
if not os.path.isdir(data_dir):
|
||||
os.mkdir(data_dir)
|
||||
|
||||
lbrynet_log = os.path.join(data_dir, LOG_FILE_NAME)
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LBRYDaemonRequest(server.Request):
|
||||
"""
|
||||
For LBRY specific request functionality. Currently just provides
|
||||
handling for large multipart POST requests, taken from here:
|
||||
http://sammitch.ca/2013/07/handling-large-requests-in-twisted/
|
||||
|
||||
For multipart POST requests, this populates self.args with temp
|
||||
file objects instead of strings. Note that these files don't auto-delete
|
||||
on close because we want to be able to move and rename them.
|
||||
|
||||
"""
|
||||
|
||||
# max amount of memory to allow any ~single~ request argument [ie: POSTed file]
|
||||
# note: this value seems to be taken with a grain of salt, memory usage may spike
|
||||
# FAR above this value in some cases.
|
||||
# eg: set the memory limit to 5 MB, write 2 blocks of 4MB, mem usage will
|
||||
# have spiked to 8MB before the data is rolled to disk after the
|
||||
# second write completes.
|
||||
memorylimit = 1024*1024*100
|
||||
|
||||
# enable/disable debug logging
|
||||
do_log = False
|
||||
|
||||
# re-defined only for debug/logging purposes
|
||||
def gotLength(self, length):
|
||||
if self.do_log:
|
||||
print '%f Headers received, Content-Length: %d' % (time.time(), length)
|
||||
server.Request.gotLength(self, length)
|
||||
|
||||
# re-definition of twisted.web.server.Request.requestreceived, the only difference
|
||||
# is that self.parse_multipart() is used rather than cgi.parse_multipart()
|
||||
def requestReceived(self, command, path, version):
|
||||
from twisted.web.http import parse_qs
|
||||
if self.do_log:
|
||||
print '%f Request Received' % time.time()
|
||||
|
||||
self.content.seek(0,0)
|
||||
self.args = {}
|
||||
self.stack = []
|
||||
|
||||
self.method, self.uri = command, path
|
||||
self.clientproto = version
|
||||
x = self.uri.split(b'?', 1)
|
||||
|
||||
if len(x) == 1:
|
||||
self.path = self.uri
|
||||
else:
|
||||
self.path, argstring = x
|
||||
self.args = parse_qs(argstring, 1)
|
||||
|
||||
# cache the client and server information, we'll need this later to be
|
||||
# serialized and sent with the request so CGIs will work remotely
|
||||
self.client = self.channel.transport.getPeer()
|
||||
self.host = self.channel.transport.getHost()
|
||||
|
||||
# Argument processing
|
||||
args = self.args
|
||||
ctype = self.requestHeaders.getRawHeaders(b'content-type')
|
||||
if ctype is not None:
|
||||
ctype = ctype[0]
|
||||
|
||||
if self.method == b"POST" and ctype:
|
||||
mfd = b'multipart/form-data'
|
||||
key, pdict = cgi.parse_header(ctype)
|
||||
if key == b'application/x-www-form-urlencoded':
|
||||
args.update(parse_qs(self.content.read(), 1))
|
||||
elif key == mfd:
|
||||
try:
|
||||
self.content.seek(0,0)
|
||||
args.update(self.parse_multipart(self.content, pdict))
|
||||
#args.update(cgi.parse_multipart(self.content, pdict))
|
||||
|
||||
except KeyError as e:
|
||||
if e.args[0] == b'content-disposition':
|
||||
# Parse_multipart can't cope with missing
|
||||
# content-dispostion headers in multipart/form-data
|
||||
# parts, so we catch the exception and tell the client
|
||||
# it was a bad request.
|
||||
self.channel.transport.write(
|
||||
b"HTTP/1.1 400 Bad Request\r\n\r\n")
|
||||
self.channel.transport.loseConnection()
|
||||
return
|
||||
raise
|
||||
|
||||
self.content.seek(0, 0)
|
||||
|
||||
self.process()
|
||||
|
||||
# re-definition of cgi.parse_multipart that uses a single temporary file to store
|
||||
# data rather than storing 2 to 3 copies in various lists.
|
||||
def parse_multipart(self, fp, pdict):
|
||||
if self.do_log:
|
||||
print '%f Parsing Multipart data: ' % time.time()
|
||||
rewind = fp.tell() #save cursor
|
||||
fp.seek(0,0) #reset cursor
|
||||
|
||||
boundary = ""
|
||||
if 'boundary' in pdict:
|
||||
boundary = pdict['boundary']
|
||||
if not cgi.valid_boundary(boundary):
|
||||
raise ValueError, ('Invalid boundary in multipart form: %r'
|
||||
% (boundary,))
|
||||
|
||||
nextpart = "--" + boundary
|
||||
lastpart = "--" + boundary + "--"
|
||||
partdict = {}
|
||||
terminator = ""
|
||||
|
||||
while terminator != lastpart:
|
||||
c_bytes = -1
|
||||
|
||||
data = tempfile.NamedTemporaryFile(delete=False)
|
||||
if terminator:
|
||||
# At start of next part. Read headers first.
|
||||
headers = mimetools.Message(fp)
|
||||
clength = headers.getheader('content-length')
|
||||
if clength:
|
||||
try:
|
||||
c_bytes = int(clength)
|
||||
except ValueError:
|
||||
pass
|
||||
if c_bytes > 0:
|
||||
data.write(fp.read(c_bytes))
|
||||
# Read lines until end of part.
|
||||
while 1:
|
||||
line = fp.readline()
|
||||
if not line:
|
||||
terminator = lastpart # End outer loop
|
||||
break
|
||||
if line[:2] == "--":
|
||||
terminator = line.strip()
|
||||
if terminator in (nextpart, lastpart):
|
||||
break
|
||||
data.write(line)
|
||||
# Done with part.
|
||||
if data.tell() == 0:
|
||||
continue
|
||||
if c_bytes < 0:
|
||||
# if a Content-Length header was not supplied with the MIME part
|
||||
# then the trailing line break must be removed.
|
||||
# we have data, read the last 2 bytes
|
||||
rewind = min(2, data.tell())
|
||||
data.seek(-rewind, os.SEEK_END)
|
||||
line = data.read(2)
|
||||
if line[-2:] == "\r\n":
|
||||
data.seek(-2, os.SEEK_END)
|
||||
data.truncate()
|
||||
elif line[-1:] == "\n":
|
||||
data.seek(-1, os.SEEK_END)
|
||||
data.truncate()
|
||||
|
||||
line = headers['content-disposition']
|
||||
if not line:
|
||||
continue
|
||||
key, params = cgi.parse_header(line)
|
||||
if key != 'form-data':
|
||||
continue
|
||||
if 'name' in params:
|
||||
name = params['name']
|
||||
# kludge in the filename
|
||||
if 'filename' in params:
|
||||
fname_index = name + '_filename'
|
||||
if fname_index in partdict:
|
||||
partdict[fname_index].append(params['filename'])
|
||||
else:
|
||||
partdict[fname_index] = [params['filename']]
|
||||
else:
|
||||
# Unnamed parts are not returned at all.
|
||||
continue
|
||||
data.seek(0,0)
|
||||
if name in partdict:
|
||||
partdict[name].append(data)
|
||||
else:
|
||||
partdict[name] = [data]
|
||||
|
||||
fp.seek(rewind) # Restore cursor
|
||||
return partdict
|
||||
|
||||
class LBRYindex(resource.Resource):
|
||||
def __init__(self, ui_dir):
|
||||
resource.Resource.__init__(self)
|
||||
self.ui_dir = ui_dir
|
||||
|
||||
isLeaf = False
|
||||
|
||||
def _delayed_render(self, request, results):
|
||||
request.write(str(results))
|
||||
request.finish()
|
||||
|
||||
def getChild(self, name, request):
|
||||
if name == '':
|
||||
return self
|
||||
return resource.Resource.getChild(self, name, request)
|
||||
|
||||
def render_GET(self, request):
|
||||
request.setHeader('cache-control','no-cache, no-store, must-revalidate')
|
||||
request.setHeader('expires', '0')
|
||||
return static.File(os.path.join(self.ui_dir, "index.html")).render_GET(request)
|
||||
|
||||
|
||||
class LBRYFileStreamer(object):
|
||||
"""
|
||||
Writes downloaded LBRY file to request as the download comes in, pausing and resuming as requested
|
||||
used for Chrome
|
||||
"""
|
||||
|
||||
implements(interfaces.IPushProducer)
|
||||
|
||||
def __init__(self, request, path, start, stop, size):
|
||||
self._request = request
|
||||
self._fileObject = file(path)
|
||||
self._content_type = mimetypes.guess_type(path)[0]
|
||||
self._stop_pos = size - 1 if stop == '' else int(stop) #chrome and firefox send range requests for "0-"
|
||||
self._cursor = self._start_pos = int(start)
|
||||
self._file_size = size
|
||||
self._depth = 0
|
||||
|
||||
self._paused = self._sent_bytes = self._stopped = False
|
||||
self._delay = 0.25
|
||||
self._deferred = defer.succeed(None)
|
||||
|
||||
self._request.setResponseCode(206)
|
||||
self._request.setHeader('accept-ranges', 'bytes')
|
||||
self._request.setHeader('content-type', self._content_type)
|
||||
self._request.setHeader("Content-Security-Policy", "sandbox")
|
||||
|
||||
self.resumeProducing()
|
||||
|
||||
def pauseProducing(self):
|
||||
self._paused = True
|
||||
log.info("Pausing producer")
|
||||
return defer.succeed(None)
|
||||
|
||||
def resumeProducing(self):
|
||||
def _check_for_new_data():
|
||||
self._depth += 1
|
||||
self._fileObject.seek(self._start_pos, os.SEEK_END)
|
||||
readable_bytes = self._fileObject.tell()
|
||||
self._fileObject.seek(self._cursor)
|
||||
|
||||
self._sent_bytes = False
|
||||
|
||||
if (readable_bytes > self._cursor) and not (self._stopped or self._paused):
|
||||
read_length = min(readable_bytes, self._stop_pos) - self._cursor + 1
|
||||
self._request.setHeader('content-range', 'bytes %s-%s/%s' % (self._cursor, self._cursor + read_length - 1, self._file_size))
|
||||
self._request.setHeader('content-length', str(read_length))
|
||||
start_cur = self._cursor
|
||||
for i in range(read_length):
|
||||
if self._paused or self._stopped:
|
||||
break
|
||||
else:
|
||||
data = self._fileObject.read(1)
|
||||
self._request.write(data)
|
||||
self._cursor += 1
|
||||
|
||||
log.info("Wrote range %s-%s/%s, length: %s, readable: %s, depth: %s" %
|
||||
(start_cur, self._cursor, self._file_size, self._cursor - start_cur, readable_bytes, self._depth))
|
||||
self._sent_bytes = True
|
||||
|
||||
if self._cursor == self._stop_pos + 1:
|
||||
self.stopProducing()
|
||||
return defer.succeed(None)
|
||||
elif self._paused or self._stopped:
|
||||
return defer.succeed(None)
|
||||
else:
|
||||
self._deferred.addCallback(lambda _: threads.deferToThread(reactor.callLater, self._delay, _check_for_new_data))
|
||||
return defer.succeed(None)
|
||||
|
||||
log.info("Resuming producer")
|
||||
self._paused = False
|
||||
self._deferred.addCallback(lambda _: _check_for_new_data())
|
||||
|
||||
def stopProducing(self):
|
||||
log.info("Stopping producer")
|
||||
self._stopped = True
|
||||
# self._fileObject.close()
|
||||
self._deferred.addErrback(lambda err: err.trap(defer.CancelledError))
|
||||
self._deferred.addErrback(lambda err: err.trap(error.ConnectionDone))
|
||||
self._deferred.cancel()
|
||||
# self._request.finish()
|
||||
self._request.unregisterProducer()
|
||||
return defer.succeed(None)
|
||||
|
||||
|
||||
class HostedLBRYFile(resource.Resource):
|
||||
def __init__(self, api):
|
||||
self._api = api
|
||||
self._producer = None
|
||||
resource.Resource.__init__(self)
|
||||
|
||||
# todo: fix LBRYFileStreamer and use it instead of static.File
|
||||
# def makeProducer(self, request, stream):
|
||||
# def _save_producer(producer):
|
||||
# self._producer = producer
|
||||
# return defer.succeed(None)
|
||||
#
|
||||
# range_header = request.getAllHeaders()['range'].replace('bytes=', '').split('-')
|
||||
# start, stop = int(range_header[0]), range_header[1]
|
||||
# log.info("GET range %s-%s" % (start, stop))
|
||||
# path = os.path.join(self._api.download_directory, stream.file_name)
|
||||
#
|
||||
# d = stream.get_total_bytes()
|
||||
# d.addCallback(lambda size: _save_producer(LBRYFileStreamer(request, path, start, stop, size)))
|
||||
# d.addCallback(lambda _: request.registerProducer(self._producer, streaming=True))
|
||||
# # request.notifyFinish().addCallback(lambda _: self._producer.stopProducing())
|
||||
# request.notifyFinish().addErrback(self._responseFailed, d)
|
||||
# return d
|
||||
|
||||
def render_GET(self, request):
|
||||
request.setHeader("Content-Security-Policy", "sandbox")
|
||||
if 'name' in request.args.keys():
|
||||
if request.args['name'][0] != 'lbry' and request.args['name'][0] not in self._api.waiting_on.keys():
|
||||
d = self._api._download_name(request.args['name'][0])
|
||||
# d.addCallback(lambda stream: self.makeProducer(request, stream))
|
||||
d.addCallback(lambda stream: static.File(os.path.join(self._api.download_directory,
|
||||
stream.file_name)).render_GET(request))
|
||||
|
||||
elif request.args['name'][0] in self._api.waiting_on.keys():
|
||||
request.redirect(UI_ADDRESS + "/?watch=" + request.args['name'][0])
|
||||
request.finish()
|
||||
else:
|
||||
request.redirect(UI_ADDRESS)
|
||||
request.finish()
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
# def _responseFailed(self, err, call):
|
||||
# call.addErrback(lambda err: err.trap(error.ConnectionDone))
|
||||
# call.addErrback(lambda err: err.trap(defer.CancelledError))
|
||||
# call.addErrback(lambda err: log.info("Error: " + str(err)))
|
||||
# call.cancel()
|
||||
|
||||
class LBRYFileUpload(resource.Resource):
|
||||
"""
|
||||
Accepts a file sent via the file upload widget in the web UI, saves
|
||||
it into a temporary dir, and responds with a JSON string containing
|
||||
the path of the newly created file.
|
||||
"""
|
||||
|
||||
def __init__(self, api):
|
||||
self._api = api
|
||||
|
||||
def render_POST(self, request):
|
||||
origfilename = request.args['file_filename'][0]
|
||||
uploaded_file = request.args['file'][0] # Temp file created by request
|
||||
|
||||
# Move to a new temporary dir and restore the original file name
|
||||
newdirpath = tempfile.mkdtemp()
|
||||
newpath = os.path.join(newdirpath, origfilename)
|
||||
if os.name == "nt":
|
||||
shutil.copy(uploaded_file.name, newpath)
|
||||
# TODO Still need to remove the file
|
||||
|
||||
# TODO deal with pylint error in cleaner fashion than this
|
||||
try:
|
||||
from exceptions import WindowsError as win_except
|
||||
except ImportError as e:
|
||||
log.error("This shouldn't happen")
|
||||
win_except = Exception
|
||||
|
||||
try:
|
||||
os.remove(uploaded_file.name)
|
||||
except win_except as e:
|
||||
pass
|
||||
else:
|
||||
shutil.move(uploaded_file.name, newpath)
|
||||
self._api.uploaded_temp_files.append(newpath)
|
||||
|
||||
return json.dumps(newpath)
|
||||
|
||||
|
||||
class LBRYDaemonServer(object):
|
||||
def _setup_server(self, wallet):
|
||||
self.root = LBRYindex(os.path.join(os.path.join(data_dir, "lbry-ui"), "active"))
|
||||
self._api = LBRYDaemon(self.root, wallet_type=wallet)
|
||||
self.root.putChild("view", HostedLBRYFile(self._api))
|
||||
self.root.putChild("upload", LBRYFileUpload(self._api))
|
||||
self.root.putChild(API_ADDRESS, self._api)
|
||||
return defer.succeed(True)
|
||||
|
||||
def start(self, branch=DEFAULT_UI_BRANCH, user_specified=False, branch_specified=False, wallet=None):
|
||||
d = self._setup_server(wallet)
|
||||
d.addCallback(lambda _: self._api.setup(branch, user_specified, branch_specified))
|
||||
return d
|
||||
@@ -0,0 +1,166 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
from copy import deepcopy
|
||||
from appdirs import user_data_dir
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.task import LoopingCall
|
||||
|
||||
from lbrynet.core.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed
|
||||
from lbrynet.core.PaymentRateManager import PaymentRateManager
|
||||
from lbrynet.core.StreamDescriptor import download_sd_blob
|
||||
from lbrynet.metadata.LBRYFee import LBRYFeeValidator
|
||||
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloaderFactory
|
||||
from lbrynet.conf import DEFAULT_TIMEOUT, LOG_FILE_NAME
|
||||
|
||||
INITIALIZING_CODE = 'initializing'
|
||||
DOWNLOAD_METADATA_CODE = 'downloading_metadata'
|
||||
DOWNLOAD_TIMEOUT_CODE = 'timeout'
|
||||
DOWNLOAD_RUNNING_CODE = 'running'
|
||||
DOWNLOAD_STOPPED_CODE = 'stopped'
|
||||
STREAM_STAGES = [
|
||||
(INITIALIZING_CODE, 'Initializing...'),
|
||||
(DOWNLOAD_METADATA_CODE, 'Downloading metadata'),
|
||||
(DOWNLOAD_RUNNING_CODE, 'Started stream'),
|
||||
(DOWNLOAD_STOPPED_CODE, 'Paused stream'),
|
||||
(DOWNLOAD_TIMEOUT_CODE, 'Stream timed out')
|
||||
]
|
||||
|
||||
if sys.platform != "darwin":
|
||||
log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
|
||||
else:
|
||||
log_dir = user_data_dir("LBRY")
|
||||
|
||||
if not os.path.isdir(log_dir):
|
||||
os.mkdir(log_dir)
|
||||
|
||||
lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME)
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class GetStream(object):
|
||||
def __init__(self, sd_identifier, session, wallet, lbry_file_manager, exchange_rate_manager,
|
||||
max_key_fee, data_rate=0.5, timeout=DEFAULT_TIMEOUT, download_directory=None, file_name=None):
|
||||
self.wallet = wallet
|
||||
self.resolved_name = None
|
||||
self.description = None
|
||||
self.fee = None
|
||||
self.data_rate = data_rate
|
||||
self.name = None
|
||||
self.file_name = file_name
|
||||
self.session = session
|
||||
self.exchange_rate_manager = exchange_rate_manager
|
||||
self.payment_rate_manager = PaymentRateManager(self.session.base_payment_rate_manager)
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
self.sd_identifier = sd_identifier
|
||||
self.stream_hash = None
|
||||
self.max_key_fee = max_key_fee
|
||||
self.stream_info = None
|
||||
self.stream_info_manager = None
|
||||
self.d = defer.Deferred(None)
|
||||
self.timeout = timeout
|
||||
self.timeout_counter = 0
|
||||
self.download_directory = download_directory
|
||||
self.download_path = None
|
||||
self.downloader = None
|
||||
self.finished = defer.Deferred(None)
|
||||
self.checker = LoopingCall(self.check_status)
|
||||
self.code = STREAM_STAGES[0]
|
||||
|
||||
def check_status(self):
|
||||
self.timeout_counter += 1
|
||||
|
||||
# TODO: Why is this the stopping condition for the finished callback?
|
||||
if self.download_path:
|
||||
self.checker.stop()
|
||||
self.finished.callback((True, self.stream_hash, self.download_path))
|
||||
|
||||
elif self.timeout_counter >= self.timeout:
|
||||
log.info("Timeout downloading lbry://%s" % self.resolved_name)
|
||||
self.checker.stop()
|
||||
self.d.cancel()
|
||||
self.code = STREAM_STAGES[4]
|
||||
self.finished.callback((False, None, None))
|
||||
|
||||
def _convert_max_fee(self):
|
||||
if isinstance(self.max_key_fee, dict):
|
||||
max_fee = LBRYFeeValidator(self.max_key_fee)
|
||||
if max_fee.currency_symbol == "LBC":
|
||||
return max_fee.amount
|
||||
return self.exchange_rate_manager.to_lbc(self.fee).amount
|
||||
elif isinstance(self.max_key_fee, float):
|
||||
return float(self.max_key_fee)
|
||||
|
||||
def start(self, stream_info, name):
|
||||
def _cause_timeout(err):
|
||||
log.info('Cancelling download')
|
||||
self.timeout_counter = self.timeout * 2
|
||||
|
||||
def _set_status(x, status):
|
||||
log.info("Download lbry://%s status changed to %s" % (self.resolved_name, status))
|
||||
self.code = next(s for s in STREAM_STAGES if s[0] == status)
|
||||
return x
|
||||
|
||||
def get_downloader_factory(metadata):
|
||||
for factory in metadata.factories:
|
||||
if isinstance(factory, ManagedLBRYFileDownloaderFactory):
|
||||
return factory, metadata
|
||||
raise Exception('No suitable factory was found in {}'.format(metadata.factories))
|
||||
|
||||
def make_downloader(args):
|
||||
factory, metadata = args
|
||||
return factory.make_downloader(metadata,
|
||||
[self.data_rate, True],
|
||||
self.payment_rate_manager,
|
||||
download_directory=self.download_directory,
|
||||
file_name=self.file_name)
|
||||
|
||||
self.resolved_name = name
|
||||
self.stream_info = deepcopy(stream_info)
|
||||
self.description = self.stream_info['description']
|
||||
self.stream_hash = self.stream_info['sources']['lbry_sd_hash']
|
||||
|
||||
if 'fee' in self.stream_info:
|
||||
self.fee = LBRYFeeValidator(self.stream_info['fee'])
|
||||
max_key_fee = self._convert_max_fee()
|
||||
if self.exchange_rate_manager.to_lbc(self.fee).amount > max_key_fee:
|
||||
log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.fee.amount,
|
||||
self.max_key_fee,
|
||||
self.resolved_name))
|
||||
return defer.fail(KeyFeeAboveMaxAllowed())
|
||||
log.info("Key fee %s below limit of %f, downloading lbry://%s" % (json.dumps(self.fee),
|
||||
max_key_fee,
|
||||
self.resolved_name))
|
||||
|
||||
self.checker.start(1)
|
||||
|
||||
self.d.addCallback(lambda _: _set_status(None, DOWNLOAD_METADATA_CODE))
|
||||
self.d.addCallback(lambda _: download_sd_blob(self.session, self.stream_hash, self.payment_rate_manager))
|
||||
self.d.addCallback(self.sd_identifier.get_metadata_for_sd_blob)
|
||||
self.d.addCallback(lambda r: _set_status(r, DOWNLOAD_RUNNING_CODE))
|
||||
self.d.addCallback(get_downloader_factory)
|
||||
self.d.addCallback(make_downloader)
|
||||
self.d.addCallbacks(self._start_download, _cause_timeout)
|
||||
self.d.callback(None)
|
||||
|
||||
return self.finished
|
||||
|
||||
def _start_download(self, downloader):
|
||||
log.info('Starting download for %s', self.name)
|
||||
self.downloader = downloader
|
||||
self.download_path = os.path.join(downloader.download_directory, downloader.file_name)
|
||||
|
||||
d = self._pay_key_fee()
|
||||
d.addCallback(lambda _: log.info("Downloading %s --> %s", self.stream_hash, self.downloader.file_name))
|
||||
d.addCallback(lambda _: self.downloader.start())
|
||||
|
||||
def _pay_key_fee(self):
|
||||
if self.fee is not None:
|
||||
fee_lbc = self.exchange_rate_manager.to_lbc(self.fee).amount
|
||||
reserved_points = self.wallet.reserve_points(self.fee.address, fee_lbc)
|
||||
if reserved_points is None:
|
||||
return defer.fail(InsufficientFundsError())
|
||||
return self.wallet.send_points_to_address(reserved_points, fee_lbc)
|
||||
return defer.succeed(None)
|
||||
@@ -0,0 +1,215 @@
|
||||
import time
|
||||
import requests
|
||||
import logging
|
||||
import json
|
||||
import googlefinance
|
||||
from twisted.internet import defer, reactor
|
||||
from twisted.internet.task import LoopingCall
|
||||
|
||||
from lbrynet.metadata.LBRYFee import LBRYFeeValidator
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
CURRENCY_PAIRS = ["USDBTC", "BTCLBC"]
|
||||
BITTREX_FEE = 0.0025
|
||||
COINBASE_FEE = 0.0 #add fee
|
||||
|
||||
|
||||
class ExchangeRate(object):
|
||||
def __init__(self, market, spot, ts):
|
||||
assert int(time.time()) - ts < 600
|
||||
self.currency_pair = (market[0:3], market[3:6])
|
||||
self.spot = spot
|
||||
self.ts = ts
|
||||
|
||||
def as_dict(self):
|
||||
return {'spot': self.spot, 'ts': self.ts}
|
||||
|
||||
|
||||
class MarketFeed(object):
|
||||
def __init__(self, market, name, url, params, fee):
|
||||
self.market = market
|
||||
self.name = name
|
||||
self.url = url
|
||||
self.params = params
|
||||
self.fee = fee
|
||||
self.rate = None
|
||||
self._updater = LoopingCall(self._update_price)
|
||||
|
||||
def _make_request(self):
|
||||
r = requests.get(self.url, self.params)
|
||||
return r.text
|
||||
|
||||
def _handle_response(self, response):
|
||||
return NotImplementedError
|
||||
|
||||
def _subtract_fee(self, from_amount):
|
||||
return defer.succeed(from_amount / (1.0 - self.fee))
|
||||
|
||||
def _save_price(self, price):
|
||||
log.debug("Saving price update %f for %s" % (price, self.market))
|
||||
self.rate = ExchangeRate(self.market, price, int(time.time()))
|
||||
|
||||
def _update_price(self):
|
||||
d = defer.succeed(self._make_request())
|
||||
d.addCallback(self._handle_response)
|
||||
d.addCallback(self._subtract_fee)
|
||||
d.addCallback(self._save_price)
|
||||
|
||||
def start(self):
|
||||
if not self._updater.running:
|
||||
self._updater.start(300)
|
||||
|
||||
def stop(self):
|
||||
if self._updater.running:
|
||||
self._updater.stop()
|
||||
|
||||
|
||||
class BittrexFeed(MarketFeed):
|
||||
def __init__(self):
|
||||
MarketFeed.__init__(
|
||||
self,
|
||||
"BTCLBC",
|
||||
"Bittrex",
|
||||
"https://bittrex.com/api/v1.1/public/getmarkethistory",
|
||||
{'market': 'BTC-LBC', 'count': 50},
|
||||
BITTREX_FEE
|
||||
)
|
||||
|
||||
def _handle_response(self, response):
|
||||
trades = json.loads(response)['result']
|
||||
vwap = sum([i['Total'] for i in trades]) / sum([i['Quantity'] for i in trades])
|
||||
return defer.succeed(float(1.0 / vwap))
|
||||
|
||||
|
||||
class GoogleBTCFeed(MarketFeed):
|
||||
def __init__(self):
|
||||
MarketFeed.__init__(
|
||||
self,
|
||||
"USDBTC",
|
||||
"Coinbase via Google finance",
|
||||
None,
|
||||
None,
|
||||
COINBASE_FEE
|
||||
)
|
||||
|
||||
def _make_request(self):
|
||||
return googlefinance.getQuotes('CURRENCY:USDBTC')[0]
|
||||
|
||||
def _handle_response(self, response):
|
||||
return float(response['LastTradePrice'])
|
||||
|
||||
|
||||
def get_default_market_feed(currency_pair):
|
||||
currencies = None
|
||||
if isinstance(currency_pair, str):
|
||||
currencies = (currency_pair[0:3], currency_pair[3:6])
|
||||
elif isinstance(currency_pair, tuple):
|
||||
currencies = currency_pair
|
||||
assert currencies is not None
|
||||
|
||||
if currencies == ("USD", "BTC"):
|
||||
return GoogleBTCFeed()
|
||||
elif currencies == ("BTC", "LBC"):
|
||||
return BittrexFeed()
|
||||
|
||||
|
||||
class ExchangeRateManager(object):
|
||||
def __init__(self):
|
||||
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
|
||||
self.market_feeds = [get_default_market_feed(currency_pair) for currency_pair in CURRENCY_PAIRS]
|
||||
|
||||
def start(self):
|
||||
log.info("Starting exchange rate manager")
|
||||
for feed in self.market_feeds:
|
||||
feed.start()
|
||||
|
||||
def stop(self):
|
||||
log.info("Stopping exchange rate manager")
|
||||
for source in self.market_feeds:
|
||||
source.stop()
|
||||
|
||||
def convert_currency(self, from_currency, to_currency, amount):
|
||||
log.info("Converting %f %s to %s" % (amount, from_currency, to_currency))
|
||||
if from_currency == to_currency:
|
||||
return amount
|
||||
for market in self.market_feeds:
|
||||
if market.rate.currency_pair == (from_currency, to_currency):
|
||||
return amount * market.rate.spot
|
||||
for market in self.market_feeds:
|
||||
if market.rate.currency_pair[0] == from_currency:
|
||||
return self.convert_currency(market.rate.currency_pair[1], to_currency, amount * market.rate.spot)
|
||||
raise Exception('Unable to convert {} from {} to {}'.format(amount, from_currency, to_currency))
|
||||
|
||||
def fee_dict(self):
|
||||
return {market: market.rate.as_dict() for market in self.market_feeds}
|
||||
|
||||
def to_lbc(self, fee):
|
||||
if fee is None:
|
||||
return None
|
||||
if not isinstance(fee, LBRYFeeValidator):
|
||||
fee_in = LBRYFeeValidator(fee)
|
||||
else:
|
||||
fee_in = fee
|
||||
|
||||
return LBRYFeeValidator({fee_in.currency_symbol:
|
||||
{
|
||||
'amount': self.convert_currency(fee_in.currency_symbol, "LBC", fee_in.amount),
|
||||
'address': fee_in.address
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
class DummyBTCLBCFeed(MarketFeed):
|
||||
def __init__(self):
|
||||
MarketFeed.__init__(
|
||||
self,
|
||||
"BTCLBC",
|
||||
"market name",
|
||||
"derp.com",
|
||||
None,
|
||||
0.0
|
||||
)
|
||||
|
||||
|
||||
class DummyUSDBTCFeed(MarketFeed):
|
||||
def __init__(self):
|
||||
MarketFeed.__init__(
|
||||
self,
|
||||
"USDBTC",
|
||||
"market name",
|
||||
"derp.com",
|
||||
None,
|
||||
0.0
|
||||
)
|
||||
|
||||
|
||||
class DummyExchangeRateManager(object):
|
||||
def __init__(self, rates):
|
||||
self.market_feeds = [DummyBTCLBCFeed(), DummyUSDBTCFeed()]
|
||||
for feed in self.market_feeds:
|
||||
feed.rate = ExchangeRate(feed.market, rates[feed.market]['spot'], rates[feed.market]['ts'])
|
||||
|
||||
def convert_currency(self, from_currency, to_currency, amount):
|
||||
log.debug("Converting %f %s to %s" % (amount, from_currency, to_currency))
|
||||
for market in self.market_feeds:
|
||||
if market.rate.currency_pair == (from_currency, to_currency):
|
||||
return amount * market.rate.spot
|
||||
for market in self.market_feeds:
|
||||
if market.rate.currency_pair[0] == from_currency:
|
||||
return self.convert_currency(market.rate.currency_pair[1], to_currency, amount * market.rate.spot)
|
||||
|
||||
def to_lbc(self, fee):
|
||||
if fee is None:
|
||||
return None
|
||||
if not isinstance(fee, LBRYFeeValidator):
|
||||
fee_in = LBRYFeeValidator(fee)
|
||||
else:
|
||||
fee_in = fee
|
||||
|
||||
return LBRYFeeValidator({fee_in.currency_symbol:
|
||||
{
|
||||
'amount': self.convert_currency(fee_in.currency_symbol, "LBC", fee_in.amount),
|
||||
'address': fee_in.address
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,164 @@
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
import sys
|
||||
import random
|
||||
|
||||
from appdirs import user_data_dir
|
||||
|
||||
from lbrynet.core.Error import InsufficientFundsError
|
||||
from lbrynet.lbryfilemanager.LBRYFileCreator import create_lbry_file
|
||||
from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob
|
||||
from lbrynet.core.PaymentRateManager import PaymentRateManager
|
||||
from lbrynet.metadata.LBRYMetadata import Metadata
|
||||
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader
|
||||
from lbrynet import reflector
|
||||
from lbrynet.conf import LOG_FILE_NAME, REFLECTOR_SERVERS
|
||||
from twisted.internet import threads, defer, reactor
|
||||
|
||||
if sys.platform != "darwin":
|
||||
log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
|
||||
else:
|
||||
log_dir = user_data_dir("LBRY")
|
||||
|
||||
if not os.path.isdir(log_dir):
|
||||
os.mkdir(log_dir)
|
||||
|
||||
lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME)
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Publisher(object):
|
||||
def __init__(self, session, lbry_file_manager, wallet):
|
||||
self.session = session
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
self.wallet = wallet
|
||||
self.received_file_name = False
|
||||
self.file_path = None
|
||||
self.file_name = None
|
||||
self.publish_name = None
|
||||
self.bid_amount = None
|
||||
self.verified = False
|
||||
self.lbry_file = None
|
||||
self.txid = None
|
||||
self.stream_hash = None
|
||||
reflector_server = random.choice(REFLECTOR_SERVERS)
|
||||
self.reflector_server, self.reflector_port = reflector_server[0], reflector_server[1]
|
||||
self.metadata = {}
|
||||
|
||||
def start(self, name, file_path, bid, metadata):
|
||||
log.info('Starting publish for %s', name)
|
||||
def _show_result():
|
||||
log.info("Success! Published %s --> lbry://%s txid: %s", self.file_name, self.publish_name, self.txid)
|
||||
return defer.succeed(self.txid)
|
||||
|
||||
self.publish_name = name
|
||||
self.file_path = file_path
|
||||
self.bid_amount = bid
|
||||
self.metadata = metadata
|
||||
|
||||
if os.name == "nt":
|
||||
file_mode = 'rb'
|
||||
else:
|
||||
file_mode = 'r'
|
||||
|
||||
d = self._check_file_path(self.file_path)
|
||||
d.addCallback(lambda _: create_lbry_file(self.session, self.lbry_file_manager,
|
||||
self.file_name, open(self.file_path, file_mode)))
|
||||
d.addCallback(self.add_to_lbry_files)
|
||||
d.addCallback(lambda _: self._create_sd_blob())
|
||||
d.addCallback(lambda _: self._claim_name())
|
||||
d.addCallback(lambda _: self.set_status())
|
||||
d.addCallback(lambda _: self.start_reflector())
|
||||
d.addCallbacks(lambda _: _show_result(), self._show_publish_error)
|
||||
|
||||
return d
|
||||
|
||||
def start_reflector(self):
|
||||
reflector_server = random.choice(REFLECTOR_SERVERS)
|
||||
reflector_address, reflector_port = reflector_server[0], reflector_server[1]
|
||||
log.info("Reflecting new publication")
|
||||
factory = reflector.ClientFactory(
|
||||
self.session.blob_manager,
|
||||
self.lbry_file_manager.stream_info_manager,
|
||||
self.stream_hash
|
||||
)
|
||||
d = reactor.resolve(reflector_address)
|
||||
d.addCallback(lambda ip: reactor.connectTCP(ip, reflector_port, factory))
|
||||
d.addCallback(lambda _: factory.finished_deferred)
|
||||
return d
|
||||
|
||||
def _check_file_path(self, file_path):
|
||||
def check_file_threaded():
|
||||
f = open(file_path)
|
||||
f.close()
|
||||
self.file_name = os.path.basename(self.file_path)
|
||||
return True
|
||||
return threads.deferToThread(check_file_threaded)
|
||||
|
||||
def set_lbry_file(self, lbry_file_downloader):
|
||||
self.lbry_file = lbry_file_downloader
|
||||
return defer.succeed(None)
|
||||
|
||||
def add_to_lbry_files(self, stream_hash):
|
||||
self.stream_hash = stream_hash
|
||||
prm = PaymentRateManager(self.session.base_payment_rate_manager)
|
||||
d = self.lbry_file_manager.add_lbry_file(stream_hash, prm)
|
||||
d.addCallback(self.set_lbry_file)
|
||||
return d
|
||||
|
||||
def _create_sd_blob(self):
|
||||
log.debug('Creating stream descriptor blob')
|
||||
d = publish_sd_blob(self.lbry_file_manager.stream_info_manager,
|
||||
self.session.blob_manager,
|
||||
self.lbry_file.stream_hash)
|
||||
|
||||
def set_sd_hash(sd_hash):
|
||||
log.debug('stream descriptor hash: %s', sd_hash)
|
||||
if 'sources' not in self.metadata:
|
||||
self.metadata['sources'] = {}
|
||||
self.metadata['sources']['lbry_sd_hash'] = sd_hash
|
||||
|
||||
d.addCallback(set_sd_hash)
|
||||
return d
|
||||
|
||||
def set_status(self):
|
||||
log.debug('Setting status')
|
||||
d = self.lbry_file_manager.change_lbry_file_status(self.lbry_file, ManagedLBRYFileDownloader.STATUS_FINISHED)
|
||||
d.addCallback(lambda _: self.lbry_file.restore())
|
||||
return d
|
||||
|
||||
def _claim_name(self):
|
||||
log.debug('Claiming name')
|
||||
self._update_metadata()
|
||||
m = Metadata(self.metadata)
|
||||
|
||||
def set_tx_hash(txid):
|
||||
log.debug('Name claimed using txid: %s', txid)
|
||||
self.txid = txid
|
||||
|
||||
d = self.wallet.claim_name(self.publish_name, self.bid_amount, m)
|
||||
d.addCallback(set_tx_hash)
|
||||
return d
|
||||
|
||||
def _update_metadata(self):
|
||||
filename = os.path.join(self.lbry_file.download_directory, self.lbry_file.file_name)
|
||||
self.metadata['content_type'] = get_content_type(filename)
|
||||
self.metadata['ver'] = Metadata.current_version
|
||||
|
||||
def _show_publish_error(self, err):
|
||||
log.info(err.getTraceback())
|
||||
message = "An error occurred publishing %s to %s. Error: %s."
|
||||
if err.check(InsufficientFundsError):
|
||||
error_message = "Insufficient funds"
|
||||
else:
|
||||
error_message = err.getErrorMessage()
|
||||
|
||||
log.error(error_message)
|
||||
log.error(message, str(self.file_name), str(self.publish_name), err.getTraceback())
|
||||
|
||||
return defer.fail(Exception("Publish failed"))
|
||||
|
||||
|
||||
def get_content_type(filename):
|
||||
return mimetypes.guess_type(filename)[0]
|
||||
Loaded 100 of 189 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user