Compare commits
1646
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b62cdb35ed | ||
|
|
e775aab89b | ||
|
|
bb3337da5f | ||
|
|
47f7ab7bd6 | ||
|
|
2005cc2948 | ||
|
|
2e5c6a6d2a | ||
|
|
ca34d703a2 | ||
|
|
af681d969c | ||
|
|
45799bf330 | ||
|
|
f8bb89c8cd | ||
|
|
8faaf9f465 | ||
|
|
973ee4f08c | ||
|
|
b83360f3e2 | ||
|
|
dac1b82ea7 | ||
|
|
56f80cbcda | ||
|
|
28abd9c449 | ||
|
|
8d86b0754c | ||
|
|
bed3255b89 | ||
|
|
a17a31acf5 | ||
|
|
1f815cf2d2 | ||
|
|
07ee73b653 | ||
|
|
7a9e8c6769 | ||
|
|
a2901e4331 | ||
|
|
7f8268703c | ||
|
|
32d2208fd9 | ||
|
|
704ec9e553 | ||
|
|
e0f7066163 | ||
|
|
32b26c9fa5 | ||
|
|
0d9d576436 | ||
|
|
e6c275f86e | ||
|
|
937adbf439 | ||
|
|
6a5ff0636c | ||
|
|
888d47f88b | ||
|
|
d7e50b269f | ||
|
|
46ce175481 | ||
|
|
6b2d4175be | ||
|
|
16bfb8589b | ||
|
|
a4bb4db8dd | ||
|
|
de1e2d0e3b | ||
|
|
a4880c1cf0 | ||
|
|
28f25538a3 | ||
|
|
83c8576b3f | ||
|
|
6fc909ea41 | ||
|
|
c17544d8ef | ||
|
|
7c46cc0805 | ||
|
|
fb4dc8342a | ||
|
|
df91f4754a | ||
|
|
7d8bc38cb9 | ||
|
|
a319595f37 | ||
|
|
31312af517 | ||
|
|
95ec1f3af4 | ||
|
|
b093aa3911 | ||
|
|
7bd157ef17 | ||
|
|
7f67cbfb40 | ||
|
|
2ea48bc8c2 | ||
|
|
8d42b375a0 | ||
|
|
c2acceaed5 | ||
|
|
a2db18010b | ||
|
|
545b7c33b1 | ||
|
|
07d584133e | ||
|
|
a10eb30771 | ||
|
|
98e264f4cd | ||
|
|
083d6a3bc3 | ||
|
|
94e87f99d8 | ||
|
|
fc4114621c | ||
|
|
63bd6f5792 | ||
|
|
0c86ed924b | ||
|
|
4a93b906d7 | ||
|
|
b727d2815f | ||
|
|
bad8ae7832 | ||
|
|
cffe895d22 | ||
|
|
ca4fec272a | ||
|
|
66419f1aa6 | ||
|
|
a13735769b | ||
|
|
749f72a8c5 | ||
|
|
e5f124fe68 | ||
|
|
88ed67a5b3 | ||
|
|
d5598462b6 | ||
|
|
31c60e167a | ||
|
|
fe04bfa10a | ||
|
|
c15bedfb6d | ||
|
|
0ff62495c7 | ||
|
|
b4be712a50 | ||
|
|
faa43fc20e | ||
|
|
f7deaa3303 | ||
|
|
d7ecde7040 | ||
|
|
e2a75758f8 | ||
|
|
f449cf61ab | ||
|
|
04db81e954 | ||
|
|
b49c9fd050 | ||
|
|
3ff2bcf913 | ||
|
|
556056c60d | ||
|
|
3c03fff380 | ||
|
|
b1441d4247 | ||
|
|
81458b75e4 | ||
|
|
f0f8ef044b | ||
|
|
47305e7446 | ||
|
|
ba6b985d71 | ||
|
|
49802b39cb | ||
|
|
46bcc5d725 | ||
|
|
98f8fd0556 | ||
|
|
c0ce27ccf3 | ||
|
|
cfae30a364 | ||
|
|
53e3828965 | ||
|
|
4f16f1c829 | ||
|
|
478bd0510b | ||
|
|
499ee74dfc | ||
|
|
d3da442727 | ||
|
|
358fa21eaf | ||
|
|
20f35d02fa | ||
|
|
77e64ef028 | ||
|
|
0a71e2ff91 | ||
|
|
dd5b9ca81b | ||
|
|
89ed04f8a7 | ||
|
|
ec0d9f06c5 | ||
|
|
03b59ac6fc | ||
|
|
43ac3336d7 | ||
|
|
d12c78db74 | ||
|
|
bfaf1b0957 | ||
|
|
bb60c385d5 | ||
|
|
c96d1d9c32 | ||
|
|
7c7a0d4bdf | ||
|
|
cc829a7bf4 | ||
|
|
e0ea6383e2 | ||
|
|
bcec5dc2ae | ||
|
|
cba9c16a06 | ||
|
|
dd68fb077b | ||
|
|
c2294e97db | ||
|
|
c0f512ace7 | ||
|
|
3305eb67c6 | ||
|
|
c9d637b4da | ||
|
|
ae3e8fadf5 | ||
|
|
a1abd94387 | ||
|
|
9b463a8cab | ||
|
|
babc54a240 | ||
|
|
5836a93b21 | ||
|
|
557348e345 | ||
|
|
9adfec6b00 | ||
|
|
3a496902f8 | ||
|
|
b5ead91746 | ||
|
|
302461b446 | ||
|
|
ac201c718e | ||
|
|
f78e3825ca | ||
|
|
0618053bd4 | ||
|
|
8e6fa3490c | ||
|
|
8a1a1a4000 | ||
|
|
fd9dcbf9a8 | ||
|
|
beb8583436 | ||
|
|
b44e2c0b38 | ||
|
|
06e94640b5 | ||
|
|
ff36bdc802 | ||
|
|
46f576de46 | ||
|
|
7b09c34fce | ||
|
|
a22f50aa84 | ||
|
|
2d9130b4e0 | ||
|
|
470ee72462 | ||
|
|
add147b409 | ||
|
|
371df6e6c2 | ||
|
|
7ed5fe8f66 | ||
|
|
a6ca7a6f38 | ||
|
|
1c857b8dd8 | ||
|
|
87ff3f95ff | ||
|
|
5cb4c06d0c | ||
|
|
e7d9079389 | ||
|
|
9cdcff0e1e | ||
|
|
a4dce8cf9f | ||
|
|
aaa11c02bf | ||
|
|
d2ebbf5db6 | ||
|
|
e6efc1ad4a | ||
|
|
a8523996a9 | ||
|
|
f586de2bbe | ||
|
|
7df02303b2 | ||
|
|
f89c75e642 | ||
|
|
d2c1961101 | ||
|
|
2a4c5a48bf | ||
|
|
5f5f39a4aa | ||
|
|
df54cc04af | ||
|
|
0439616480 | ||
|
|
19fa274227 | ||
|
|
8076000c27 | ||
|
|
c80b30f070 | ||
|
|
486d5c48b0 | ||
|
|
4822792ee2 | ||
|
|
569f1d42b1 | ||
|
|
23c10faff5 | ||
|
|
1eaa195363 | ||
|
|
fb57cfa5d8 | ||
|
|
d33086c8f7 | ||
|
|
d815a6f02c | ||
|
|
8216f4a873 | ||
|
|
e4cc4521d9 | ||
|
|
6bd9b3744d | ||
|
|
f741b00768 | ||
|
|
5eb95d7dd4 | ||
|
|
e5268f43e7 | ||
|
|
54d6fb9da4 | ||
|
|
3d5c9cc1c2 | ||
|
|
442326f1d8 | ||
|
|
d66f46e07b | ||
|
|
757b53443d | ||
|
|
3436965b33 | ||
|
|
df71132957 | ||
|
|
1b322dc404 | ||
|
|
58341f4ff1 | ||
|
|
0d3ca80008 | ||
|
|
63437712cd | ||
|
|
26d0e87f46 | ||
|
|
2cad4fa1ce | ||
|
|
7bb293e5d6 | ||
|
|
e4777f9314 | ||
|
|
3508f562a7 | ||
|
|
1aa66c6038 | ||
|
|
e7458edb72 | ||
|
|
7f97013703 | ||
|
|
9e43060d41 | ||
|
|
d69486fb6e | ||
|
|
d4ebfdbc3c | ||
|
|
e00c3db71a | ||
|
|
11c3ea0b87 | ||
|
|
7531401623 | ||
|
|
e6c1dc251e | ||
|
|
dca7977051 | ||
|
|
d19e07d661 | ||
|
|
751ff6e21f | ||
|
|
3f6fe995b8 | ||
|
|
1e00fb369d | ||
|
|
54b522383a | ||
|
|
90a7de3b5c | ||
|
|
3fe1582432 | ||
|
|
85eddd2100 | ||
|
|
f5f8775c59 | ||
|
|
0ca98678f7 | ||
|
|
a19060c08d | ||
|
|
fa2ad88cc4 | ||
|
|
63cbcd0956 | ||
|
|
d6d0ebf8f4 | ||
|
|
0d810d92ca | ||
|
|
1ff914a6f4 | ||
|
|
5959b1be72 | ||
|
|
d12a214c05 | ||
|
|
3a83052f2e | ||
|
|
510b44ca92 | ||
|
|
15edb6756d | ||
|
|
fbfd02b08b | ||
|
|
b39c26fc86 | ||
|
|
95b2c8d175 | ||
|
|
d52748b09f | ||
|
|
34d18a3a9a | ||
|
|
3b27d6a9b5 | ||
|
|
703c391f99 | ||
|
|
4f1dc29df1 | ||
|
|
13667df374 | ||
|
|
8800d6985f | ||
|
|
364b8f2605 | ||
|
|
67b9ea9deb | ||
|
|
b78f2336a7 | ||
|
|
c7ba637c7d | ||
|
|
23a5ce3df7 | ||
|
|
8f88e28e50 | ||
|
|
9cf6139557 | ||
|
|
d556065a8b | ||
|
|
951716f7dc | ||
|
|
1ddc7ddda3 | ||
|
|
903ed9f3dc | ||
|
|
c42b76dcb8 | ||
|
|
a73582d9ae | ||
|
|
42c4fc7557 | ||
|
|
ddbbb6f1dd | ||
|
|
ff21a92330 | ||
|
|
07f76f7ad1 | ||
|
|
c90ccffd7b | ||
|
|
a00d5f18af | ||
|
|
1e391d211b | ||
|
|
d87f9672fa | ||
|
|
2b5838aa01 | ||
|
|
e10486d6ec | ||
|
|
1a74d6604d | ||
|
|
6d118536b6 | ||
|
|
ca4d758db9 | ||
|
|
dc18c26aa4 | ||
|
|
48505c2968 | ||
|
|
a98ea1e66a | ||
|
|
3dec697816 | ||
|
|
88fd41e597 | ||
|
|
b05d071a1c | ||
|
|
a27d3b9689 | ||
|
|
1facc0cd01 | ||
|
|
837f91d830 | ||
|
|
9c5f5aefb0 | ||
|
|
6b8d4a444b | ||
|
|
6bef09a3b1 | ||
|
|
e35319e5a2 | ||
|
|
0e548b3812 | ||
|
|
bfac02ccab | ||
|
|
7ea1a2b361 | ||
|
|
99df418f1d | ||
|
|
6416d8ce9c | ||
|
|
22b43a2b01 | ||
|
|
05e5d24c5e | ||
|
|
eabcc30367 | ||
|
|
f5e0ef5223 | ||
|
|
f46d9330b0 | ||
|
|
b62a0b4607 | ||
|
|
1f044321fb | ||
|
|
a841d49483 | ||
|
|
9509acc490 | ||
|
|
02d356ef12 | ||
|
|
d3516f299e | ||
|
|
79630767c2 | ||
|
|
084a76d075 | ||
|
|
bc6822e397 | ||
|
|
43432a9e48 | ||
|
|
d64a5bc12f | ||
|
|
b2922d18e2 | ||
|
|
ccf03fc07b | ||
|
|
a7c45da10c | ||
|
|
e03f01e24a | ||
|
|
0939589557 | ||
|
|
8167af9b4a | ||
|
|
4cf76123e5 | ||
|
|
01ee4b23e6 | ||
|
|
b198f79214 | ||
|
|
09db868a28 | ||
|
|
33e8ef75ff | ||
|
|
11dcb16b14 | ||
|
|
86f21da28b | ||
|
|
89cd6a9aa4 | ||
|
|
18e1256037 | ||
|
|
02cf478d91 | ||
|
|
6ec70192fe | ||
|
|
8c75098a9a | ||
|
|
72500f6948 | ||
|
|
37ec9ab464 | ||
|
|
82fe2a4c8d | ||
|
|
aa50e6ee66 | ||
|
|
91a07cfaee | ||
|
|
709f5e9a65 | ||
|
|
b2f9ef21cc | ||
|
|
be6b72edcd | ||
|
|
ece2d1e78a | ||
|
|
1ee1a5f2a1 | ||
|
|
a567326853 | ||
|
|
6231861dd6 | ||
|
|
1ff7b77ee0 | ||
|
|
9365708bb2 | ||
|
|
d23a0a8589 | ||
|
|
701b39b043 | ||
|
|
58ad1f3876 | ||
|
|
2138e7ea33 | ||
|
|
32f8c9e59f | ||
|
|
57028eab39 | ||
|
|
3a16edd8a6 | ||
|
|
165f3bb270 | ||
|
|
0ba75153f3 | ||
|
|
db2789990f | ||
|
|
acaf299bcb | ||
|
|
1940301824 | ||
|
|
34576e880d | ||
|
|
65c0668d40 | ||
|
|
53bd2bcbfe | ||
|
|
388724fccb | ||
|
|
231eabb013 | ||
|
|
54903fc2ea | ||
|
|
3a1baf0700 | ||
|
|
0c0e36b6f8 | ||
|
|
234c03db09 | ||
|
|
59db5e7889 | ||
|
|
28aa7da349 | ||
|
|
c51e344b87 | ||
|
|
54461dfa75 | ||
|
|
2d48e93f74 | ||
|
|
af22646322 | ||
|
|
722b42a93e | ||
|
|
8f9e7f77a7 | ||
|
|
09bb1ba494 | ||
|
|
d4137428ff | ||
|
|
b4d6c4f5b7 | ||
|
|
ffbe59ece5 | ||
|
|
fab9c90ccb | ||
|
|
fb1a774bc4 | ||
|
|
98bc7d1e0e | ||
|
|
f7622f24b2 | ||
|
|
f0a195a6d4 | ||
|
|
180ba27d84 | ||
|
|
f944671f86 | ||
|
|
def2903f7d | ||
|
|
0273a4e839 | ||
|
|
f8d2f02c5d | ||
|
|
25147d8897 | ||
|
|
0fb6f05fba | ||
|
|
4e4e899356 | ||
|
|
5a01dbf269 | ||
|
|
30b923b283 | ||
|
|
73ba381d20 | ||
|
|
1a5912877e | ||
|
|
813e506b68 | ||
|
|
077ca987f7 | ||
|
|
c632a7a6a5 | ||
|
|
e33e767510 | ||
|
|
ac82617aa9 | ||
|
|
a35dfd1fd1 | ||
|
|
c28aae9913 | ||
|
|
c26a99e65c | ||
|
|
ca57dcfc2f | ||
|
|
df5662dd69 | ||
|
|
8927a4889e | ||
|
|
1ac7831f3c | ||
|
|
292d272a94 | ||
|
|
a6ee8dc66e | ||
|
|
496f89f184 | ||
|
|
7a56eff1ac | ||
|
|
07e182aa16 | ||
|
|
7de06aa1e0 | ||
|
|
3955b64405 | ||
|
|
2bb55d681d | ||
|
|
f94e6ac527 | ||
|
|
b344f17b86 | ||
|
|
677b8cb633 | ||
|
|
6f3342e09e | ||
|
|
a1ddd762e0 | ||
|
|
68474e4057 | ||
|
|
a84b9ee396 | ||
|
|
b9c2ee745a | ||
|
|
c91a47fcaa | ||
|
|
615e489d8d | ||
|
|
c68f9f6f16 | ||
|
|
229cb85a6a | ||
|
|
e5c22fa665 | ||
|
|
8bcfff05d7 | ||
|
|
6416ee8151 | ||
|
|
f8eceb48e6 | ||
|
|
310c483bfa | ||
|
|
a8f20361aa | ||
|
|
290be69d99 | ||
|
|
3b96bd7ea0 | ||
|
|
dc2f22f5fa | ||
|
|
821be29f41 | ||
|
|
52ff1a12ff | ||
|
|
814699ef11 | ||
|
|
0c30838b25 | ||
|
|
cf66c2a1ee | ||
|
|
2ee419ffca | ||
|
|
bfb9d696d7 | ||
|
|
bb2a34dd6b | ||
|
|
ed652c0c56 | ||
|
|
1dc961d6eb | ||
|
|
d119fcfc98 | ||
|
|
4d3573724a | ||
|
|
8b37a66075 | ||
|
|
ba4f32075a | ||
|
|
218be22576 | ||
|
|
7688293716 | ||
|
|
458f8533c4 | ||
|
|
34502752fc | ||
|
|
d6758fd823 | ||
|
|
65700e790e | ||
|
|
7c34e4bb96 | ||
|
|
d0d6e3563b | ||
|
|
a2619f8c78 | ||
|
|
42d07fd2f0 | ||
|
|
8bea10960f | ||
|
|
9cbb19c304 | ||
|
|
1b94dfd712 | ||
|
|
9f3604d739 | ||
|
|
4a1b2be269 | ||
|
|
962dc1b55b | ||
|
|
07c86502f6 | ||
|
|
adb188e5d0 | ||
|
|
ce031dc6b8 | ||
|
|
18b5f03247 | ||
|
|
8a555ecf1c | ||
|
|
1b325b9acd | ||
|
|
1bdaddb319 | ||
|
|
7896e177ef | ||
|
|
ce8e659008 | ||
|
|
27be5deeb2 | ||
|
|
515f270c3a | ||
|
|
ffff3bd334 | ||
|
|
f493f13b25 | ||
|
|
e605c14b13 | ||
|
|
338488f16d | ||
|
|
2abc67c3e8 | ||
|
|
eb1ba143ec | ||
|
|
6f5bca0f67 | ||
|
|
407cd8dd4b | ||
|
|
62a4f0fc04 | ||
|
|
77cde411f1 | ||
|
|
3eb9d23108 | ||
|
|
410d4aeb21 | ||
|
|
0a28d216fd | ||
|
|
b69faf6920 | ||
|
|
efb92ea37a | ||
|
|
e77f9981df | ||
|
|
d27c2cc1e9 | ||
|
|
586b19675e | ||
|
|
f2907536b4 | ||
|
|
4aa4e35d1c | ||
|
|
9a11ac06bf | ||
|
|
aa3b18f848 | ||
|
|
103bdc151f | ||
|
|
6d4c1cd879 | ||
|
|
cacbe30871 | ||
|
|
bfeeacb230 | ||
|
|
04bb7b4919 | ||
|
|
b7df277a5c | ||
|
|
c681041b48 | ||
|
|
923834c784 | ||
|
|
588edf98be | ||
|
|
28c603ad5f | ||
|
|
6988a47e02 | ||
|
|
2c8ceb1217 | ||
|
|
ccac4ffa24 | ||
|
|
4258cef9bd | ||
|
|
62cc6dfe76 | ||
|
|
9f224a971b | ||
|
|
cf5dba9157 | ||
|
|
23035b9aa0 | ||
|
|
84908ec8ec | ||
|
|
dade49743b | ||
|
|
f29bf35c2a | ||
|
|
dfa6701c43 | ||
|
|
763ca69a73 | ||
|
|
6bf3b152bf | ||
|
|
aa19f85996 | ||
|
|
156d89567e | ||
|
|
ecc71baf61 | ||
|
|
90c743d963 | ||
|
|
b926293fa7 | ||
|
|
71a19191f8 | ||
|
|
38a0f20a33 | ||
|
|
c35192108c | ||
|
|
245b564f13 | ||
|
|
0d8d1ea4f3 | ||
|
|
27a427a363 | ||
|
|
2ff028a694 | ||
|
|
c211338218 | ||
|
|
8ac89af8bd | ||
|
|
bbbaf59591 | ||
|
|
169419896f | ||
|
|
0543dca502 | ||
|
|
cc6011d57a | ||
|
|
fc4407ef7e | ||
|
|
03735a125f | ||
|
|
5baeda9ff1 | ||
|
|
9b9794b5e0 | ||
|
|
0697d60a48 | ||
|
|
cfe6c82a31 | ||
|
|
3e30228d95 | ||
|
|
7264b53e5f | ||
|
|
60836d8523 | ||
|
|
ef89c2e47a | ||
|
|
2d9e3e1847 | ||
|
|
30136a9697 | ||
|
|
db7ccd66d3 | ||
|
|
cfe6483102 | ||
|
|
561566e723 | ||
|
|
c2dcc4c898 | ||
|
|
d09bfdc4ff | ||
|
|
358ef4536f | ||
|
|
5061a35e66 | ||
|
|
cd9a1e8c9e | ||
|
|
646902e75e | ||
|
|
40d26cb868 | ||
|
|
b64aa51c0c | ||
|
|
8206441834 | ||
|
|
d713783736 | ||
|
|
57dffaa2ce | ||
|
|
9e81dd2360 | ||
|
|
e2798969d7 | ||
|
|
1c31ec66f2 | ||
|
|
241f9fc7b0 | ||
|
|
270192486a | ||
|
|
a799503c97 | ||
|
|
9685928087 | ||
|
|
0e4b2fad99 | ||
|
|
3c4571a4e0 | ||
|
|
046147eb1d | ||
|
|
7834520e54 | ||
|
|
8e5b4d4b6f | ||
|
|
4544a074d9 | ||
|
|
9b78501392 | ||
|
|
f59ddcc88d | ||
|
|
a4955a2b79 | ||
|
|
92ae1a565b | ||
|
|
15a56ca25e | ||
|
|
9dcaa829ea | ||
|
|
9f65799a3d | ||
|
|
886587848b | ||
|
|
a97fc6dba8 | ||
|
|
c124e88d12 | ||
|
|
17f3870296 | ||
|
|
4626d42d08 | ||
|
|
e1e760055c | ||
|
|
45bf6c3bf3 | ||
|
|
fef0cc764d | ||
|
|
72049afcf6 | ||
|
|
d26c06dbf3 | ||
|
|
268decd655 | ||
|
|
7ae246c839 | ||
|
|
c7c454e4fb | ||
|
|
8e27297a81 | ||
|
|
2cdec72985 | ||
|
|
0085ac534d | ||
|
|
7828a79a96 | ||
|
|
5576c21e67 | ||
|
|
e49cfb1d2b | ||
|
|
1e541d0225 | ||
|
|
0974afd26d | ||
|
|
8d93594771 | ||
|
|
1136ac70e8 | ||
|
|
dc8d5a39ea | ||
|
|
8329e649b0 | ||
|
|
66da8b164f | ||
|
|
ea48577864 | ||
|
|
597146b136 | ||
|
|
30dd0c1e11 | ||
|
|
88772c4266 | ||
|
|
dc1d9e1c84 | ||
|
|
69ea65835d | ||
|
|
d5bae3a8c6 | ||
|
|
f14010bd5b | ||
|
|
87094fc83f | ||
|
|
7c179cfeab | ||
|
|
7582c221d1 | ||
|
|
c109895848 | ||
|
|
eccedada40 | ||
|
|
25d54accf8 | ||
|
|
d07685f0e9 | ||
|
|
2445c00c7e | ||
|
|
4c1d3ef514 | ||
|
|
4614c7d4c2 | ||
|
|
bbf1ef0dc3 | ||
|
|
3433c9e708 | ||
|
|
2cd5d75a2e | ||
|
|
2535b8adef | ||
|
|
4edab7bb7f | ||
|
|
fd8658e317 | ||
|
|
51d21d8c86 | ||
|
|
b4c3307cdf | ||
|
|
4e8d10cb44 | ||
|
|
e96875a425 | ||
|
|
5ab0035348 | ||
|
|
4ddff96b1e | ||
|
|
a08d84c1df | ||
|
|
21c71bfac1 | ||
|
|
6baaed3581 | ||
|
|
152dbfd5d1 | ||
|
|
a56d14086b | ||
|
|
aee87693f8 | ||
|
|
976b4affd9 | ||
|
|
e222b6ad9c | ||
|
|
19b17374e8 | ||
|
|
43989122bb | ||
|
|
72712d6047 | ||
|
|
0b52d2cc15 | ||
|
|
8304102136 | ||
|
|
3381aefcfa | ||
|
|
279a365cb1 | ||
|
|
2c9e00da56 | ||
|
|
f7cae69704 | ||
|
|
b7d58bcdbc | ||
|
|
13a856b843 | ||
|
|
8da38985c3 | ||
|
|
60cf6c6b97 | ||
|
|
35c2b34564 | ||
|
|
ef2e048efc | ||
|
|
6b3261aa33 | ||
|
|
1849c02cb6 | ||
|
|
1ec74a89e2 | ||
|
|
c591792de9 | ||
|
|
3108543ae5 | ||
|
|
1eb221c743 | ||
|
|
bebf6bc2e7 | ||
|
|
9e91cc2138 | ||
|
|
c5b939cfb7 | ||
|
|
5bd411ca27 | ||
|
|
a533cda6f0 | ||
|
|
fe4b07b8ae | ||
|
|
f9f2ccd904 | ||
|
|
d9e87d7c32 | ||
|
|
a0092c0770 | ||
|
|
3100131125 | ||
|
|
988880cf83 | ||
|
|
c3fb9672c4 | ||
|
|
0a2d94e425 | ||
|
|
8d9073cd31 | ||
|
|
d075961ffa | ||
|
|
7a72409b61 | ||
|
|
34fc530fba | ||
|
|
f257ff2f97 | ||
|
|
7ad5822c5b | ||
|
|
9a8f9f0a94 | ||
|
|
6421cecafb | ||
|
|
be544d6d89 | ||
|
|
3c89ecafdd | ||
|
|
35ec4eec52 | ||
|
|
e47f737a2f | ||
|
|
ac671a065b | ||
|
|
74116cc550 | ||
|
|
406070a5c3 | ||
|
|
0ccafd5b53 | ||
|
|
940f517aa3 | ||
|
|
216e5f65ad | ||
|
|
a74685d66d | ||
|
|
b7791d2845 | ||
|
|
d151a82d78 | ||
|
|
8ce61fbd52 | ||
|
|
90c24aade3 | ||
|
|
6b3f787fee | ||
|
|
4ebe4ce1b7 | ||
|
|
8c79740ee8 | ||
|
|
59d027ca02 | ||
|
|
37a7345a90 | ||
|
|
c519d4651b | ||
|
|
9b3b609e40 | ||
|
|
6254f53716 | ||
|
|
f05dc46432 | ||
|
|
3de0982a4a | ||
|
|
c2184fb3bf | ||
|
|
919c09fcb0 | ||
|
|
1d9dbd40ec | ||
|
|
0cd953a6f3 | ||
|
|
4db2b72351 | ||
|
|
dd54fcbdbd | ||
|
|
3123cf7ac6 | ||
|
|
6b579dd4ce | ||
|
|
16dfaa3e27 | ||
|
|
d7842b9f84 | ||
|
|
115034fccb | ||
|
|
309e957a85 | ||
|
|
d7007e402e | ||
|
|
91323a21cf | ||
|
|
fea893d76c | ||
|
|
761bc6ba4c | ||
|
|
75172feb4e | ||
|
|
3285fb1608 | ||
|
|
03a4c6910d | ||
|
|
485b958599 | ||
|
|
da47ba2f67 | ||
|
|
c39195488a | ||
|
|
227fb0ae9b | ||
|
|
b12ff5b503 | ||
|
|
0946c72b88 | ||
|
|
7d49b046d4 | ||
|
|
5f0426c840 | ||
|
|
73e239cc5f | ||
|
|
ad670f721a | ||
|
|
028a4a70cf | ||
|
|
77d7960347 | ||
|
|
39821146bd | ||
|
|
7d505a41ac | ||
|
|
e457b2f0d6 | ||
|
|
c9cf7fd4d4 | ||
|
|
b0371dd33d | ||
|
|
25e16c3565 | ||
|
|
7b39527863 | ||
|
|
d861b08866 | ||
|
|
fb438dc108 | ||
|
|
4e6b4f179b | ||
|
|
00d038c8f3 | ||
|
|
a9f6a68952 | ||
|
|
b9142bbc5a | ||
|
|
6c812f663e | ||
|
|
a93ec9783a | ||
|
|
2d184d77b6 | ||
|
|
bce299ccc7 | ||
|
|
235cebd14a | ||
|
|
a638aa9d53 | ||
|
|
67cce0ef7e | ||
|
|
82f4267bf6 | ||
|
|
45a9ca29c4 | ||
|
|
7f4e813277 | ||
|
|
3805ff4a0c | ||
|
|
464cfd475e | ||
|
|
fe469ae57f | ||
|
|
550ef9a1c4 | ||
|
|
935adfb51a | ||
|
|
3974df4a62 | ||
|
|
4870974161 | ||
|
|
8c4b0037f5 | ||
|
|
2c6f763ef2 | ||
|
|
ca28de02d8 | ||
|
|
bfc15ea029 | ||
|
|
6e8b8a5920 | ||
|
|
099f3b6a62 | ||
|
|
142d182bc1 | ||
|
|
1437871d88 | ||
|
|
352bf69409 | ||
|
|
9bdf3d23e1 | ||
|
|
be8ecfa707 | ||
|
|
51da0d0259 | ||
|
|
f55b78a994 | ||
|
|
e1a44c93f8 | ||
|
|
07e7087a09 | ||
|
|
2c79c7e2f6 | ||
|
|
09f6637fe0 | ||
|
|
3784db3308 | ||
|
|
2b950ff5dd | ||
|
|
09339c9cfb | ||
|
|
ccadd88af5 | ||
|
|
cc02a0efc2 | ||
|
|
43a1385b79 | ||
|
|
5101464e3b | ||
|
|
3d71478d38 | ||
|
|
4989ed445e | ||
|
|
d9413039ec | ||
|
|
eba0c9be34 | ||
|
|
48c9e9f3cc | ||
|
|
81ebde88db | ||
|
|
79ced9d0f8 | ||
|
|
a4058b84ce | ||
|
|
7bf211a52b | ||
|
|
d5f722792f | ||
|
|
0f02906c9b | ||
|
|
9582e228b1 | ||
|
|
45f20431f9 | ||
|
|
7554e6d7f9 | ||
|
|
cb8f26f177 | ||
|
|
b5dfce7861 | ||
|
|
2ca5a65544 | ||
|
|
17deb136db | ||
|
|
8c9710c76c | ||
|
|
32f7ecb261 | ||
|
|
fb77fde710 | ||
|
|
3c67bb90d7 | ||
|
|
dabb168853 | ||
|
|
45e5b3b219 | ||
|
|
a6b7469923 | ||
|
|
cb5dab3033 | ||
|
|
21d0038ff2 | ||
|
|
c094d8f2e8 | ||
|
|
c465d6a6c2 | ||
|
|
73d35bc985 | ||
|
|
2a8ccb065b | ||
|
|
8f04a50ce1 | ||
|
|
888aa5586b | ||
|
|
99f56f5d22 | ||
|
|
ad6281090d | ||
|
|
f0d334d3e2 | ||
|
|
5f829b048f | ||
|
|
1a961e66ff | ||
|
|
fdb0e22656 | ||
|
|
132ee1915f | ||
|
|
44bf4f3c8f | ||
|
|
6237767d5a | ||
|
|
dec9d96417 | ||
|
|
b167c87267 | ||
|
|
2280fe8e8e | ||
|
|
575d6dcd2d | ||
|
|
f729490c6b | ||
|
|
b32124cdd6 | ||
|
|
3d4321ee38 | ||
|
|
85034b382e | ||
|
|
77a51d1ad4 | ||
|
|
33e0cdc2d7 | ||
|
|
6519faa2fe | ||
|
|
5e3a234cbe | ||
|
|
e54c31d2d5 | ||
|
|
66c0537251 | ||
|
|
ac58516593 | ||
|
|
c3da6322b5 | ||
|
|
3d241500cf | ||
|
|
ded8224f66 | ||
|
|
f8814881a1 | ||
|
|
cc2852cd48 | ||
|
|
467637a9eb | ||
|
|
3cfc292d84 | ||
|
|
6acf94a810 | ||
|
|
31367fb4c4 | ||
|
|
12d6074e3b | ||
|
|
ff30386051 | ||
|
|
601f99ac16 | ||
|
|
87fe5c6101 | ||
|
|
68399ca31c | ||
|
|
2a6d7fd80f | ||
|
|
4725f510d8 | ||
|
|
be0ba22222 | ||
|
|
c8781392be | ||
|
|
b97164fcfb | ||
|
|
0dfb92281b | ||
|
|
4fe80c40da | ||
|
|
f0fac5115a | ||
|
|
46dd389d0d | ||
|
|
1e28e21ab5 | ||
|
|
7832c62c5d | ||
|
|
d025ee9dbe | ||
|
|
a9a9cb4319 | ||
|
|
aa727cb9b1 | ||
|
|
b8c9a99f20 | ||
|
|
aff995b0d0 | ||
|
|
2cc7e5dfdc | ||
|
|
5235a150b1 | ||
|
|
c6372ea9de | ||
|
|
7df4cc44c4 | ||
|
|
d47cf40544 | ||
|
|
7f5d88e95c | ||
|
|
d09663c066 | ||
|
|
ef97c9b69f | ||
|
|
d855e6c8b1 | ||
|
|
cd66f7eb43 | ||
|
|
6a35a7ba4c | ||
|
|
a3e146dc68 | ||
|
|
b81305a4a9 | ||
|
|
73884b34bc | ||
|
|
6166a34db2 | ||
|
|
6fa7da4b1c | ||
|
|
c3e426c491 | ||
|
|
21e023f0db | ||
|
|
063be001b3 | ||
|
|
5dff02e8bc | ||
|
|
60a59407d8 | ||
|
|
20a5aecfca | ||
|
|
c2e7b5a67d | ||
|
|
8f32303d07 | ||
|
|
891b1e7782 | ||
|
|
f26394fd3b | ||
|
|
4d83d42b4c | ||
|
|
57f1108df2 | ||
|
|
2641a9abe5 | ||
|
|
6b193ab350 | ||
|
|
b1bb37511c | ||
|
|
319187d6d6 | ||
|
|
02eb789f84 | ||
|
|
5a9338a27f | ||
|
|
eb6924277f | ||
|
|
325419404d | ||
|
|
bd8f371fd5 | ||
|
|
1783ff2845 | ||
|
|
d388527ffa | ||
|
|
19494088bd | ||
|
|
920dad524a | ||
|
|
ec89bcac8e | ||
|
|
a916c1f4ad | ||
|
|
a9a0ac92d7 | ||
|
|
da8a8bd1ef | ||
|
|
d9c746891d | ||
|
|
67817005b5 | ||
|
|
24d11de5a7 | ||
|
|
9251c87323 | ||
|
|
e12fab90d1 | ||
|
|
0a194b5b01 | ||
|
|
8d028adc53 | ||
|
|
dfca15395e | ||
|
|
e21f2362fe | ||
|
|
1ce328e8a9 | ||
|
|
038a5f999f | ||
|
|
5d3704c7ea | ||
|
|
87037c06c9 | ||
|
|
dd412c0f50 | ||
|
|
bf44befff6 | ||
|
|
e61874bb6f | ||
|
|
1e5331768f | ||
|
|
ec9a3a4f7c | ||
|
|
e439a3a8dc | ||
|
|
19f70d7a11 | ||
|
|
afe7ed5b05 | ||
|
|
d4bf004d74 | ||
|
|
e4d06a088b | ||
|
|
0929088b12 | ||
|
|
7b4838fc9b | ||
|
|
0cf9533248 | ||
|
|
84ff0b8a9f | ||
|
|
d467dcfeaf | ||
|
|
8e68ba4751 | ||
|
|
0f2a85ba9f | ||
|
|
7674a0a91e | ||
|
|
5bc1a66572 | ||
|
|
9b56067213 | ||
|
|
9a9df2fc3c | ||
|
|
9989d8d1d4 | ||
|
|
f9471f297e | ||
|
|
146b693e4a | ||
|
|
7295b7e329 | ||
|
|
e2441ea3e7 | ||
|
|
119e51912e | ||
|
|
dd950f5b0d | ||
|
|
78a9bad1e1 | ||
|
|
0c6eaf5484 | ||
|
|
1010068ddb | ||
|
|
82eec3d8d7 | ||
|
|
ee7b37d3f3 | ||
|
|
143d82d242 | ||
|
|
8b91b38855 | ||
|
|
1098f0d2a3 | ||
|
|
ab53cec022 | ||
|
|
6f5f8e5648 | ||
|
|
edfd707c22 | ||
|
|
1870f30af8 | ||
|
|
90106f5f08 | ||
|
|
9924b7b438 | ||
|
|
aa37faab0a | ||
|
|
dc10f8ce72 | ||
|
|
996686c1da | ||
|
|
488785d013 | ||
|
|
3abdc01230 | ||
|
|
8da04a584f | ||
|
|
27cc61d45e | ||
|
|
7371c30064 | ||
|
|
140d163895 | ||
|
|
dc33bdc1dc | ||
|
|
74df4fab83 | ||
|
|
1e5cd3d7a1 | ||
|
|
a54e9b64aa | ||
|
|
74660704e3 | ||
|
|
7439893a2a | ||
|
|
e27e49e9dc | ||
|
|
34ed729c59 | ||
|
|
adaeeca3fd | ||
|
|
dac75563d3 | ||
|
|
cbc76adcaa | ||
|
|
69a9cb383d | ||
|
|
4343073c00 | ||
|
|
fe60d4be88 | ||
|
|
ae337807f5 | ||
|
|
9ae30ac08e | ||
|
|
62fa85c0a4 | ||
|
|
7bb873dad9 | ||
|
|
5f6c1c14cb | ||
|
|
d43189ad33 | ||
|
|
fcad76fc51 | ||
|
|
97e6e1684e | ||
|
|
67a0d3e926 | ||
|
|
183fb9f9ff | ||
|
|
9815ddef1f | ||
|
|
f6d0847453 | ||
|
|
b0b9f0d65f | ||
|
|
0cec80f676 | ||
|
|
48c64143e3 | ||
|
|
a8712422bc | ||
|
|
97f65bd283 | ||
|
|
fd3c1c50f1 | ||
|
|
b153e4bb9f | ||
|
|
db9856a8db | ||
|
|
75ecea265d | ||
|
|
be8751cb73 | ||
|
|
fb25ecb4a1 | ||
|
|
f1cb7d27ac | ||
|
|
dee494e12f | ||
|
|
b13a121915 | ||
|
|
7486ee9537 | ||
|
|
4a20ccc28e | ||
|
|
f80dd2b307 | ||
|
|
b208cf6d32 | ||
|
|
39e78ff17e | ||
|
|
a8177ea7fe | ||
|
|
bedcfc154b | ||
|
|
4c38b4aa3c | ||
|
|
f6cfe266e0 | ||
|
|
4905e65f14 | ||
|
|
ccb250b410 | ||
|
|
aca57ffc62 | ||
|
|
7f375f42d8 | ||
|
|
eedcc2034d | ||
|
|
24c9a167d7 | ||
|
|
909df8ef1f | ||
|
|
3b27cb3671 | ||
|
|
3fe0db4a7d | ||
|
|
8b55814ab2 | ||
|
|
575e471553 | ||
|
|
0f5f1aebed | ||
|
|
50e17eb1ab | ||
|
|
158cc2f660 | ||
|
|
1066a31acd | ||
|
|
1f9d0f4582 | ||
|
|
a6d65233f1 | ||
|
|
eff2fe7a1b | ||
|
|
20efdc70b3 | ||
|
|
f0d8fb8f1a | ||
|
|
f7a380e9b7 | ||
|
|
e9c7cf6f63 | ||
|
|
68f1661452 | ||
|
|
36fd1b91ae | ||
|
|
a4ec430ac0 | ||
|
|
519614b2fd | ||
|
|
bf0118c8ef | ||
|
|
a4db0820bc | ||
|
|
ee7528413e | ||
|
|
7952fc8324 | ||
|
|
652773d2cf | ||
|
|
2a17787242 | ||
|
|
0a53ad5721 | ||
|
|
6da6bdc863 | ||
|
|
42ad2bb83f | ||
|
|
f309a65cb4 | ||
|
|
77e19ab1a4 | ||
|
|
1a996b6ef3 | ||
|
|
b882f1a010 | ||
|
|
82a030e6ff | ||
|
|
0758b85179 | ||
|
|
ab3d9bd080 | ||
|
|
8ff813f689 | ||
|
|
88ae72c0d3 | ||
|
|
312aa4be26 | ||
|
|
cbb06fce9d | ||
|
|
f259e497c4 | ||
|
|
dd4172ac66 | ||
|
|
66029e60d3 | ||
|
|
364f484f04 | ||
|
|
9dd5159414 | ||
|
|
13e38d6fd8 | ||
|
|
10dcb64715 | ||
|
|
7551b51e7d | ||
|
|
adb418aafc | ||
|
|
270da80d64 | ||
|
|
b2027cfd66 | ||
|
|
7f1f4eeac6 | ||
|
|
7a7446c8bd | ||
|
|
ddbae294e6 | ||
|
|
8c71b744f3 | ||
|
|
479b5d31a9 | ||
|
|
4cbf4230e8 | ||
|
|
6a610187e0 | ||
|
|
eb2a4aebba | ||
|
|
21a2e67755 | ||
|
|
3b9e312615 | ||
|
|
26dab04c9e | ||
|
|
00713c0d11 | ||
|
|
751b5f3027 | ||
|
|
e8261b000e | ||
|
|
41ecb70297 | ||
|
|
09ee104b8c | ||
|
|
e3a4964787 | ||
|
|
9bf72910a4 | ||
|
|
ee39e20e6d | ||
|
|
399d6db6f6 | ||
|
|
0821ce44b5 | ||
|
|
ea279111c6 | ||
|
|
674ce02e58 | ||
|
|
8dfa2767ec | ||
|
|
20dad7f07f | ||
|
|
751cc4c44d | ||
|
|
2318e6d8e9 | ||
|
|
61b4a492c3 | ||
|
|
9db3d01e09 | ||
|
|
8da73ad3dd | ||
|
|
b8c16d8ac5 | ||
|
|
429c0951f3 | ||
|
|
74e103c791 | ||
|
|
f941950ee2 | ||
|
|
846df2eef1 | ||
|
|
34ed058c97 | ||
|
|
eae0290978 | ||
|
|
561368570e | ||
|
|
3467d1fed0 | ||
|
|
d02ff232e5 | ||
|
|
2d1c6a5402 | ||
|
|
eab3b65629 | ||
|
|
20b435732a | ||
|
|
929617273d | ||
|
|
2717bf7d49 | ||
|
|
5cd2ebc960 | ||
|
|
9b4afe9816 | ||
|
|
23bb5598d5 | ||
|
|
af1d7813e9 | ||
|
|
16c2e5a585 | ||
|
|
c02750edbd | ||
|
|
7204ddafec | ||
|
|
faeba9a7e4 | ||
|
|
190d238a1f | ||
|
|
715451b5fb | ||
|
|
87f1895405 | ||
|
|
923d817751 | ||
|
|
0728209b66 | ||
|
|
b8b9dcc2ee | ||
|
|
f35e879852 | ||
|
|
34f4f12eb9 | ||
|
|
fa63bf758d | ||
|
|
f6b396ae64 | ||
|
|
2c7fd58e34 | ||
|
|
982f2c9634 | ||
|
|
f2fd42b47a | ||
|
|
1b4ccad938 | ||
|
|
a9de1ce8e0 | ||
|
|
ac752d5ec2 | ||
|
|
632d8d02d2 | ||
|
|
48aeb26e02 | ||
|
|
1694af8b5e | ||
|
|
83bcab9cd2 | ||
|
|
bdc7f4b3f5 | ||
|
|
39202a3d79 | ||
|
|
912065a121 | ||
|
|
c8466afac2 | ||
|
|
2619e162c1 | ||
|
|
e1112e17f8 | ||
|
|
92b2ead74c | ||
|
|
bbed9b94c1 | ||
|
|
73d07311db | ||
|
|
1cdff47477 | ||
|
|
511a5c3f82 | ||
|
|
853885e2ff | ||
|
|
d83936a66a | ||
|
|
5517d2bf56 | ||
|
|
f21ab49ac5 | ||
|
|
925a458abe | ||
|
|
76946c447f | ||
|
|
2faa29b1c4 | ||
|
|
6826cc311d | ||
|
|
5e17ce0a0b | ||
|
|
e8d299d3b6 | ||
|
|
7637aa2ab6 | ||
|
|
ab067d1d3a | ||
|
|
e6f84666c7 | ||
|
|
4c5429af15 | ||
|
|
0a0ac3b7c9 | ||
|
|
24833ce9fb | ||
|
|
ec2c18dc87 | ||
|
|
7384609e74 | ||
|
|
3047649650 | ||
|
|
d298dac3f3 | ||
|
|
1574bca8a8 | ||
|
|
ec2f6c6b80 | ||
|
|
838cc60161 | ||
|
|
310c61a5cc | ||
|
|
318cc15323 | ||
|
|
3a64ceb4d6 | ||
|
|
d0f21c0095 | ||
|
|
46dc15dd29 | ||
|
|
8dc654b513 | ||
|
|
7000ac3f3f | ||
|
|
43c2e8d8e9 | ||
|
|
0231139b01 | ||
|
|
d6ee6446dd | ||
|
|
7b666efcf8 | ||
|
|
eb5d2198fc | ||
|
|
34e44ebd1c | ||
|
|
bf2f4bc040 | ||
|
|
9dc4559aba | ||
|
|
eba8856261 | ||
|
|
a8c5aa471a | ||
|
|
52f6dcf092 | ||
|
|
dec79f3742 | ||
|
|
8bdcac0f3e | ||
|
|
8426b674a3 | ||
|
|
da391bcc8d | ||
|
|
2d7443d454 | ||
|
|
991987ed76 | ||
|
|
ec24ebf2cf | ||
|
|
6ba0976085 | ||
|
|
2b88d01a01 | ||
|
|
0c09f24cbf | ||
|
|
61d22afeba | ||
|
|
9f1ed6e8c3 | ||
|
|
bbc4113cac | ||
|
|
91194bf422 | ||
|
|
9c5f940b00 | ||
|
|
455b4043b8 | ||
|
|
bd83ee7931 | ||
|
|
f6bdf7c09a | ||
|
|
2db8afb8c2 | ||
|
|
e033129dd3 | ||
|
|
f9dc590100 | ||
|
|
8996aafe0d | ||
|
|
9dc25ef7af | ||
|
|
2b84f4d407 | ||
|
|
097c8b674c | ||
|
|
ba649d4b94 | ||
|
|
6ed1614db0 | ||
|
|
df5b6a8380 | ||
|
|
1f82a8b99e | ||
|
|
0c95d96f32 | ||
|
|
c2f5f84118 | ||
|
|
b3b5e3d8f0 | ||
|
|
506d3f3cd9 | ||
|
|
516a8c5ee5 | ||
|
|
2f81e9d374 | ||
|
|
2d8703bb8d | ||
|
|
76e60d9bc3 | ||
|
|
9d5370be5f | ||
|
|
fc1a06bc45 | ||
|
|
fce80374f4 | ||
|
|
420c9f10c2 | ||
|
|
5a39681a2e | ||
|
|
7a1b7db7c8 | ||
|
|
03a643da52 | ||
|
|
383f0ce450 | ||
|
|
c6c668676c | ||
|
|
7b01dde063 | ||
|
|
8c25f65024 | ||
|
|
1e478e3545 | ||
|
|
6c75a8978b | ||
|
|
640267fc2b | ||
|
|
33c7c3ee12 | ||
|
|
f9b41d34ae | ||
|
|
d4bec79451 | ||
|
|
ac1a8b4daf | ||
|
|
28838c1759 | ||
|
|
50ecb0dac9 | ||
|
|
e22bc01cbd | ||
|
|
6c28713a4c | ||
|
|
fc9023386c | ||
|
|
e6cae9bcc3 | ||
|
|
a9eeca1302 | ||
|
|
8c695e42ca | ||
|
|
0aa7fd47d5 | ||
|
|
70596042d6 | ||
|
|
caf616234b | ||
|
|
375187aa70 | ||
|
|
71eccdc0e3 | ||
|
|
639b1e48f5 | ||
|
|
0bb4cb4472 | ||
|
|
cc51543851 | ||
|
|
22540390e1 | ||
|
|
98565eb67c | ||
|
|
cf6a47ecb7 | ||
|
|
fa60b9f9d3 | ||
|
|
644120ca31 | ||
|
|
a50a625b3b | ||
|
|
7297c13331 | ||
|
|
f73399bfac | ||
|
|
a056cd78f7 | ||
|
|
a30f3c86c2 | ||
|
|
e70bdd86a7 | ||
|
|
bc9f33c2e0 | ||
|
|
872b89ee93 | ||
|
|
ae53062518 | ||
|
|
17f76c9cb3 | ||
|
|
5de944146a | ||
|
|
9dc6092cb0 | ||
|
|
a32a2ef04e | ||
|
|
ecfa0ae3da | ||
|
|
03595052ce | ||
|
|
1f94c53dd2 | ||
|
|
9c426373f2 | ||
|
|
c03e30a01f | ||
|
|
07f7a77ac0 | ||
|
|
3b9ea2c9a4 | ||
|
|
1beb13dd80 | ||
|
|
ddae84abb3 | ||
|
|
863b9a2c98 | ||
|
|
9d44bbdb48 | ||
|
|
8d93dd5adc | ||
|
|
48502961cf | ||
|
|
3c8bec61d3 | ||
|
|
7296c7df1a | ||
|
|
f3ee6603de | ||
|
|
ee0aabda1d | ||
|
|
08d37a4b0f | ||
|
|
f975ea99cb | ||
|
|
f030d41dc7 | ||
|
|
8d079bfcd1 | ||
|
|
d39b8654a6 | ||
|
|
ce7816a968 | ||
|
|
2ee572e68f | ||
|
|
4bbd850898 | ||
|
|
34eae6e608 | ||
|
|
6a0302fec6 | ||
|
|
c94cc293c2 | ||
|
|
cae7792a1e | ||
|
|
7f6b2fe4f1 | ||
|
|
eba430bbc0 | ||
|
|
01280c8d04 | ||
|
|
ec141ac9c9 | ||
|
|
590c892a6a | ||
|
|
ff8a50c366 | ||
|
|
46ef6c8ab7 | ||
|
|
b09eabc478 | ||
|
|
e49fcea6e3 | ||
|
|
68ed9f4ffc | ||
|
|
af94687d45 | ||
|
|
26964ecf0f | ||
|
|
77d19af359 | ||
|
|
120b82f243 | ||
|
|
a0fea30a11 | ||
|
|
8bb3e0a64d | ||
|
|
bbded12923 | ||
|
|
e8ba5d7606 | ||
|
|
af66b31a44 | ||
|
|
b000a40f28 | ||
|
|
3c85322523 | ||
|
|
a469b8bc04 | ||
|
|
78b8261a3a | ||
|
|
f20ca70c01 | ||
|
|
4e4148fc1c | ||
|
|
c22482f907 | ||
|
|
870e139fce | ||
|
|
4d58648c02 | ||
|
|
ebbb182537 | ||
|
|
1cd5377b45 | ||
|
|
1d1f0527ee | ||
|
|
37a5f77415 | ||
|
|
ced3c7efe4 | ||
|
|
c3b8f366ed | ||
|
|
de78876b1a | ||
|
|
64c25b049c | ||
|
|
8a4fe4f3ad | ||
|
|
8811b8c1fd | ||
|
|
190b01fdf9 | ||
|
|
f145d08c10 | ||
|
|
53382b7e15 | ||
|
|
6ad0242617 | ||
|
|
a7c2408c0a | ||
|
|
ce1eabaed6 | ||
|
|
f602541ede | ||
|
|
3f718e6efc | ||
|
|
ce7a985df6 | ||
|
|
6d83f7e7bd | ||
|
|
b73c00943c | ||
|
|
abaac8ef48 | ||
|
|
a2f8e7068e | ||
|
|
4d47873219 | ||
|
|
cf985486e5 | ||
|
|
b930c3fc93 | ||
|
|
dd26a96828 | ||
|
|
6865ddfc12 | ||
|
|
e888e69d4d | ||
|
|
2089059792 | ||
|
|
27739e0364 | ||
|
|
698ee271d6 | ||
|
|
543c75b293 | ||
|
|
b09c46f6f7 | ||
|
|
f2cc19e6aa | ||
|
|
814a0a123f | ||
|
|
179383540f | ||
|
|
8be1c8310d | ||
|
|
ef02d776ca | ||
|
|
750ff448ad | ||
|
|
e3abab6d4d | ||
|
|
d3ffae72fb | ||
|
|
87f751188e | ||
|
|
3469abaefd | ||
|
|
797364ee5c | ||
|
|
36c05fc4b9 | ||
|
|
79624febc0 | ||
|
|
0a9d4de126 | ||
|
|
b6f6994db4 | ||
|
|
ff7bed720a | ||
|
|
00c0f48b02 | ||
|
|
94c45cf2a0 | ||
|
|
58f77b2a1c | ||
|
|
7170e69b22 | ||
|
|
239ee2437c | ||
|
|
ced368db31 | ||
|
|
6a991e5c15 | ||
|
|
523d22262b | ||
|
|
08197a327e | ||
|
|
b7cb2a7aa5 | ||
|
|
decc5c74ef | ||
|
|
fbe0f886b6 | ||
|
|
7e23d6e2ef | ||
|
|
49458d1085 | ||
|
|
289c12bd9a | ||
|
|
9a6326b027 | ||
|
|
51f573f1ea | ||
|
|
c8b7cd8862 | ||
|
|
21c112d059 | ||
|
|
9432e1b5b2 | ||
|
|
8269d2f83c | ||
|
|
5a4b6be974 | ||
|
|
35e8ce60a9 | ||
|
|
8b6dd9f603 | ||
|
|
084f0ebdab | ||
|
|
58640c1521 | ||
|
|
7ffdfd12f8 | ||
|
|
cb9a30f285 | ||
|
|
e48bef809f | ||
|
|
f5d7570102 | ||
|
|
a600c60cf8 | ||
|
|
695eabd026 | ||
|
|
e81b51a647 | ||
|
|
b7e95ff090 | ||
|
|
3ca41be686 | ||
|
|
3152046173 | ||
|
|
3a98dc8a95 | ||
|
|
d737b28916 | ||
|
|
97c0dac876 | ||
|
|
006494b1fa | ||
|
|
149d343201 | ||
|
|
496cc79ba8 | ||
|
|
3591768745 | ||
|
|
d615f6761a | ||
|
|
5b29894048 | ||
|
|
d17f0cfa40 | ||
|
|
b2f70c7120 | ||
|
|
238707bd93 | ||
|
|
87cdf1e3a0 | ||
|
|
57fd47022e | ||
|
|
5eafd3bf6b | ||
|
|
640b5b0ea9 | ||
|
|
e4fb2f4680 | ||
|
|
a4909f54e4 | ||
|
|
f7065c6f0c | ||
|
|
64f7f837e7 | ||
|
|
130d36acd9 | ||
|
|
e94c28cfa2 | ||
|
|
a697dec30c | ||
|
|
bac09e9b9f | ||
|
|
b179f8f86a | ||
|
|
962d04ae17 | ||
|
|
f28e3bfe37 | ||
|
|
6474c86d32 | ||
|
|
769ea8cdfe | ||
|
|
052e77dd5a | ||
|
|
1a4b0cb12a | ||
|
|
f9aa95c987 | ||
|
|
a5d06fb4a4 | ||
|
|
d5e5d90bdc | ||
|
|
e62678e4e6 | ||
|
|
558ac24f7e | ||
|
|
6f22f6a59f | ||
|
|
5ec74f8abe | ||
|
|
16d7547e03 | ||
|
|
767112dcda | ||
|
|
81e23d1d8c | ||
|
|
ace90a4354 | ||
|
|
267e7096cc | ||
|
|
e7cded7511 | ||
|
|
33fbd715c0 | ||
|
|
25ba5b867c | ||
|
|
c8f431447c | ||
|
|
1b83a1d09a | ||
|
|
d2fb7a7151 | ||
|
|
7ad3447598 | ||
|
|
5e5bc8e705 | ||
|
|
886d1e8a19 | ||
|
|
6de7a035fa | ||
|
|
48d2497eb2 | ||
|
|
6494754ab9 | ||
|
|
a8153627c6 | ||
|
|
ed38966edb | ||
|
|
85551d1e54 | ||
|
|
ca31363180 | ||
|
|
a8f11eb3c3 | ||
|
|
151805121c | ||
|
|
0422d2a021 | ||
|
|
8586762dde | ||
|
|
ffdf70257b | ||
|
|
6079d60aa1 | ||
|
|
d46d012f8c | ||
|
|
baf926b360 | ||
|
|
82339869dc | ||
|
|
f70343bb63 | ||
|
|
4ab56ae3d1 | ||
|
|
39b4031684 | ||
|
|
71f8965393 | ||
|
|
66857e72a4 | ||
|
|
15eb5d47eb | ||
|
|
86242139da | ||
|
|
460bdc4148 | ||
|
|
1052126522 | ||
|
|
fdb42ac876 | ||
|
|
6486f986e8 | ||
|
|
bc4075e2ed | ||
|
|
11525c7d0d | ||
|
|
bb12ae9ce6 | ||
|
|
66b4ad0c44 | ||
|
|
4bff2d718e | ||
|
|
af7a7b4dc6 | ||
|
|
ec20d9a2a8 | ||
|
|
b0aee3d335 | ||
|
|
5e2ddbfd86 | ||
|
|
f086ebbb8e | ||
|
|
e5bf6a5bfc | ||
|
|
76fa86d54b | ||
|
|
60d89506a5 | ||
|
|
2893f1eb9e | ||
|
|
a08cbf412d | ||
|
|
0bd65356f9 | ||
|
|
2e2b39455f | ||
|
|
1f9fbe34e4 | ||
|
|
506d7ae50e | ||
|
|
342cb00625 | ||
|
|
952fc01efd | ||
|
|
3eebe301fe | ||
|
|
19c0a81c42 | ||
|
|
9fc7f9904b | ||
|
|
b04a516063 | ||
|
|
241e946d91 | ||
|
|
e45375dc26 | ||
|
|
ec8e243323 | ||
|
|
af0e9368d4 | ||
|
|
285483b81a | ||
|
|
73a91d5569 | ||
|
|
e24c78be32 | ||
|
|
b11184de68 | ||
|
|
61d02fc5d7 | ||
|
|
f0217f6821 | ||
|
|
09644914a6 | ||
|
|
b5c24d6a48 | ||
|
|
bdd2ac2c25 | ||
|
|
76376f0d33 | ||
|
|
9749da46ae | ||
|
|
1c05295e89 | ||
|
|
4d0f28215a | ||
|
|
0bba72bc5a | ||
|
|
87089b8e83 | ||
|
|
6a58148a89 | ||
|
|
15091052be | ||
|
|
5cd7e9a9b8 | ||
|
|
7cb530c334 | ||
|
|
5e0324cc91 | ||
|
|
6293e227ea | ||
|
|
d6d83a5c76 | ||
|
|
93fc883b90 | ||
|
|
dd21803598 | ||
|
|
af2f2282c2 | ||
|
|
869a76c9bb | ||
|
|
44b2964a6a | ||
|
|
8fddb57e0a | ||
|
|
c7a5a0cab0 | ||
|
|
c13aab3ffc | ||
|
|
95a9a76598 | ||
|
|
923cfa3d50 | ||
|
|
2c489168c2 | ||
|
|
f3292b4d34 | ||
|
|
9769829b72 | ||
|
|
36243d15cc | ||
|
|
f4645f570c | ||
|
|
61603ccfce | ||
|
|
a26cfc639c | ||
|
|
90602931d8 | ||
|
|
d1b330028c | ||
|
|
7a6b1930bf | ||
|
|
c271361552 | ||
|
|
17789bc814 | ||
|
|
d7c16e161a | ||
|
|
e71c17d7e7 | ||
|
|
17d87eb157 | ||
|
|
bf11bcc084 | ||
|
|
bd291109df | ||
|
|
ac89ba9b8d | ||
|
|
5ab634e375 | ||
|
|
bb1978d976 | ||
|
|
bea94ce8ac | ||
|
|
9561f93594 | ||
|
|
d44d5c3304 | ||
|
|
ec541e2057 | ||
|
|
8169bf6b97 | ||
|
|
56c8ad1221 | ||
|
|
6814f2e38c | ||
|
|
5f043b9a78 | ||
|
|
c9092cd1c7 | ||
|
|
86cc65d894 | ||
|
|
8ef2647fa9 | ||
|
|
3ff9e99416 | ||
|
|
1731046011 | ||
|
|
c3d96184b6 | ||
|
|
c255c606a7 | ||
|
|
cc69faa1fd | ||
|
|
0f53cd86c8 | ||
|
|
212c8f188d | ||
|
|
dec248adec | ||
|
|
98e6a066f4 | ||
|
|
c90db54a3d | ||
|
|
927d156933 | ||
|
|
c71f91073f | ||
|
|
ab28387692 |
No files matched your search
+151
-22
@@ -1,31 +1,74 @@
|
||||
name: ci
|
||||
on: pull_request
|
||||
on: ["push", "pull_request"]
|
||||
|
||||
jobs:
|
||||
|
||||
lint:
|
||||
name: lint
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- run: make install tools
|
||||
- name: extract pip cache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
|
||||
restore-keys: ${{ runner.os }}-pip-
|
||||
- run: pip install --user --upgrade pip wheel
|
||||
- run: pip install -e .[torrent,lint]
|
||||
- run: make lint
|
||||
|
||||
tests-unit:
|
||||
name: "tests / unit"
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
os:
|
||||
- ubuntu-latest
|
||||
- macos-latest
|
||||
- windows-latest
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- run: make install tools
|
||||
- working-directory: lbry
|
||||
- name: set pip cache dir
|
||||
id: pip-cache
|
||||
run: echo "::set-output name=dir::$(pip cache dir)"
|
||||
- name: extract pip cache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ${{ steps.pip-cache.outputs.dir }}
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
|
||||
restore-keys: ${{ runner.os }}-pip-
|
||||
- id: os-name
|
||||
uses: ASzc/change-string-case-action@v1
|
||||
with:
|
||||
string: ${{ runner.os }}
|
||||
- run: pip install --user --upgrade pip wheel
|
||||
- if: startsWith(runner.os, 'linux')
|
||||
run: pip install -e .[torrent,test]
|
||||
- if: startsWith(runner.os, 'linux')
|
||||
env:
|
||||
HOME: /tmp
|
||||
run: coverage run -p --source=lbry -m unittest discover -vv tests.unit
|
||||
run: make test-unit-coverage
|
||||
- if: startsWith(runner.os, 'linux') != true
|
||||
run: pip install -e .[test]
|
||||
- if: startsWith(runner.os, 'linux') != true
|
||||
env:
|
||||
HOME: /tmp
|
||||
run: coverage run --source=lbry -m unittest tests/unit/test_conf.py
|
||||
- name: submit coverage report
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
COVERALLS_FLAG_NAME: tests-unit-${{ steps.os-name.outputs.lowercase }}
|
||||
COVERALLS_PARALLEL: true
|
||||
run: |
|
||||
pip install coveralls
|
||||
coveralls --service=github
|
||||
|
||||
tests-integration:
|
||||
name: "tests / integration"
|
||||
@@ -35,45 +78,131 @@ jobs:
|
||||
test:
|
||||
- datanetwork
|
||||
- blockchain
|
||||
- claims
|
||||
- takeovers
|
||||
- transactions
|
||||
- claims_legacy_search
|
||||
- takeovers_legacy_search
|
||||
- other
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- name: Configure sysctl limits
|
||||
run: |
|
||||
sudo swapoff -a
|
||||
sudo sysctl -w vm.swappiness=1
|
||||
sudo sysctl -w fs.file-max=262144
|
||||
sudo sysctl -w vm.max_map_count=262144
|
||||
- name: Runs Elasticsearch
|
||||
uses: elastic/elastic-github-actions/elasticsearch@master
|
||||
with:
|
||||
stack-version: 7.12.1
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- if: matrix.test == 'other'
|
||||
run: sudo apt install -y --no-install-recommends ffmpeg
|
||||
- run: pip install tox-travis
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends ffmpeg
|
||||
- name: extract pip cache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ./.tox
|
||||
key: tox-integration-${{ matrix.test }}-${{ hashFiles('setup.py') }}
|
||||
restore-keys: txo-integration-${{ matrix.test }}-
|
||||
- run: pip install tox coverage coveralls
|
||||
- if: matrix.test == 'claims'
|
||||
run: rm -rf .tox
|
||||
- run: tox -e ${{ matrix.test }}
|
||||
- name: submit coverage report
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
COVERALLS_FLAG_NAME: tests-integration-${{ matrix.test }}
|
||||
COVERALLS_PARALLEL: true
|
||||
run: |
|
||||
coverage combine tests
|
||||
coveralls --service=github
|
||||
|
||||
|
||||
coverage:
|
||||
needs: ["tests-unit", "tests-integration"]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: finalize coverage report submission
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
pip install coveralls
|
||||
coveralls --service=github --finish
|
||||
|
||||
build:
|
||||
needs: ["lint", "tests-unit", "tests-integration"]
|
||||
name: "build"
|
||||
name: "build / binary"
|
||||
strategy:
|
||||
matrix:
|
||||
os:
|
||||
- ubuntu-latest
|
||||
- ubuntu-18.04
|
||||
- macos-latest
|
||||
- windows-latest
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- name: Setup
|
||||
run: |
|
||||
pip install pyinstaller
|
||||
pip install -e .
|
||||
- if: startsWith(matrix.os, 'windows') == false
|
||||
- id: os-name
|
||||
uses: ASzc/change-string-case-action@v1
|
||||
with:
|
||||
string: ${{ runner.os }}
|
||||
- name: set pip cache dir
|
||||
id: pip-cache
|
||||
run: echo "::set-output name=dir::$(pip cache dir)"
|
||||
- name: extract pip cache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ${{ steps.pip-cache.outputs.dir }}
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
|
||||
restore-keys: ${{ runner.os }}-pip-
|
||||
- run: pip install pyinstaller==4.4
|
||||
- run: pip install -e .
|
||||
- if: startsWith(github.ref, 'refs/tags/v')
|
||||
run: python docker/set_build.py
|
||||
- if: startsWith(runner.os, 'linux') || startsWith(runner.os, 'mac')
|
||||
name: Build & Run (Unix)
|
||||
run: |
|
||||
pyinstaller --onefile --name lbrynet lbry/extras/cli.py
|
||||
chmod +x dist/lbrynet
|
||||
dist/lbrynet --version
|
||||
- if: startsWith(matrix.os, 'windows')
|
||||
- if: startsWith(runner.os, 'windows')
|
||||
name: Build & Run (Windows)
|
||||
run: |
|
||||
pip install pywin32
|
||||
pip install pywin32==301
|
||||
pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico --onefile --name lbrynet lbry/extras/cli.py
|
||||
dist/lbrynet.exe --version
|
||||
- uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: lbrynet-${{ steps.os-name.outputs.lowercase }}
|
||||
path: dist/
|
||||
|
||||
release:
|
||||
name: "release"
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
needs: ["build"]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/download-artifact@v2
|
||||
- name: upload binaries
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.RELEASE_API_TOKEN }}
|
||||
run: |
|
||||
pip install githubrelease
|
||||
chmod +x lbrynet-macos/lbrynet
|
||||
chmod +x lbrynet-linux/lbrynet
|
||||
zip --junk-paths lbrynet-mac.zip lbrynet-macos/lbrynet
|
||||
zip --junk-paths lbrynet-linux.zip lbrynet-linux/lbrynet
|
||||
zip --junk-paths lbrynet-windows.zip lbrynet-windows/lbrynet.exe
|
||||
ls -lh
|
||||
githubrelease release lbryio/lbry-sdk info ${GITHUB_REF#refs/tags/}
|
||||
githubrelease asset lbryio/lbry-sdk upload ${GITHUB_REF#refs/tags/} \
|
||||
lbrynet-mac.zip lbrynet-linux.zip lbrynet-windows.zip
|
||||
githubrelease release lbryio/lbry-sdk publish ${GITHUB_REF#refs/tags/}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
name: slack
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
|
||||
jobs:
|
||||
release:
|
||||
name: "slack notification"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: LoveToKnow/slackify-markdown-action@v1.0.0
|
||||
id: markdown
|
||||
with:
|
||||
text: "There is a new SDK release: ${{github.event.release.html_url}}\n${{ github.event.release.body }}"
|
||||
- uses: slackapi/slack-github-action@v1.14.0
|
||||
env:
|
||||
CHANGELOG: '<!channel> ${{ steps.markdown.outputs.text }}'
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_RELEASE_BOT_WEBHOOK }}
|
||||
with:
|
||||
payload: '{"type": "mrkdwn", "text": ${{ toJSON(env.CHANGELOG) }} }'
|
||||
|
||||
+7
-2
@@ -6,12 +6,17 @@
|
||||
/.coverage*
|
||||
/lbry-venv
|
||||
/venv
|
||||
/lbry/blockchain
|
||||
|
||||
lbry.egg-info
|
||||
__pycache__
|
||||
_trial_temp/
|
||||
trending*.log
|
||||
|
||||
/tests/integration/blockchain/files
|
||||
/tests/integration/claims/files
|
||||
/tests/.coverage.*
|
||||
|
||||
/lbry/wallet/bin
|
||||
/lbry/wallet/bin
|
||||
|
||||
/.vscode
|
||||
/.gitignore
|
||||
-210
@@ -1,210 +0,0 @@
|
||||
default:
|
||||
image: python:3.7
|
||||
|
||||
|
||||
#cache:
|
||||
# directories:
|
||||
# - $HOME/venv
|
||||
# - $HOME/.cache/pip
|
||||
# - $HOME/Library/Caches/pip
|
||||
# - $HOME/Library/Caches/Homebrew
|
||||
# - $TRAVIS_BUILD_DIR/.tox
|
||||
|
||||
|
||||
stages:
|
||||
- test
|
||||
- build
|
||||
- assets
|
||||
- release
|
||||
|
||||
|
||||
.tagged:
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG =~ /^v[0-9\.]+$/'
|
||||
when: on_success
|
||||
|
||||
|
||||
|
||||
test:lint:
|
||||
stage: test
|
||||
script:
|
||||
- make install tools
|
||||
- make lint
|
||||
|
||||
test:unit:
|
||||
stage: test
|
||||
script:
|
||||
- make install tools
|
||||
- HOME=/tmp coverage run -p --source=lbry -m unittest discover -vv tests.unit
|
||||
|
||||
test:datanetwork-integration:
|
||||
stage: test
|
||||
script:
|
||||
- pip install tox-travis
|
||||
- tox -e datanetwork
|
||||
|
||||
test:blockchain-integration:
|
||||
stage: test
|
||||
script:
|
||||
- pip install tox-travis
|
||||
- tox -e blockchain
|
||||
|
||||
test:other-integration:
|
||||
stage: test
|
||||
script:
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends ffmpeg
|
||||
- pip install tox-travis
|
||||
- tox -e other
|
||||
|
||||
test:json-api:
|
||||
stage: test
|
||||
script:
|
||||
- make install tools
|
||||
- HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
|
||||
|
||||
|
||||
|
||||
.build:
|
||||
stage: build
|
||||
artifacts:
|
||||
expire_in: 1 day
|
||||
paths:
|
||||
- lbrynet-${OS}.zip
|
||||
script:
|
||||
- pip install --upgrade 'setuptools<45.0.0'
|
||||
- pip install pyinstaller
|
||||
- pip install -e .
|
||||
- python3.7 docker/set_build.py # must come after lbry is installed because it imports lbry
|
||||
- pyinstaller --onefile --name lbrynet lbry/extras/cli.py
|
||||
- chmod +x dist/lbrynet
|
||||
- zip --junk-paths ${CI_PROJECT_DIR}/lbrynet-${OS}.zip dist/lbrynet # gitlab expects artifacts to be in $CI_PROJECT_DIR
|
||||
- openssl dgst -sha256 ${CI_PROJECT_DIR}/lbrynet-${OS}.zip | egrep -o [0-9a-f]+$ # get sha256 of asset. works on mac and ubuntu
|
||||
- dist/lbrynet --version
|
||||
|
||||
build:linux:
|
||||
extends: .build
|
||||
image: ubuntu:16.04
|
||||
variables:
|
||||
OS: linux
|
||||
before_script:
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends software-properties-common zip curl build-essential
|
||||
- add-apt-repository -y ppa:deadsnakes/ppa
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends python3.7-dev
|
||||
- python3.7 <(curl -q https://bootstrap.pypa.io/get-pip.py) # make sure we get pip with python3.7
|
||||
|
||||
build:mac:
|
||||
extends: .build
|
||||
tags: [macos] # makes gitlab use the mac runner
|
||||
variables:
|
||||
OS: mac
|
||||
GIT_DEPTH: 5
|
||||
VENV: /tmp/gitlab-lbry-sdk-venv
|
||||
before_script:
|
||||
# - brew upgrade python || true
|
||||
- python3 --version | grep -q '^Python 3\.7\.' # dont upgrade python on every run. just make sure we're on the right Python
|
||||
# - pip3 install --user --upgrade pip virtualenv
|
||||
- pip3 --version | grep -q '\(python 3\.7\)'
|
||||
- virtualenv --python=python3.7 "${VENV}"
|
||||
- source "${VENV}/bin/activate"
|
||||
after_script:
|
||||
- rm -rf "${VENV}"
|
||||
|
||||
build:windows:
|
||||
extends: .build
|
||||
tags: [windows] # makes gitlab use the windows runner
|
||||
variables:
|
||||
OS: windows
|
||||
GIT_DEPTH: 5
|
||||
before_script:
|
||||
- ./docker/install_choco.ps1
|
||||
- choco install -y --x86 python3 7zip checksum
|
||||
- python --version # | findstr /B "Python 3\.7\." # dont upgrade python on every run. just make sure we're on the right Python
|
||||
- pip --version # | findstr /E '\(python 3\.7\)'
|
||||
- pip install virtualenv pywin32
|
||||
- virtualenv venv
|
||||
- venv/Scripts/activate.ps1
|
||||
- pip install pip==19.3.1; $true # $true ignores errors. need this to get the correct coincurve wheel. see commit notes for details.
|
||||
after_script:
|
||||
- rmdir -Recurse venv
|
||||
script:
|
||||
- pip install --upgrade 'setuptools<45.0.0'
|
||||
- pip install pyinstaller==3.5
|
||||
- pip install -e .
|
||||
- python docker/set_build.py # must come after lbry is installed because it imports lbry
|
||||
- pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico -F -n lbrynet lbry/extras/cli.py
|
||||
- 7z a -tzip $env:CI_PROJECT_DIR/lbrynet-${OS}.zip ./dist/lbrynet.exe
|
||||
- checksum --type=sha256 --file=$env:CI_PROJECT_DIR/lbrynet-${OS}.zip
|
||||
- dist/lbrynet.exe --version
|
||||
|
||||
|
||||
|
||||
# s3 = upload asset to s3 (build.lbry.io)
|
||||
.s3:
|
||||
stage: assets
|
||||
variables:
|
||||
GIT_STRATEGY: none
|
||||
script:
|
||||
- "[ -f lbrynet-${OS}.zip ]" # check that asset exists before trying to upload
|
||||
- pip install awscli
|
||||
- S3_PATH="daemon/gitlab-build-${CI_PIPELINE_ID}_commit-${CI_COMMIT_SHA:0:7}$( if [ ! -z ${CI_COMMIT_TAG} ]; then echo _tag-${CI_COMMIT_TAG}; else echo _branch-${CI_COMMIT_REF_NAME}; fi )"
|
||||
- AWS_ACCESS_KEY_ID=${ARTIFACTS_KEY} AWS_SECRET_ACCESS_KEY=${ARTIFACTS_SECRET} AWS_REGION=${ARTIFACTS_REGION}
|
||||
aws s3 cp lbrynet-${OS}.zip s3://${ARTIFACTS_BUCKET}/${S3_PATH}/lbrynet-${OS}.zip
|
||||
|
||||
s3:linux:
|
||||
extends: .s3
|
||||
variables: {OS: linux}
|
||||
needs: ["build:linux"]
|
||||
|
||||
s3:mac:
|
||||
extends: .s3
|
||||
variables: {OS: mac}
|
||||
needs: ["build:mac"]
|
||||
|
||||
s3:windows:
|
||||
extends: .s3
|
||||
variables: {OS: windows}
|
||||
needs: ["build:windows"]
|
||||
|
||||
|
||||
|
||||
# github = upload assets to github when there's a tagged release
|
||||
.github:
|
||||
extends: .tagged
|
||||
stage: assets
|
||||
variables:
|
||||
GIT_STRATEGY: none
|
||||
script:
|
||||
- "[ -f lbrynet-${OS}.zip ]" # check that asset exists before trying to upload. githubrelease won't error if its missing
|
||||
- pip install githubrelease
|
||||
- githubrelease --no-progress --github-token ${GITHUB_CI_USER_ACCESS_TOKEN} asset lbryio/lbry-sdk upload ${CI_COMMIT_TAG} lbrynet-${OS}.zip
|
||||
|
||||
github:linux:
|
||||
extends: .github
|
||||
variables: {OS: linux}
|
||||
needs: ["build:linux"]
|
||||
|
||||
github:mac:
|
||||
extends: .github
|
||||
variables: {OS: mac}
|
||||
needs: ["build:mac"]
|
||||
|
||||
github:windows:
|
||||
extends: .github
|
||||
variables: {OS: windows}
|
||||
needs: ["build:windows"]
|
||||
|
||||
|
||||
|
||||
publish:
|
||||
extends: .tagged
|
||||
stage: release
|
||||
variables:
|
||||
GIT_STRATEGY: none
|
||||
script:
|
||||
- pip install githubrelease
|
||||
- githubrelease --no-progress --github-token ${GITHUB_CI_USER_ACCESS_TOKEN} release lbryio/lbry-sdk publish ${CI_COMMIT_TAG}
|
||||
- >
|
||||
curl -X POST -H 'Content-type: application/json' --data '{"text":"<!channel> There is a new SDK release: https://github.com/lbryio/lbry-sdk/releases/tag/'"${CI_COMMIT_TAG}"'\n'"$(curl -s "https://api.github.com/repos/lbryio/lbry-sdk/releases/tags/${CI_COMMIT_TAG}" | egrep '\w*\"body\":' | cut -d':' -f 2- | tail -c +3 | head -c -2)"'", "channel":"tech"}' "$(echo ${SLACK_WEBHOOK_URL_BASE64} | base64 -d)"
|
||||
+113
-44
@@ -9,20 +9,29 @@ Here's a video walkthrough of this setup, which is itself hosted by the LBRY net
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Running `lbrynet` from source requires Python 3.7 or higher. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/).
|
||||
Running `lbrynet` from source requires Python 3.7. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/).
|
||||
|
||||
After installing python 3, you'll need to install some additional libraries depending on your operating system.
|
||||
After installing Python 3.7, you'll need to install some additional libraries depending on your operating system.
|
||||
|
||||
Because of [issue #2769](https://github.com/lbryio/lbry-sdk/issues/2769)
|
||||
at the moment the `lbrynet` daemon will only work correctly with Python 3.7.
|
||||
If Python 3.8+ is used, the daemon will start but the RPC server
|
||||
may not accept messages, returning the following:
|
||||
```
|
||||
Could not connect to daemon. Are you sure it's running?
|
||||
```
|
||||
|
||||
### macOS
|
||||
|
||||
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
|
||||
|
||||
These environment variables also need to be set:
|
||||
1. PYTHONUNBUFFERED=1
|
||||
2. EVENT_NOKQUEUE=1
|
||||
```
|
||||
PYTHONUNBUFFERED=1
|
||||
EVENT_NOKQUEUE=1
|
||||
```
|
||||
|
||||
Remaining dependencies can then be installed by running:
|
||||
|
||||
```
|
||||
brew install python protobuf
|
||||
```
|
||||
@@ -31,14 +40,17 @@ Assistance installing Python3: https://docs.python-guide.org/starting/install3/o
|
||||
|
||||
### Linux
|
||||
|
||||
On Ubuntu (16.04 minimum, we recommend 18.04), install the following:
|
||||
|
||||
On Ubuntu (we recommend 18.04 or 20.04), install the following:
|
||||
```
|
||||
sudo add-apt-repository ppa:deadsnakes/ppa
|
||||
sudo apt-get update
|
||||
sudo apt-get install build-essential python3.7 python3.7-dev git python3.7-venv libssl-dev python-protobuf
|
||||
```
|
||||
|
||||
The [deadsnakes PPA](https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa) provides Python 3.7
|
||||
for those Ubuntu distributions that no longer have it in their
|
||||
official repositories.
|
||||
|
||||
On Raspbian, you will also need to install `python-pyparsing`.
|
||||
|
||||
If you're running another Linux distro, install the equivalent of the above packages for your system.
|
||||
@@ -47,62 +59,119 @@ If you're running another Linux distro, install the equivalent of the above pack
|
||||
|
||||
### Linux/Mac
|
||||
|
||||
To install on Linux/Mac:
|
||||
Clone the repository:
|
||||
```bash
|
||||
git clone https://github.com/lbryio/lbry-sdk.git
|
||||
cd lbry-sdk
|
||||
```
|
||||
|
||||
```
|
||||
Clone the repository:
|
||||
$ git clone https://github.com/lbryio/lbry-sdk.git
|
||||
$ cd lbry-sdk
|
||||
Create a Python virtual environment for lbry-sdk:
|
||||
```bash
|
||||
python3.7 -m venv lbry-venv
|
||||
```
|
||||
|
||||
Create a Python virtual environment for lbry-sdk:
|
||||
$ python3.7 -m venv lbry-venv
|
||||
|
||||
Activating lbry-sdk virtual environment:
|
||||
$ source lbry-venv/bin/activate
|
||||
|
||||
Make sure you're on Python 3.7+ (as the default Python in virtual environment):
|
||||
$ python --version
|
||||
Activate virtual environment:
|
||||
```bash
|
||||
source lbry-venv/bin/activate
|
||||
```
|
||||
|
||||
Install packages:
|
||||
$ make install
|
||||
Make sure you're on Python 3.7+ as default in the virtual environment:
|
||||
```bash
|
||||
python --version
|
||||
```
|
||||
|
||||
If you are on Linux and using PyCharm, generates initial configs:
|
||||
$ make idea
|
||||
```
|
||||
Install packages:
|
||||
```bash
|
||||
make install
|
||||
```
|
||||
|
||||
To verify your installation, `which lbrynet` should return a path inside of the `lbry-venv` folder created by the `python3.7 -m venv lbry-venv` command.
|
||||
If you are on Linux and using PyCharm, generates initial configs:
|
||||
```bash
|
||||
make idea
|
||||
```
|
||||
|
||||
To verify your installation, `which lbrynet` should return a path inside
|
||||
of the `lbry-venv` folder.
|
||||
```bash
|
||||
(lbry-venv) $ which lbrynet
|
||||
/opt/lbry-sdk/lbry-venv/bin/lbrynet
|
||||
```
|
||||
|
||||
To exit the virtual environment simply use the command `deactivate`.
|
||||
|
||||
### Windows
|
||||
|
||||
To install on Windows:
|
||||
Clone the repository:
|
||||
```bash
|
||||
git clone https://github.com/lbryio/lbry-sdk.git
|
||||
cd lbry-sdk
|
||||
```
|
||||
|
||||
```
|
||||
Clone the repository:
|
||||
> git clone https://github.com/lbryio/lbry-sdk.git
|
||||
> cd lbry-sdk
|
||||
Create a Python virtual environment for lbry-sdk:
|
||||
```bash
|
||||
python -m venv lbry-venv
|
||||
```
|
||||
|
||||
Create a Python virtual environment for lbry-sdk:
|
||||
> python -m venv lbry-venv
|
||||
Activate virtual environment:
|
||||
```bash
|
||||
lbry-venv\Scripts\activate
|
||||
```
|
||||
|
||||
Activating lbry-sdk virtual environment:
|
||||
> lbry-venv\Scripts\activate
|
||||
|
||||
Install packages:
|
||||
> pip install -e .
|
||||
```
|
||||
Install packages:
|
||||
```bash
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
## Run the tests
|
||||
### Elasticsearch
|
||||
|
||||
To run the unit tests from the repo directory:
|
||||
For running integration tests, Elasticsearch is required to be available at localhost:9200/
|
||||
|
||||
```
|
||||
python -m unittest discover tests.unit
|
||||
```
|
||||
The easiest way to start it is using docker with:
|
||||
```bash
|
||||
make elastic-docker
|
||||
```
|
||||
|
||||
Alternative installation methods are available [at Elasticsearch website](https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html).
|
||||
|
||||
To run the unit and integration tests from the repo directory:
|
||||
```
|
||||
python -m unittest discover tests.unit
|
||||
python -m unittest discover tests.integration
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
To start the API server:
|
||||
`lbrynet start`
|
||||
```
|
||||
lbrynet start
|
||||
```
|
||||
|
||||
Whenever the code inside [lbry-sdk/lbry](./lbry)
|
||||
is modified we should run `make install` to recompile the `lbrynet`
|
||||
executable with the newest code.
|
||||
|
||||
## Development
|
||||
|
||||
When developing, remember to enter the environment,
|
||||
and if you wish start the server interactively.
|
||||
```bash
|
||||
$ source lbry-venv/bin/activate
|
||||
|
||||
(lbry-venv) $ python lbry/extras/cli.py start
|
||||
```
|
||||
|
||||
Parameters can be passed in the same way.
|
||||
```bash
|
||||
(lbry-venv) $ python lbry/extras/cli.py wallet balance
|
||||
```
|
||||
|
||||
If a Python debugger (`pdb` or `ipdb`) is installed we can also start it
|
||||
in this way, set up break points, and step through the code.
|
||||
```bash
|
||||
(lbry-venv) $ pip install ipdb
|
||||
|
||||
(lbry-venv) $ ipdb lbry/extras/cli.py
|
||||
```
|
||||
|
||||
Happy hacking!
|
||||
@@ -1,23 +1,26 @@
|
||||
.PHONY: install tools lint test idea
|
||||
.PHONY: install tools lint test test-unit test-unit-coverage test-integration idea
|
||||
|
||||
install:
|
||||
CFLAGS="-DSQLITE_MAX_VARIABLE_NUMBER=2500000" pip install -U https://github.com/rogerbinns/apsw/releases/download/3.30.1-r1/apsw-3.30.1-r1.zip \
|
||||
--global-option=fetch \
|
||||
--global-option=--version --global-option=3.30.1 --global-option=--all \
|
||||
--global-option=build --global-option=--enable --global-option=fts5
|
||||
pip install -e .
|
||||
|
||||
tools:
|
||||
pip install mypy==0.701
|
||||
pip install coverage astroid pylint
|
||||
|
||||
lint:
|
||||
pylint --rcfile=setup.cfg lbry
|
||||
#mypy --ignore-missing-imports lbry
|
||||
|
||||
test:
|
||||
test: test-unit test-integration
|
||||
|
||||
test-unit:
|
||||
python -m unittest discover tests.unit
|
||||
|
||||
test-unit-coverage:
|
||||
coverage run --source=lbry -m unittest discover -vv tests.unit
|
||||
|
||||
test-integration:
|
||||
tox
|
||||
|
||||
idea:
|
||||
mkdir -p .idea
|
||||
cp -r scripts/idea/* .idea
|
||||
|
||||
elastic-docker:
|
||||
docker run -d -v lbryhub:/usr/share/elasticsearch/data -p 9200:9200 -p 9300:9300 -e"ES_JAVA_OPTS=-Xms512m -Xmx512m" -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.12.1
|
||||
@@ -1,10 +1,10 @@
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry-sdk/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://ci.lbry.tech/lbry/lbry-sdk)
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry-sdk/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://github.com/lbryio/lbry-sdk/actions/workflows/main.yml) [](https://coveralls.io/github/lbryio/lbry-sdk)
|
||||
|
||||
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
|
||||
LBRY SDK for Python is currently the most fully featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components include:
|
||||
|
||||
* Built on Python 3.7+ and `asyncio`.
|
||||
* Built on Python 3.7 and `asyncio`.
|
||||
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbry.dht](https://github.com/lbryio/lbry-sdk/tree/master/lbry/dht)).
|
||||
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbry.blob_exchange](https://github.com/lbryio/lbry-sdk/tree/master/lbry/blob_exchange)).
|
||||
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbry.schema](https://github.com/lbryio/lbry-sdk/tree/master/lbry/schema)).
|
||||
@@ -41,7 +41,7 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
|
||||
|
||||
## Security
|
||||
|
||||
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.com/faq/gpg-key) if you need it.
|
||||
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our PGP key is here](https://lbry.com/faq/pgp-key) if you need it.
|
||||
|
||||
## Contact
|
||||
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
# Security Policy
|
||||
|
||||
## Supported Versions
|
||||
|
||||
While we are not at v1.0 yet, only the latest release will be supported.
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
See https://lbry.com/faq/security
|
||||
@@ -0,0 +1,38 @@
|
||||
FROM debian:10-slim
|
||||
|
||||
ARG user=lbry
|
||||
ARG projects_dir=/home/$user
|
||||
|
||||
ARG DOCKER_TAG
|
||||
ARG DOCKER_COMMIT=docker
|
||||
ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get -y --no-install-recommends install \
|
||||
wget \
|
||||
automake libtool \
|
||||
tar unzip \
|
||||
build-essential \
|
||||
pkg-config \
|
||||
libleveldb-dev \
|
||||
python3.7 \
|
||||
python3-dev \
|
||||
python3-pip \
|
||||
python3-wheel \
|
||||
python3-setuptools && \
|
||||
update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN groupadd -g 999 $user && useradd -m -u 999 -g $user $user
|
||||
|
||||
COPY . $projects_dir
|
||||
RUN chown -R $user:$user $projects_dir
|
||||
|
||||
USER $user
|
||||
WORKDIR $projects_dir
|
||||
|
||||
RUN make install
|
||||
RUN python3 docker/set_build.py
|
||||
RUN rm ~/.cache -rf
|
||||
ENTRYPOINT ["python3", "scripts/dht_node.py"]
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM ubuntu:19.10
|
||||
FROM debian:10-slim
|
||||
|
||||
ARG user=lbry
|
||||
ARG db_dir=/database
|
||||
@@ -13,10 +13,13 @@ RUN apt-get update && \
|
||||
wget \
|
||||
tar unzip \
|
||||
build-essential \
|
||||
python3 \
|
||||
automake libtool \
|
||||
pkg-config \
|
||||
python3.7 \
|
||||
python3-dev \
|
||||
python3-pip \
|
||||
python3-wheel \
|
||||
python3-cffi \
|
||||
python3-setuptools && \
|
||||
update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
# FROM debian:10-slim
|
||||
FROM python:3.7.12-slim-buster
|
||||
|
||||
ARG user=lbry
|
||||
ARG db_dir=/database
|
||||
ARG projects_dir=/home/$user
|
||||
|
||||
ARG DOCKER_TAG
|
||||
ARG DOCKER_COMMIT=docker
|
||||
ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get -y --no-install-recommends install \
|
||||
wget \
|
||||
tar unzip \
|
||||
build-essential \
|
||||
automake libtool \
|
||||
pkg-config
|
||||
|
||||
RUN pip install uvloop
|
||||
RUN groupadd -g 999 $user && useradd -m -u 999 -g $user $user
|
||||
RUN mkdir -p $db_dir
|
||||
RUN chown -R $user:$user $db_dir
|
||||
|
||||
COPY . $projects_dir
|
||||
RUN chown -R $user:$user $projects_dir
|
||||
|
||||
USER $user
|
||||
WORKDIR $projects_dir
|
||||
|
||||
RUN make install
|
||||
RUN python3 docker/set_build.py
|
||||
RUN rm ~/.cache -rf
|
||||
|
||||
# entry point
|
||||
ARG host=0.0.0.0
|
||||
ARG tcp_port=50001
|
||||
ARG daemon_url=http://lbry:lbry@192.99.151.178:9245/
|
||||
VOLUME $db_dir
|
||||
ENV TCP_PORT=$tcp_port
|
||||
ENV HOST=$host
|
||||
ENV DAEMON_URL=$daemon_url
|
||||
ENV DB_DIRECTORY=$db_dir
|
||||
ENV MAX_SESSIONS=1000000000
|
||||
ENV MAX_SEND=1000000000000000000
|
||||
ENV EVENT_LOOP_POLICY=uvloop
|
||||
COPY ./docker/wallet_server_entrypoint.sh /entrypoint.sh
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
@@ -0,0 +1,45 @@
|
||||
FROM debian:10-slim
|
||||
|
||||
ARG user=lbry
|
||||
ARG downloads_dir=/database
|
||||
ARG projects_dir=/home/$user
|
||||
|
||||
ARG DOCKER_TAG
|
||||
ARG DOCKER_COMMIT=docker
|
||||
ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get -y --no-install-recommends install \
|
||||
wget \
|
||||
automake libtool \
|
||||
tar unzip \
|
||||
build-essential \
|
||||
pkg-config \
|
||||
libleveldb-dev \
|
||||
python3.7 \
|
||||
python3-dev \
|
||||
python3-pip \
|
||||
python3-wheel \
|
||||
python3-setuptools && \
|
||||
update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN groupadd -g 999 $user && useradd -m -u 999 -g $user $user
|
||||
RUN mkdir -p $downloads_dir
|
||||
RUN chown -R $user:$user $downloads_dir
|
||||
|
||||
COPY . $projects_dir
|
||||
RUN chown -R $user:$user $projects_dir
|
||||
|
||||
USER $user
|
||||
WORKDIR $projects_dir
|
||||
|
||||
RUN pip install uvloop
|
||||
RUN make install
|
||||
RUN python3 docker/set_build.py
|
||||
RUN rm ~/.cache -rf
|
||||
|
||||
# entry point
|
||||
VOLUME $downloads_dir
|
||||
COPY ./docker/webconf.yaml /webconf.yaml
|
||||
ENTRYPOINT ["/home/lbry/.local/bin/lbrynet", "start", "--config=/webconf.yaml"]
|
||||
@@ -0,0 +1,9 @@
|
||||
### How to run with docker-compose
|
||||
1. Edit config file and after that fix permissions with
|
||||
```
|
||||
sudo chown -R 999:999 webconf.yaml
|
||||
```
|
||||
2. Start SDK with
|
||||
```
|
||||
docker-compose up -d
|
||||
```
|
||||
@@ -1,34 +1,49 @@
|
||||
version: "3"
|
||||
|
||||
volumes:
|
||||
lbrycrd:
|
||||
wallet_server:
|
||||
es01:
|
||||
|
||||
services:
|
||||
lbrycrd:
|
||||
image: lbry/lbrycrd:${LBRYCRD_TAG:-latest-release}
|
||||
restart: always
|
||||
ports: # accessible from host
|
||||
- "9246:9246" # rpc port
|
||||
expose: # internal to docker network. also this doesn't do anything. its for documentation only.
|
||||
- "9245" # node-to-node comms port
|
||||
volumes:
|
||||
- "lbrycrd:/data/.lbrycrd"
|
||||
environment:
|
||||
- RUN_MODE=default
|
||||
- SNAPSHOT_URL=${LBRYCRD_SNAPSHOT_URL-https://lbry.com/snapshot/blockchain}
|
||||
- RPC_ALLOW_IP=0.0.0.0/0
|
||||
wallet_server:
|
||||
depends_on:
|
||||
- es01
|
||||
image: lbry/wallet-server:${WALLET_SERVER_TAG:-latest-release}
|
||||
depends_on:
|
||||
- lbrycrd
|
||||
restart: always
|
||||
network_mode: host
|
||||
ports:
|
||||
- "50001:50001" # rpc port
|
||||
- "50005:50005" # websocket port
|
||||
#- "2112:2112" # uncomment to enable prometheus
|
||||
- "2112:2112" # uncomment to enable prometheus
|
||||
volumes:
|
||||
- "wallet_server:/database"
|
||||
environment:
|
||||
- SNAPSHOT_URL=${WALLET_SERVER_SNAPSHOT_URL-https://lbry.com/snapshot/wallet}
|
||||
- DAEMON_URL=http://lbry:lbry@lbrycrd:9245
|
||||
- DAEMON_URL=http://lbry:lbry@127.0.0.1:9245
|
||||
- MAX_QUERY_WORKERS=4
|
||||
- CACHE_MB=1024
|
||||
- CACHE_ALL_TX_HASHES=
|
||||
- CACHE_ALL_CLAIM_TXOS=
|
||||
- MAX_SEND=1000000000000000000
|
||||
- MAX_RECEIVE=1000000000000000000
|
||||
- MAX_SESSIONS=100000
|
||||
- HOST=0.0.0.0
|
||||
- TCP_PORT=50001
|
||||
- PROMETHEUS_PORT=2112
|
||||
- FILTERING_CHANNEL_IDS=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8
|
||||
- BLOCKING_CHANNEL_IDS=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6
|
||||
es01:
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0
|
||||
container_name: es01
|
||||
environment:
|
||||
- node.name=es01
|
||||
- discovery.type=single-node
|
||||
- indices.query.bool.max_clause_count=8192
|
||||
- bootstrap.memory_lock=true
|
||||
- "ES_JAVA_OPTS=-Xms4g -Xmx4g" # no more than 32, remember to disable swap
|
||||
ulimits:
|
||||
memlock:
|
||||
soft: -1
|
||||
hard: -1
|
||||
volumes:
|
||||
- es01:/usr/share/elasticsearch/data
|
||||
ports:
|
||||
- 127.0.0.1:9200:9200
|
||||
@@ -0,0 +1,9 @@
|
||||
version: '3'
|
||||
services:
|
||||
websdk:
|
||||
image: vshyba/websdk
|
||||
ports:
|
||||
- '5279:5279'
|
||||
- '5280:5280'
|
||||
volumes:
|
||||
- ./webconf.yaml:/webconf.yaml
|
||||
+5
-3
@@ -20,7 +20,7 @@ def _check_and_set(d: dict, key: str, value: str):
|
||||
def main():
|
||||
build_info = {item: build_info_mod.__dict__[item] for item in dir(build_info_mod) if not item.startswith("__")}
|
||||
|
||||
commit_hash = os.getenv('DOCKER_COMMIT', os.getenv('CI_COMMIT_SHA', os.getenv('TRAVIS_COMMIT')))
|
||||
commit_hash = os.getenv('DOCKER_COMMIT', os.getenv('GITHUB_SHA'))
|
||||
if commit_hash is None:
|
||||
raise ValueError("Commit hash not found in env vars")
|
||||
_check_and_set(build_info, "COMMIT_HASH", commit_hash[:6])
|
||||
@@ -30,8 +30,10 @@ def main():
|
||||
_check_and_set(build_info, "DOCKER_TAG", docker_tag)
|
||||
_check_and_set(build_info, "BUILD", "docker")
|
||||
else:
|
||||
ci_tag = os.getenv('CI_COMMIT_TAG', os.getenv('TRAVIS_TAG'))
|
||||
_check_and_set(build_info, "BUILD", "release" if re.match(r'v\d+\.\d+\.\d+$', str(ci_tag)) else "qa")
|
||||
if re.match(r'refs/tags/v\d+\.\d+\.\d+$', str(os.getenv('GITHUB_REF'))):
|
||||
_check_and_set(build_info, "BUILD", "release")
|
||||
else:
|
||||
_check_and_set(build_info, "BUILD", "qa")
|
||||
|
||||
log.debug("build info: %s", ", ".join([f"{k}={v}" for k, v in build_info.items()]))
|
||||
with open(build_info_mod.__file__, 'w') as f:
|
||||
|
||||
@@ -6,7 +6,7 @@ set -euo pipefail
|
||||
|
||||
SNAPSHOT_URL="${SNAPSHOT_URL:-}" #off by default. latest snapshot at https://lbry.com/snapshot/wallet
|
||||
|
||||
if [[ -n "$SNAPSHOT_URL" ]] && [[ ! -f /database/claims.db ]]; then
|
||||
if [[ -n "$SNAPSHOT_URL" ]] && [[ ! -f /database/lbry-leveldb ]]; then
|
||||
files="$(ls)"
|
||||
echo "Downloading wallet snapshot from $SNAPSHOT_URL"
|
||||
wget --no-verbose --trust-server-names --content-disposition "$SNAPSHOT_URL"
|
||||
@@ -20,4 +20,14 @@ if [[ -n "$SNAPSHOT_URL" ]] && [[ ! -f /database/claims.db ]]; then
|
||||
rm "$filename"
|
||||
fi
|
||||
|
||||
/home/lbry/.local/bin/torba-server "$@"
|
||||
if [ -z "$HUB_COMMAND" ]; then
|
||||
echo "HUB_COMMAND env variable must be writer, reader, or es_sync"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
case "$HUB_COMMAND" in
|
||||
writer ) /home/lbry/.local/bin/lbry-hub-writer "$@" ;;
|
||||
reader ) /home/lbry/.local/bin/lbry-hub-server "$@" ;;
|
||||
es_sync ) /home/lbry/.local/bin/lbry-hub-elastic-sync ;;
|
||||
* ) "HUB_COMMAND env variable must be writer, reader, or es_sync" && exit 1 ;;
|
||||
esac
|
||||
@@ -0,0 +1,9 @@
|
||||
allowed_origin: "*"
|
||||
max_key_fee: "0.0 USD"
|
||||
save_files: false
|
||||
save_blobs: false
|
||||
streaming_server: "0.0.0.0:5280"
|
||||
api: "0.0.0.0:5279"
|
||||
data_dir: /tmp
|
||||
download_dir: /tmp
|
||||
wallet_dir: /tmp
|
||||
+1158
-407
File diff suppressed because it is too large.
Load diff
+1
-1
@@ -1,2 +1,2 @@
|
||||
__version__ = "0.62.0"
|
||||
__version__ = "0.106.0"
|
||||
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
|
||||
+58
-32
@@ -1,5 +1,6 @@
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import asyncio
|
||||
import binascii
|
||||
import logging
|
||||
@@ -70,12 +71,16 @@ class AbstractBlob:
|
||||
'writers',
|
||||
'verified',
|
||||
'writing',
|
||||
'readers'
|
||||
'readers',
|
||||
'added_on',
|
||||
'is_mine',
|
||||
]
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None):
|
||||
def __init__(
|
||||
self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None, added_on: typing.Optional[int] = None, is_mine: bool = False,
|
||||
):
|
||||
self.loop = loop
|
||||
self.blob_hash = blob_hash
|
||||
self.length = length
|
||||
@@ -85,6 +90,8 @@ class AbstractBlob:
|
||||
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||
self.writing: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||
self.readers: typing.List[typing.BinaryIO] = []
|
||||
self.added_on = added_on or time.time()
|
||||
self.is_mine = is_mine
|
||||
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise InvalidBlobHashError(blob_hash)
|
||||
@@ -110,7 +117,7 @@ class AbstractBlob:
|
||||
if reader in self.readers:
|
||||
self.readers.remove(reader)
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
def _write_blob(self, blob_bytes: bytes) -> asyncio.Task:
|
||||
raise NotImplementedError()
|
||||
|
||||
def set_length(self, length) -> None:
|
||||
@@ -180,29 +187,36 @@ class AbstractBlob:
|
||||
|
||||
@classmethod
|
||||
async def create_from_unencrypted(
|
||||
cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
|
||||
unencrypted: bytes, blob_num: int,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None) -> BlobInfo:
|
||||
cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
|
||||
unencrypted: bytes, blob_num: int, added_on: int, is_mine: bool,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None,
|
||||
) -> BlobInfo:
|
||||
"""
|
||||
Create an encrypted BlobFile from plaintext bytes
|
||||
"""
|
||||
|
||||
blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted)
|
||||
length = len(blob_bytes)
|
||||
blob = cls(loop, blob_hash, length, blob_completed_callback, blob_dir)
|
||||
blob = cls(loop, blob_hash, length, blob_completed_callback, blob_dir, added_on, is_mine)
|
||||
writer = blob.get_blob_writer()
|
||||
writer.write(blob_bytes)
|
||||
await blob.verified.wait()
|
||||
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash)
|
||||
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash, added_on, is_mine)
|
||||
|
||||
def save_verified_blob(self, verified_bytes: bytes):
|
||||
if self.verified.is_set():
|
||||
return
|
||||
if self.is_writeable():
|
||||
self._write_blob(verified_bytes)
|
||||
|
||||
def update_events(_):
|
||||
self.verified.set()
|
||||
self.writing.clear()
|
||||
|
||||
if self.is_writeable():
|
||||
self.writing.set()
|
||||
task = self._write_blob(verified_bytes)
|
||||
task.add_done_callback(update_events)
|
||||
if self.blob_completed_callback:
|
||||
self.blob_completed_callback(self)
|
||||
task.add_done_callback(lambda _: self.blob_completed_callback(self))
|
||||
|
||||
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
|
||||
peer_port: typing.Optional[int] = None) -> HashBlobWriter:
|
||||
@@ -242,11 +256,13 @@ class BlobBuffer(AbstractBlob):
|
||||
"""
|
||||
An in-memory only blob
|
||||
"""
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None):
|
||||
def __init__(
|
||||
self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None, added_on: typing.Optional[int] = None, is_mine: bool = False
|
||||
):
|
||||
self._verified_bytes: typing.Optional[BytesIO] = None
|
||||
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
|
||||
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory, added_on, is_mine)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
@@ -261,9 +277,11 @@ class BlobBuffer(AbstractBlob):
|
||||
self.verified.clear()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
if self._verified_bytes:
|
||||
raise OSError("already have bytes for blob")
|
||||
self._verified_bytes = BytesIO(blob_bytes)
|
||||
async def write():
|
||||
if self._verified_bytes:
|
||||
raise OSError("already have bytes for blob")
|
||||
self._verified_bytes = BytesIO(blob_bytes)
|
||||
return self.loop.create_task(write())
|
||||
|
||||
def delete(self):
|
||||
if self._verified_bytes:
|
||||
@@ -281,10 +299,12 @@ class BlobFile(AbstractBlob):
|
||||
"""
|
||||
A blob existing on the local file system
|
||||
"""
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None):
|
||||
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
|
||||
def __init__(
|
||||
self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None, added_on: typing.Optional[int] = None, is_mine: bool = False
|
||||
):
|
||||
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory, added_on, is_mine)
|
||||
if not blob_directory or not os.path.isdir(blob_directory):
|
||||
raise OSError(f"invalid blob directory '{blob_directory}'")
|
||||
self.file_path = os.path.join(self.blob_directory, self.blob_hash)
|
||||
@@ -319,22 +339,28 @@ class BlobFile(AbstractBlob):
|
||||
handle.close()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
with open(self.file_path, 'wb') as f:
|
||||
f.write(blob_bytes)
|
||||
def _write_blob():
|
||||
with open(self.file_path, 'wb') as f:
|
||||
f.write(blob_bytes)
|
||||
|
||||
async def write_blob():
|
||||
await self.loop.run_in_executor(None, _write_blob)
|
||||
|
||||
return self.loop.create_task(write_blob())
|
||||
|
||||
def delete(self):
|
||||
super().delete()
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
return super().delete()
|
||||
|
||||
@classmethod
|
||||
async def create_from_unencrypted(
|
||||
cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
|
||||
unencrypted: bytes, blob_num: int,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'],
|
||||
asyncio.Task]] = None) -> BlobInfo:
|
||||
cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
|
||||
unencrypted: bytes, blob_num: int, added_on: float, is_mine: bool,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None
|
||||
) -> BlobInfo:
|
||||
if not blob_dir or not os.path.isdir(blob_dir):
|
||||
raise OSError(f"cannot create blob in directory: '{blob_dir}'")
|
||||
return await super().create_from_unencrypted(
|
||||
loop, blob_dir, key, iv, unencrypted, blob_num, blob_completed_callback
|
||||
loop, blob_dir, key, iv, unencrypted, blob_num, added_on, is_mine, blob_completed_callback
|
||||
)
|
||||
@@ -7,13 +7,19 @@ class BlobInfo:
|
||||
'blob_num',
|
||||
'length',
|
||||
'iv',
|
||||
'added_on',
|
||||
'is_mine'
|
||||
]
|
||||
|
||||
def __init__(self, blob_num: int, length: int, iv: str, blob_hash: typing.Optional[str] = None):
|
||||
def __init__(
|
||||
self, blob_num: int, length: int, iv: str,
|
||||
blob_hash: typing.Optional[str] = None, added_on=0, is_mine=False):
|
||||
self.blob_hash = blob_hash
|
||||
self.blob_num = blob_num
|
||||
self.length = length
|
||||
self.iv = iv
|
||||
self.added_on = added_on
|
||||
self.is_mine = is_mine
|
||||
|
||||
def as_dict(self) -> typing.Dict:
|
||||
d = {
|
||||
|
||||
+14
-10
@@ -2,7 +2,7 @@ import os
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
from lbry.utils import LRUCache
|
||||
from lbry.utils import LRUCacheWithMetrics
|
||||
from lbry.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
|
||||
from lbry.stream.descriptor import StreamDescriptor
|
||||
from lbry.connection_manager import ConnectionManager
|
||||
@@ -32,34 +32,34 @@ class BlobManager:
|
||||
else self._node_data_store.completed_blobs
|
||||
self.blobs: typing.Dict[str, AbstractBlob] = {}
|
||||
self.config = config
|
||||
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCache(
|
||||
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCacheWithMetrics(
|
||||
self.config.blob_lru_cache_size)
|
||||
self.connection_manager = ConnectionManager(loop)
|
||||
|
||||
def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None):
|
||||
def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None, is_mine: bool = False):
|
||||
if self.config.save_blobs or (
|
||||
is_valid_blobhash(blob_hash) and os.path.isfile(os.path.join(self.blob_dir, blob_hash))):
|
||||
return BlobFile(
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir, is_mine=is_mine
|
||||
)
|
||||
return BlobBuffer(
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir, is_mine=is_mine
|
||||
)
|
||||
|
||||
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
|
||||
def get_blob(self, blob_hash, length: typing.Optional[int] = None, is_mine: bool = False):
|
||||
if blob_hash in self.blobs:
|
||||
if self.config.save_blobs and isinstance(self.blobs[blob_hash], BlobBuffer):
|
||||
buffer = self.blobs.pop(blob_hash)
|
||||
if blob_hash in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.remove(blob_hash)
|
||||
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
|
||||
self.blobs[blob_hash] = self._get_blob(blob_hash, length, is_mine)
|
||||
if buffer.is_readable():
|
||||
with buffer.reader_context() as reader:
|
||||
self.blobs[blob_hash].write_blob(reader.read())
|
||||
if length and self.blobs[blob_hash].length is None:
|
||||
self.blobs[blob_hash].set_length(length)
|
||||
else:
|
||||
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
|
||||
self.blobs[blob_hash] = self._get_blob(blob_hash, length, is_mine)
|
||||
return self.blobs[blob_hash]
|
||||
|
||||
def is_blob_verified(self, blob_hash: str, length: typing.Optional[int] = None) -> bool:
|
||||
@@ -105,9 +105,13 @@ class BlobManager:
|
||||
if isinstance(blob, BlobFile):
|
||||
if blob.blob_hash not in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.add(blob.blob_hash)
|
||||
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=True))
|
||||
return self.loop.create_task(self.storage.add_blobs(
|
||||
(blob.blob_hash, blob.length, blob.added_on, blob.is_mine), finished=True)
|
||||
)
|
||||
else:
|
||||
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=False))
|
||||
return self.loop.create_task(self.storage.add_blobs(
|
||||
(blob.blob_hash, blob.length, blob.added_on, blob.is_mine), finished=False)
|
||||
)
|
||||
|
||||
def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]:
|
||||
"""Returns of the blobhashes_to_check, which are valid"""
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DiskSpaceManager:
|
||||
|
||||
def __init__(self, config, db, blob_manager, cleaning_interval=30 * 60, analytics=None):
|
||||
self.config = config
|
||||
self.db = db
|
||||
self.blob_manager = blob_manager
|
||||
self.cleaning_interval = cleaning_interval
|
||||
self.running = False
|
||||
self.task = None
|
||||
self.analytics = analytics
|
||||
self._used_space_bytes = None
|
||||
|
||||
async def get_free_space_mb(self, is_network_blob=False):
|
||||
limit_mb = self.config.network_storage_limit if is_network_blob else self.config.blob_storage_limit
|
||||
space_used_mb = await self.get_space_used_mb()
|
||||
space_used_mb = space_used_mb['network_storage'] if is_network_blob else space_used_mb['content_storage']
|
||||
return max(0, limit_mb - space_used_mb)
|
||||
|
||||
async def get_space_used_bytes(self):
|
||||
self._used_space_bytes = await self.db.get_stored_blob_disk_usage()
|
||||
return self._used_space_bytes
|
||||
|
||||
async def get_space_used_mb(self, cached=True):
|
||||
cached = cached and self._used_space_bytes is not None
|
||||
space_used_bytes = self._used_space_bytes if cached else await self.get_space_used_bytes()
|
||||
return {key: int(value/1024.0/1024.0) for key, value in space_used_bytes.items()}
|
||||
|
||||
async def clean(self):
|
||||
await self._clean(False)
|
||||
await self._clean(True)
|
||||
|
||||
async def _clean(self, is_network_blob=False):
|
||||
space_used_mb = await self.get_space_used_mb(cached=False)
|
||||
if is_network_blob:
|
||||
space_used_mb = space_used_mb['network_storage']
|
||||
else:
|
||||
space_used_mb = space_used_mb['content_storage'] + space_used_mb['private_storage']
|
||||
storage_limit_mb = self.config.network_storage_limit if is_network_blob else self.config.blob_storage_limit
|
||||
if self.analytics:
|
||||
asyncio.create_task(
|
||||
self.analytics.send_disk_space_used(space_used_mb, storage_limit_mb, is_network_blob)
|
||||
)
|
||||
delete = []
|
||||
available = storage_limit_mb - space_used_mb
|
||||
if storage_limit_mb == 0 if not is_network_blob else available >= 0:
|
||||
return 0
|
||||
for blob_hash, file_size, _ in await self.db.get_stored_blobs(is_mine=False, is_network_blob=is_network_blob):
|
||||
delete.append(blob_hash)
|
||||
available += int(file_size/1024.0/1024.0)
|
||||
if available >= 0:
|
||||
break
|
||||
if delete:
|
||||
await self.db.stop_all_files()
|
||||
await self.blob_manager.delete_blobs(delete, delete_from_db=True)
|
||||
self._used_space_bytes = None
|
||||
return len(delete)
|
||||
|
||||
async def cleaning_loop(self):
|
||||
while self.running:
|
||||
await asyncio.sleep(self.cleaning_interval)
|
||||
await self.clean()
|
||||
|
||||
async def start(self):
|
||||
self.running = True
|
||||
self.task = asyncio.create_task(self.cleaning_loop())
|
||||
self.task.add_done_callback(lambda _: log.info("Stopping blob cleanup service."))
|
||||
|
||||
async def stop(self):
|
||||
if self.running:
|
||||
self.running = False
|
||||
self.task.cancel()
|
||||
@@ -152,6 +152,8 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
log.debug(msg)
|
||||
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
|
||||
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
|
||||
# wait for the io to finish
|
||||
await self.blob.verified.wait()
|
||||
log.info("%s at %fMB/s", msg,
|
||||
round((float(self._blob_bytes_received) /
|
||||
float(time.perf_counter() - start_time)) / 1000000.0, 2))
|
||||
|
||||
@@ -3,6 +3,7 @@ import typing
|
||||
import logging
|
||||
from lbry.utils import cache_concurrent
|
||||
from lbry.blob_exchange.client import request_blob
|
||||
from lbry.dht.node import get_kademlia_peers_from_hosts
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.dht.node import Node
|
||||
@@ -87,7 +88,6 @@ class BlobDownloader:
|
||||
if blob.get_is_verified():
|
||||
return blob
|
||||
self.is_running.set()
|
||||
tried_for_this_blob: typing.Set['KademliaPeer'] = set()
|
||||
try:
|
||||
while not blob.get_is_verified() and self.is_running.is_set():
|
||||
batch: typing.Set['KademliaPeer'] = set(self.connections.keys())
|
||||
@@ -97,24 +97,15 @@ class BlobDownloader:
|
||||
"%s running, %d peers, %d ignored, %d active, %s connections", blob_hash[:6],
|
||||
len(batch), len(self.ignored), len(self.active_connections), len(self.connections)
|
||||
)
|
||||
re_add: typing.Set['KademliaPeer'] = set()
|
||||
for peer in sorted(batch, key=lambda peer: self.scores.get(peer, 0), reverse=True):
|
||||
if peer in self.ignored:
|
||||
continue
|
||||
if peer in tried_for_this_blob:
|
||||
if peer in self.active_connections or not self.should_race_continue(blob):
|
||||
continue
|
||||
if peer in self.active_connections:
|
||||
if peer not in re_add:
|
||||
re_add.add(peer)
|
||||
continue
|
||||
if not self.should_race_continue(blob):
|
||||
break
|
||||
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
|
||||
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id))
|
||||
self.active_connections[peer] = t
|
||||
tried_for_this_blob.add(peer)
|
||||
if not re_add:
|
||||
self.peer_queue.put_nowait(list(batch))
|
||||
self.peer_queue.put_nowait(list(batch))
|
||||
await self.new_peer_or_finished()
|
||||
self.cleanup_active()
|
||||
log.debug("downloaded %s", blob_hash[:8])
|
||||
@@ -133,11 +124,14 @@ class BlobDownloader:
|
||||
protocol.close()
|
||||
|
||||
|
||||
async def download_blob(loop, config: 'Config', blob_manager: 'BlobManager', node: 'Node',
|
||||
async def download_blob(loop, config: 'Config', blob_manager: 'BlobManager', dht_node: 'Node',
|
||||
blob_hash: str) -> 'AbstractBlob':
|
||||
search_queue = asyncio.Queue(loop=loop, maxsize=config.max_connections_per_download)
|
||||
search_queue.put_nowait(blob_hash)
|
||||
peer_queue, accumulate_task = node.accumulate_peers(search_queue)
|
||||
peer_queue, accumulate_task = dht_node.accumulate_peers(search_queue)
|
||||
fixed_peers = None if not config.fixed_peers else await get_kademlia_peers_from_hosts(config.fixed_peers)
|
||||
if fixed_peers:
|
||||
loop.call_later(config.fixed_peer_delay, peer_queue.put_nowait, fixed_peers)
|
||||
downloader = BlobDownloader(loop, config, blob_manager, peer_queue)
|
||||
try:
|
||||
return await downloader.download_blob(blob_hash)
|
||||
|
||||
+148
-40
@@ -1,13 +1,12 @@
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import typing
|
||||
import logging
|
||||
from typing import List, Dict, Tuple, Union, TypeVar, Generic, Optional
|
||||
from argparse import ArgumentParser
|
||||
from contextlib import contextmanager
|
||||
|
||||
import yaml
|
||||
from appdirs import user_data_dir, user_config_dir
|
||||
import yaml
|
||||
from lbry.error import InvalidCurrencyError
|
||||
from lbry.dht import constants
|
||||
from lbry.wallet.coinselection import STRATEGIES
|
||||
@@ -16,7 +15,7 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
NOT_SET = type('NOT_SET', (object,), {}) # pylint: disable=invalid-name
|
||||
T = typing.TypeVar('T')
|
||||
T = TypeVar('T')
|
||||
|
||||
CURRENCIES = {
|
||||
'BTC': {'type': 'crypto'},
|
||||
@@ -25,11 +24,11 @@ CURRENCIES = {
|
||||
}
|
||||
|
||||
|
||||
class Setting(typing.Generic[T]):
|
||||
class Setting(Generic[T]):
|
||||
|
||||
def __init__(self, doc: str, default: typing.Optional[T] = None,
|
||||
previous_names: typing.Optional[typing.List[str]] = None,
|
||||
metavar: typing.Optional[str] = None):
|
||||
def __init__(self, doc: str, default: Optional[T] = None,
|
||||
previous_names: Optional[List[str]] = None,
|
||||
metavar: Optional[str] = None):
|
||||
self.doc = doc
|
||||
self.default = default
|
||||
self.previous_names = previous_names or []
|
||||
@@ -46,7 +45,7 @@ class Setting(typing.Generic[T]):
|
||||
def no_cli_name(self):
|
||||
return f"--no-{self.name.replace('_', '-')}"
|
||||
|
||||
def __get__(self, obj: typing.Optional['BaseConfig'], owner) -> T:
|
||||
def __get__(self, obj: Optional['BaseConfig'], owner) -> T:
|
||||
if obj is None:
|
||||
return self
|
||||
for location in obj.search_order:
|
||||
@@ -54,7 +53,7 @@ class Setting(typing.Generic[T]):
|
||||
return location[self.name]
|
||||
return self.default
|
||||
|
||||
def __set__(self, obj: 'BaseConfig', val: typing.Union[T, NOT_SET]):
|
||||
def __set__(self, obj: 'BaseConfig', val: Union[T, NOT_SET]):
|
||||
if val == NOT_SET:
|
||||
for location in obj.modify_order:
|
||||
if self.name in location:
|
||||
@@ -64,6 +63,18 @@ class Setting(typing.Generic[T]):
|
||||
for location in obj.modify_order:
|
||||
location[self.name] = val
|
||||
|
||||
def is_set(self, obj: 'BaseConfig') -> bool:
|
||||
for location in obj.search_order:
|
||||
if self.name in location:
|
||||
return True
|
||||
return False
|
||||
|
||||
def is_set_to_default(self, obj: 'BaseConfig') -> bool:
|
||||
for location in obj.search_order:
|
||||
if self.name in location:
|
||||
return location[self.name] == self.default
|
||||
return False
|
||||
|
||||
def validate(self, value):
|
||||
raise NotImplementedError()
|
||||
|
||||
@@ -88,7 +99,7 @@ class String(Setting[str]):
|
||||
f"Setting '{self.name}' must be a string."
|
||||
|
||||
# TODO: removes this after pylint starts to understand generics
|
||||
def __get__(self, obj: typing.Optional['BaseConfig'], owner) -> str: # pylint: disable=useless-super-delegation
|
||||
def __get__(self, obj: Optional['BaseConfig'], owner) -> str: # pylint: disable=useless-super-delegation
|
||||
return super().__get__(obj, owner)
|
||||
|
||||
|
||||
@@ -192,7 +203,7 @@ class MaxKeyFee(Setting[dict]):
|
||||
)
|
||||
parser.add_argument(
|
||||
self.no_cli_name,
|
||||
help=f"Disable maximum key fee check.",
|
||||
help="Disable maximum key fee check.",
|
||||
dest=self.name,
|
||||
const=None,
|
||||
action="store_const",
|
||||
@@ -201,7 +212,7 @@ class MaxKeyFee(Setting[dict]):
|
||||
|
||||
|
||||
class StringChoice(String):
|
||||
def __init__(self, doc: str, valid_values: typing.List[str], default: str, *args, **kwargs):
|
||||
def __init__(self, doc: str, valid_values: List[str], default: str, *args, **kwargs):
|
||||
super().__init__(doc, default, *args, **kwargs)
|
||||
if not valid_values:
|
||||
raise ValueError("No valid values provided")
|
||||
@@ -274,17 +285,95 @@ class Strings(ListSetting):
|
||||
f"'{self.name}' must be a string."
|
||||
|
||||
|
||||
class KnownHubsList:
|
||||
|
||||
def __init__(self, config: 'Config' = None, file_name: str = 'known_hubs.yml'):
|
||||
self.file_name = file_name
|
||||
self.path = os.path.join(config.wallet_dir, self.file_name) if config else None
|
||||
self.hubs: Dict[Tuple[str, int], Dict] = {}
|
||||
if self.exists:
|
||||
self.load()
|
||||
|
||||
@property
|
||||
def exists(self):
|
||||
return self.path and os.path.exists(self.path)
|
||||
|
||||
@property
|
||||
def serialized(self) -> Dict[str, Dict]:
|
||||
return {f"{host}:{port}": details for (host, port), details in self.hubs.items()}
|
||||
|
||||
def filter(self, match_none=False, **kwargs):
|
||||
if not kwargs:
|
||||
return self.hubs
|
||||
result = {}
|
||||
for hub, details in self.hubs.items():
|
||||
for key, constraint in kwargs.items():
|
||||
value = details.get(key)
|
||||
if value == constraint or (match_none and value is None):
|
||||
result[hub] = details
|
||||
break
|
||||
return result
|
||||
|
||||
def load(self):
|
||||
if self.path:
|
||||
with open(self.path, 'r') as known_hubs_file:
|
||||
raw = known_hubs_file.read()
|
||||
for hub, details in yaml.safe_load(raw).items():
|
||||
self.set(hub, details)
|
||||
|
||||
def save(self):
|
||||
if self.path:
|
||||
with open(self.path, 'w') as known_hubs_file:
|
||||
known_hubs_file.write(yaml.safe_dump(self.serialized, default_flow_style=False))
|
||||
|
||||
def set(self, hub: str, details: Dict):
|
||||
if hub and hub.count(':') == 1:
|
||||
host, port = hub.split(':')
|
||||
hub_parts = (host, int(port))
|
||||
if hub_parts not in self.hubs:
|
||||
self.hubs[hub_parts] = details
|
||||
return hub
|
||||
|
||||
def add_hubs(self, hubs: List[str]):
|
||||
added = False
|
||||
for hub in hubs:
|
||||
if self.set(hub, {}) is not None:
|
||||
added = True
|
||||
return added
|
||||
|
||||
def items(self):
|
||||
return self.hubs.items()
|
||||
|
||||
def __bool__(self):
|
||||
return len(self) > 0
|
||||
|
||||
def __len__(self):
|
||||
return self.hubs.__len__()
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self.hubs)
|
||||
|
||||
|
||||
class EnvironmentAccess:
|
||||
PREFIX = 'LBRY_'
|
||||
|
||||
def __init__(self, environ: dict):
|
||||
self.environ = environ
|
||||
def __init__(self, config: 'BaseConfig', environ: dict):
|
||||
self.configuration = config
|
||||
self.data = {}
|
||||
if environ:
|
||||
self.load(environ)
|
||||
|
||||
def load(self, environ):
|
||||
for setting in self.configuration.get_settings():
|
||||
value = environ.get(f'{self.PREFIX}{setting.name.upper()}', NOT_SET)
|
||||
if value != NOT_SET and not (isinstance(setting, ListSetting) and value is None):
|
||||
self.data[setting.name] = setting.deserialize(value)
|
||||
|
||||
def __contains__(self, item: str):
|
||||
return f'{self.PREFIX}{item.upper()}' in self.environ
|
||||
return item in self.data
|
||||
|
||||
def __getitem__(self, item: str):
|
||||
return self.environ[f'{self.PREFIX}{item.upper()}']
|
||||
return self.data[item]
|
||||
|
||||
|
||||
class ArgumentAccess:
|
||||
@@ -325,7 +414,7 @@ class ConfigFileAccess:
|
||||
cls = type(self.configuration)
|
||||
with open(self.path, 'r') as config_file:
|
||||
raw = config_file.read()
|
||||
serialized = yaml.load(raw) or {}
|
||||
serialized = yaml.safe_load(raw) or {}
|
||||
for key, value in serialized.items():
|
||||
attr = getattr(cls, key, None)
|
||||
if attr is None:
|
||||
@@ -369,7 +458,7 @@ class ConfigFileAccess:
|
||||
del self.data[key]
|
||||
|
||||
|
||||
TBC = typing.TypeVar('TBC', bound='BaseConfig')
|
||||
TBC = TypeVar('TBC', bound='BaseConfig')
|
||||
|
||||
|
||||
class BaseConfig:
|
||||
@@ -443,7 +532,7 @@ class BaseConfig:
|
||||
self.arguments = ArgumentAccess(self, args)
|
||||
|
||||
def set_environment(self, environ=None):
|
||||
self.environment = EnvironmentAccess(environ or os.environ)
|
||||
self.environment = EnvironmentAccess(self, environ or os.environ)
|
||||
|
||||
def set_persisted(self, config_file_path=None):
|
||||
if config_file_path is None:
|
||||
@@ -464,19 +553,21 @@ class BaseConfig:
|
||||
|
||||
class TranscodeConfig(BaseConfig):
|
||||
|
||||
ffmpeg_folder = String('The path to ffmpeg and ffprobe', '')
|
||||
ffmpeg_path = String('A list of places to check for ffmpeg and ffprobe. '
|
||||
f'$data_dir/ffmpeg/bin and $PATH are checked afterward. Separator: {os.pathsep}',
|
||||
'', previous_names=['ffmpeg_folder'])
|
||||
video_encoder = String('FFmpeg codec and parameters for the video encoding. '
|
||||
'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental',
|
||||
'libx264 -crf 21 -preset faster -pix_fmt yuv420p')
|
||||
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 8400000)
|
||||
'libx264 -crf 24 -preset faster -pix_fmt yuv420p')
|
||||
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 5_000_000)
|
||||
video_scaler = String('FFmpeg scaling parameters for reducing bitrate. '
|
||||
'Example: -vf "scale=-2:720,fps=24" -maxrate 5M -bufsize 3M',
|
||||
r'-vf "scale=if(gte(iw\,ih)\,min(2560\,iw)\,-2):if(lt(iw\,ih)\,min(2560\,ih)\,-2)" '
|
||||
r'-maxrate 8400K -bufsize 5000K')
|
||||
r'-vf "scale=if(gte(iw\,ih)\,min(1920\,iw)\,-2):if(lt(iw\,ih)\,min(1920\,ih)\,-2)" '
|
||||
r'-maxrate 5500K -bufsize 5000K')
|
||||
audio_encoder = String('FFmpeg codec and parameters for the audio encoding. '
|
||||
'Example: libopus -b:a 128k',
|
||||
'aac -b:a 160k')
|
||||
volume_filter = String('FFmpeg filter for audio normalization.', '-af loudnorm')
|
||||
volume_filter = String('FFmpeg filter for audio normalization. Exmple: -af loudnorm', '')
|
||||
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', 240)
|
||||
|
||||
|
||||
@@ -498,6 +589,9 @@ class CLIConfig(TranscodeConfig):
|
||||
|
||||
|
||||
class Config(CLIConfig):
|
||||
|
||||
jurisdiction = String("Limit interactions to wallet server in this jurisdiction.")
|
||||
|
||||
# directories
|
||||
data_dir = Path("Directory path to store blobs.", metavar='DIR')
|
||||
download_dir = Path(
|
||||
@@ -519,7 +613,7 @@ class Config(CLIConfig):
|
||||
"ports or have firewall rules you likely want to disable this.", True
|
||||
)
|
||||
udp_port = Integer("UDP port for communicating on the LBRY DHT", 4444, previous_names=['dht_node_port'])
|
||||
tcp_port = Integer("TCP port to listen for incoming blob requests", 3333, previous_names=['peer_port'])
|
||||
tcp_port = Integer("TCP port to listen for incoming blob requests", 4444, previous_names=['peer_port'])
|
||||
prometheus_port = Integer("Port to expose prometheus metrics (off by default)", 0)
|
||||
network_interface = String("Interface to use for the DHT and blob exchange", '0.0.0.0')
|
||||
|
||||
@@ -534,11 +628,14 @@ class Config(CLIConfig):
|
||||
# protocol timeouts
|
||||
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
|
||||
blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
|
||||
hub_timeout = Float("Timeout when making a hub request", 30.0)
|
||||
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0)
|
||||
node_rpc_timeout = Float("Timeout when making a DHT request", constants.RPC_TIMEOUT)
|
||||
|
||||
# blob announcement and download
|
||||
save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True)
|
||||
network_storage_limit = Integer("Disk space in MB to be allocated for helping the P2P network. 0 = disable", 0)
|
||||
blob_storage_limit = Integer("Disk space in MB to be allocated for blob storage. 0 = no limit", 0)
|
||||
blob_lru_cache_size = Integer(
|
||||
"LRU cache size for decrypted downloaded blobs used to minimize re-downloading the same blobs when "
|
||||
"replying to a range request. Set to 0 to disable.", 32
|
||||
@@ -555,6 +652,7 @@ class Config(CLIConfig):
|
||||
"Maximum number of peers to connect to while downloading a blob", 4,
|
||||
previous_names=['max_connections_per_stream']
|
||||
)
|
||||
concurrent_hub_requests = Integer("Maximum number of concurrent hub requests", 32)
|
||||
fixed_peer_delay = Float(
|
||||
"Amount of seconds before adding the reflector servers as potential peers to download from in case dht"
|
||||
"peers are not found or are slow", 2.0
|
||||
@@ -575,9 +673,14 @@ class Config(CLIConfig):
|
||||
)
|
||||
|
||||
# servers
|
||||
reflector_servers = Servers("Reflector re-hosting servers", [
|
||||
reflector_servers = Servers("Reflector re-hosting servers for mirroring publishes", [
|
||||
('reflector.lbry.com', 5566)
|
||||
])
|
||||
|
||||
fixed_peers = Servers("Fixed peers to fall back to if none are found on P2P for a blob", [
|
||||
('cdn.reflector.lbry.com', 5567)
|
||||
])
|
||||
|
||||
lbryum_servers = Servers("SPV wallet servers", [
|
||||
('spv11.lbry.com', 50001),
|
||||
('spv12.lbry.com', 50001),
|
||||
@@ -590,27 +693,28 @@ class Config(CLIConfig):
|
||||
('spv19.lbry.com', 50001),
|
||||
])
|
||||
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
|
||||
('dht.lbry.grin.io', 4444), # Grin
|
||||
('dht.lbry.madiator.com', 4444), # Madiator
|
||||
('lbrynet1.lbry.com', 4444), # US EAST
|
||||
('lbrynet2.lbry.com', 4444), # US WEST
|
||||
('lbrynet3.lbry.com', 4444), # EU
|
||||
('lbrynet4.lbry.com', 4444) # ASIA
|
||||
])
|
||||
|
||||
comment_server = String("Comment server API URL", "https://comments.lbry.com/api")
|
||||
|
||||
# blockchain
|
||||
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
|
||||
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
|
||||
cache_time = Integer("Time to cache resolved claims", 150) # TODO: use this
|
||||
|
||||
# daemon
|
||||
save_files = Toggle("Save downloaded files when calling `get` by default", True)
|
||||
save_files = Toggle("Save downloaded files when calling `get` by default", False)
|
||||
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
|
||||
share_usage_data = Toggle(
|
||||
"Whether to share usage stats and diagnostic info with LBRY.", False,
|
||||
previous_names=['upload_log', 'upload_log', 'share_debug_info']
|
||||
)
|
||||
track_bandwidth = Toggle("Track bandwidth usage", True)
|
||||
allowed_origin = String(
|
||||
"Allowed `Origin` header value for API request (sent by browser), use * to allow "
|
||||
"all hosts; default is to only allow API requests with no `Origin` value.", "")
|
||||
|
||||
# media server
|
||||
streaming_server = String('Host name and port to serve streaming media over range requests',
|
||||
@@ -620,8 +724,10 @@ class Config(CLIConfig):
|
||||
|
||||
coin_selection_strategy = StringChoice(
|
||||
"Strategy to use when selecting UTXOs for a transaction",
|
||||
STRATEGIES, "standard")
|
||||
STRATEGIES, "prefer_confirmed"
|
||||
)
|
||||
|
||||
transaction_cache_size = Integer("Transaction cache size", 2 ** 17)
|
||||
save_resolved_claims = Toggle(
|
||||
"Save content claims to the database when they are resolved to keep file_list up to date, "
|
||||
"only disable this if file_x commands are not needed", True
|
||||
@@ -638,6 +744,7 @@ class Config(CLIConfig):
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.set_default_paths()
|
||||
self.known_hubs = KnownHubsList(self)
|
||||
|
||||
def set_default_paths(self):
|
||||
if 'darwin' in sys.platform.lower():
|
||||
@@ -659,7 +766,7 @@ class Config(CLIConfig):
|
||||
return os.path.join(self.data_dir, 'lbrynet.log')
|
||||
|
||||
|
||||
def get_windows_directories() -> typing.Tuple[str, str, str]:
|
||||
def get_windows_directories() -> Tuple[str, str, str]:
|
||||
from lbry.winpaths import get_path, FOLDERID, UserHandle, \
|
||||
PathNotFoundException # pylint: disable=import-outside-toplevel
|
||||
|
||||
@@ -681,19 +788,20 @@ def get_windows_directories() -> typing.Tuple[str, str, str]:
|
||||
return data_dir, lbryum_dir, download_dir
|
||||
|
||||
|
||||
def get_darwin_directories() -> typing.Tuple[str, str, str]:
|
||||
def get_darwin_directories() -> Tuple[str, str, str]:
|
||||
data_dir = user_data_dir('LBRY')
|
||||
lbryum_dir = os.path.expanduser('~/.lbryum')
|
||||
download_dir = os.path.expanduser('~/Downloads')
|
||||
return data_dir, lbryum_dir, download_dir
|
||||
|
||||
|
||||
def get_linux_directories() -> typing.Tuple[str, str, str]:
|
||||
def get_linux_directories() -> Tuple[str, str, str]:
|
||||
try:
|
||||
with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg:
|
||||
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()).group(1)
|
||||
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir)
|
||||
download_dir = re.sub('\"', '', down_dir)
|
||||
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read())
|
||||
if down_dir:
|
||||
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir.group(1))
|
||||
download_dir = re.sub('\"', '', down_dir)
|
||||
except OSError:
|
||||
download_dir = os.getenv('XDG_DOWNLOAD_DIR')
|
||||
if not download_dir:
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
|
||||
from prometheus_client import Counter, Gauge
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.dht.node import Node
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
@@ -9,6 +12,15 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobAnnouncer:
|
||||
announcements_sent_metric = Counter(
|
||||
"announcements_sent", "Number of announcements sent and their respective status.", namespace="dht_node",
|
||||
labelnames=("peers", "error"),
|
||||
)
|
||||
announcement_queue_size_metric = Gauge(
|
||||
"announcement_queue_size", "Number of hashes waiting to be announced.", namespace="dht_node",
|
||||
labelnames=("scope",)
|
||||
)
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, node: 'Node', storage: 'SQLiteStorage'):
|
||||
self.loop = loop
|
||||
self.node = node
|
||||
@@ -18,12 +30,15 @@ class BlobAnnouncer:
|
||||
|
||||
async def _submit_announcement(self, blob_hash):
|
||||
try:
|
||||
|
||||
peers = len(await self.node.announce_blob(blob_hash))
|
||||
self.announcements_sent_metric.labels(peers=peers, error=False).inc()
|
||||
if peers > 4:
|
||||
return blob_hash
|
||||
else:
|
||||
log.debug("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
|
||||
except Exception as err:
|
||||
self.announcements_sent_metric.labels(peers=0, error=True).inc()
|
||||
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
|
||||
raise err
|
||||
log.warning("error announcing %s: %s", blob_hash[:8], str(err))
|
||||
@@ -37,6 +52,7 @@ class BlobAnnouncer:
|
||||
log.warning("No peers in DHT, announce round skipped")
|
||||
continue
|
||||
self.announce_queue.extend(await self.storage.get_blobs_to_announce())
|
||||
self.announcement_queue_size_metric.labels(scope="global").set(len(self.announce_queue))
|
||||
log.debug("announcer task wake up, %d blobs to announce", len(self.announce_queue))
|
||||
while len(self.announce_queue) > 0:
|
||||
log.info("%i blobs to announce", len(self.announce_queue))
|
||||
|
||||
+37
-5
@@ -1,8 +1,10 @@
|
||||
import logging
|
||||
import asyncio
|
||||
import typing
|
||||
import binascii
|
||||
import socket
|
||||
|
||||
from prometheus_client import Gauge
|
||||
|
||||
from lbry.utils import resolve_host
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.peer import make_kademlia_peer
|
||||
@@ -18,6 +20,14 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Node:
|
||||
storing_peers_metric = Gauge(
|
||||
"storing_peers", "Number of peers storing blobs announced to this node", namespace="dht_node",
|
||||
labelnames=("scope",),
|
||||
)
|
||||
stored_blob_with_x_bytes_colliding = Gauge(
|
||||
"stored_blobs_x_bytes_colliding", "Number of blobs with at least X bytes colliding with this node id prefix",
|
||||
namespace="dht_node", labelnames=("amount",)
|
||||
)
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int,
|
||||
internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: float = constants.RPC_TIMEOUT,
|
||||
split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX,
|
||||
@@ -32,6 +42,10 @@ class Node:
|
||||
self._refresh_task: asyncio.Task = None
|
||||
self._storage = storage
|
||||
|
||||
@property
|
||||
def stored_blob_hashes(self):
|
||||
return self.protocol.data_store.keys()
|
||||
|
||||
async def refresh_node(self, force_once=False):
|
||||
while True:
|
||||
# remove peers with expired blob announcements from the datastore
|
||||
@@ -41,7 +55,18 @@ class Node:
|
||||
# add all peers in the routing table
|
||||
total_peers.extend(self.protocol.routing_table.get_peers())
|
||||
# add all the peers who have announced blobs to us
|
||||
total_peers.extend(self.protocol.data_store.get_storing_contacts())
|
||||
storing_peers = self.protocol.data_store.get_storing_contacts()
|
||||
self.storing_peers_metric.labels("global").set(len(storing_peers))
|
||||
total_peers.extend(storing_peers)
|
||||
|
||||
counts = {0: 0, 1: 0, 2: 0}
|
||||
node_id = self.protocol.node_id
|
||||
for blob_hash in self.protocol.data_store.keys():
|
||||
bytes_colliding = 0 if blob_hash[0] != node_id[0] else 2 if blob_hash[1] == node_id[1] else 1
|
||||
counts[bytes_colliding] += 1
|
||||
self.stored_blob_with_x_bytes_colliding.labels(amount=0).set(counts[0])
|
||||
self.stored_blob_with_x_bytes_colliding.labels(amount=1).set(counts[1])
|
||||
self.stored_blob_with_x_bytes_colliding.labels(amount=2).set(counts[2])
|
||||
|
||||
# get ids falling in the midpoint of each bucket that hasn't been recently updated
|
||||
node_ids = self.protocol.routing_table.get_refresh_list(0, True)
|
||||
@@ -80,7 +105,7 @@ class Node:
|
||||
await fut
|
||||
|
||||
async def announce_blob(self, blob_hash: str) -> typing.List[bytes]:
|
||||
hash_value = binascii.unhexlify(blob_hash.encode())
|
||||
hash_value = bytes.fromhex(blob_hash)
|
||||
assert len(hash_value) == constants.HASH_LENGTH
|
||||
peers = await self.peer_search(hash_value)
|
||||
|
||||
@@ -95,7 +120,7 @@ class Node:
|
||||
stored_to = [node_id for node_id, contacted in stored_to_tup if contacted]
|
||||
if stored_to:
|
||||
log.debug(
|
||||
"Stored %s to %i of %i attempted peers", binascii.hexlify(hash_value).decode()[:8],
|
||||
"Stored %s to %i of %i attempted peers", hash_value.hex()[:8],
|
||||
len(stored_to), len(peers)
|
||||
)
|
||||
else:
|
||||
@@ -223,7 +248,7 @@ class Node:
|
||||
# prioritize peers who reply to a dht ping first
|
||||
# this minimizes attempting to make tcp connections that won't work later to dead or unreachable peers
|
||||
|
||||
async for results in self.get_iterative_value_finder(binascii.unhexlify(blob_hash.encode())):
|
||||
async for results in self.get_iterative_value_finder(bytes.fromhex(blob_hash)):
|
||||
to_put = []
|
||||
for peer in results:
|
||||
if peer.address == self.protocol.external_ip and self.protocol.peer_port == peer.tcp_port:
|
||||
@@ -258,3 +283,10 @@ class Node:
|
||||
) -> typing.Tuple[asyncio.Queue, asyncio.Task]:
|
||||
queue = peer_queue or asyncio.Queue(loop=self.loop)
|
||||
return queue, self.loop.create_task(self._accumulate_peers_for_value(search_queue, queue))
|
||||
|
||||
|
||||
async def get_kademlia_peers_from_hosts(peer_list: typing.List[typing.Tuple[str, int]]) -> typing.List['KademliaPeer']:
|
||||
peer_address_list = [(await resolve_host(url, port, proto='tcp'), port) for url, port in peer_list]
|
||||
kademlia_peer_list = [make_kademlia_peer(None, address, None, tcp_port=port, allow_localhost=True)
|
||||
for address, port in peer_address_list]
|
||||
return kademlia_peer_list
|
||||
+29
-33
@@ -1,18 +1,21 @@
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
import ipaddress
|
||||
from binascii import hexlify
|
||||
from dataclasses import dataclass, field
|
||||
from functools import lru_cache
|
||||
|
||||
from prometheus_client import Gauge
|
||||
|
||||
from lbry.utils import is_valid_public_ipv4 as _is_valid_public_ipv4, LRUCache
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
|
||||
|
||||
ALLOW_LOCALHOST = False
|
||||
CACHE_SIZE = 16384
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@lru_cache(1024)
|
||||
@lru_cache(CACHE_SIZE)
|
||||
def make_kademlia_peer(node_id: typing.Optional[bytes], address: typing.Optional[str],
|
||||
udp_port: typing.Optional[int] = None,
|
||||
tcp_port: typing.Optional[int] = None,
|
||||
@@ -20,40 +23,32 @@ def make_kademlia_peer(node_id: typing.Optional[bytes], address: typing.Optional
|
||||
return KademliaPeer(address, node_id, udp_port, tcp_port=tcp_port, allow_localhost=allow_localhost)
|
||||
|
||||
|
||||
# the ipaddress module does not show these subnets as reserved
|
||||
CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
|
||||
IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
|
||||
|
||||
ALLOW_LOCALHOST = False
|
||||
|
||||
|
||||
def is_valid_public_ipv4(address, allow_localhost: bool = False):
|
||||
allow_localhost = bool(allow_localhost or ALLOW_LOCALHOST)
|
||||
try:
|
||||
parsed_ip = ipaddress.ip_address(address)
|
||||
if parsed_ip.is_loopback and allow_localhost:
|
||||
return True
|
||||
return not any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local,
|
||||
parsed_ip.is_loopback, parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private,
|
||||
parsed_ip.is_reserved,
|
||||
CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
|
||||
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
|
||||
except ipaddress.AddressValueError:
|
||||
return False
|
||||
return _is_valid_public_ipv4(address, allow_localhost)
|
||||
|
||||
|
||||
class PeerManager:
|
||||
peer_manager_keys_metric = Gauge(
|
||||
"peer_manager_keys", "Number of keys tracked by PeerManager dicts (sum)", namespace="dht_node",
|
||||
labelnames=("scope",)
|
||||
)
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop):
|
||||
self._loop = loop
|
||||
self._rpc_failures: typing.Dict[
|
||||
typing.Tuple[str, int], typing.Tuple[typing.Optional[float], typing.Optional[float]]
|
||||
] = {}
|
||||
self._last_replied: typing.Dict[typing.Tuple[str, int], float] = {}
|
||||
self._last_sent: typing.Dict[typing.Tuple[str, int], float] = {}
|
||||
self._last_requested: typing.Dict[typing.Tuple[str, int], float] = {}
|
||||
self._node_id_mapping: typing.Dict[typing.Tuple[str, int], bytes] = {}
|
||||
self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = {}
|
||||
self._node_tokens: typing.Dict[bytes, (float, bytes)] = {}
|
||||
] = LRUCache(CACHE_SIZE)
|
||||
self._last_replied: typing.Dict[typing.Tuple[str, int], float] = LRUCache(CACHE_SIZE)
|
||||
self._last_sent: typing.Dict[typing.Tuple[str, int], float] = LRUCache(CACHE_SIZE)
|
||||
self._last_requested: typing.Dict[typing.Tuple[str, int], float] = LRUCache(CACHE_SIZE)
|
||||
self._node_id_mapping: typing.Dict[typing.Tuple[str, int], bytes] = LRUCache(CACHE_SIZE)
|
||||
self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = LRUCache(CACHE_SIZE)
|
||||
self._node_tokens: typing.Dict[bytes, (float, bytes)] = LRUCache(CACHE_SIZE)
|
||||
|
||||
def count_cache_keys(self):
|
||||
return len(self._rpc_failures) + len(self._last_replied) + len(self._last_sent) + len(
|
||||
self._last_requested) + len(self._node_id_mapping) + len(self._node_id_reverse_mapping) + len(
|
||||
self._node_tokens)
|
||||
|
||||
def reset(self):
|
||||
for statistic in (self._rpc_failures, self._last_replied, self._last_sent, self._last_requested):
|
||||
@@ -103,6 +98,7 @@ class PeerManager:
|
||||
self._node_id_mapping.pop(self._node_id_reverse_mapping.pop(node_id))
|
||||
self._node_id_mapping[(address, udp_port)] = node_id
|
||||
self._node_id_reverse_mapping[node_id] = (address, udp_port)
|
||||
self.peer_manager_keys_metric.labels("global").set(self.count_cache_keys())
|
||||
|
||||
def prune(self): # TODO: periodically call this
|
||||
now = self._loop.time()
|
||||
@@ -171,11 +167,11 @@ class KademliaPeer:
|
||||
def __post_init__(self):
|
||||
if self._node_id is not None:
|
||||
if not len(self._node_id) == constants.HASH_LENGTH:
|
||||
raise ValueError("invalid node_id: {}".format(hexlify(self._node_id).decode()))
|
||||
if self.udp_port is not None and not 1 <= self.udp_port <= 65535:
|
||||
raise ValueError("invalid udp port")
|
||||
if self.tcp_port is not None and not 1 <= self.tcp_port <= 65535:
|
||||
raise ValueError("invalid tcp port")
|
||||
raise ValueError("invalid node_id: {}".format(self._node_id.hex()))
|
||||
if self.udp_port is not None and not 1024 <= self.udp_port <= 65535:
|
||||
raise ValueError(f"invalid udp port: {self.address}:{self.udp_port}")
|
||||
if self.tcp_port is not None and not 1024 <= self.tcp_port <= 65535:
|
||||
raise ValueError(f"invalid tcp port: {self.address}:{self.tcp_port}")
|
||||
if not is_valid_public_ipv4(self.address, self.allow_localhost):
|
||||
raise ValueError(f"invalid ip address: '{self.address}'")
|
||||
|
||||
|
||||
@@ -16,6 +16,12 @@ class DictDataStore:
|
||||
self._peer_manager = peer_manager
|
||||
self.completed_blobs: typing.Set[str] = set()
|
||||
|
||||
def keys(self):
|
||||
return self._data_store.keys()
|
||||
|
||||
def __len__(self):
|
||||
return self._data_store.__len__()
|
||||
|
||||
def removed_expired_peers(self):
|
||||
now = self.loop.time()
|
||||
keys = list(self._data_store.keys())
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import asyncio
|
||||
from binascii import hexlify
|
||||
from itertools import chain
|
||||
from collections import defaultdict
|
||||
import typing
|
||||
@@ -196,9 +195,9 @@ class IterativeFinder:
|
||||
continue
|
||||
self._schedule_probe(peer)
|
||||
added += 1
|
||||
log.debug("running %d probes", len(self.running_probes))
|
||||
log.debug("running %d probes for key %s", len(self.running_probes), self.key.hex()[:8])
|
||||
if not added and not self.running_probes:
|
||||
log.debug("search for %s exhausted", hexlify(self.key)[:8])
|
||||
log.debug("search for %s exhausted", self.key.hex()[:8])
|
||||
self.search_exhausted()
|
||||
|
||||
def _schedule_probe(self, peer: 'KademliaPeer'):
|
||||
@@ -271,7 +270,8 @@ class IterativeNodeFinder(IterativeFinder):
|
||||
self.yielded_peers: typing.Set['KademliaPeer'] = set()
|
||||
|
||||
async def send_probe(self, peer: 'KademliaPeer') -> FindNodeResponse:
|
||||
log.debug("probing %s:%d %s", peer.address, peer.udp_port, hexlify(peer.node_id)[:8] if peer.node_id else '')
|
||||
log.debug("probe %s:%d (%s) for NODE %s",
|
||||
peer.address, peer.udp_port, peer.node_id.hex()[:8] if peer.node_id else '', self.key.hex()[:8])
|
||||
response = await self.protocol.get_rpc_peer(peer).find_node(self.key)
|
||||
return FindNodeResponse(self.key, response)
|
||||
|
||||
@@ -286,7 +286,7 @@ class IterativeNodeFinder(IterativeFinder):
|
||||
and self.peer_manager.peer_is_good(peer) is not False
|
||||
]
|
||||
not_yet_yielded.sort(key=lambda peer: self.distance(peer.node_id))
|
||||
to_yield = not_yet_yielded[:min(constants.K, len(not_yet_yielded))]
|
||||
to_yield = not_yet_yielded[:max(constants.K, self.max_results)]
|
||||
if to_yield:
|
||||
self.yielded_peers.update(to_yield)
|
||||
self.iteration_queue.put_nowait(to_yield)
|
||||
@@ -326,6 +326,8 @@ class IterativeValueFinder(IterativeFinder):
|
||||
self.discovered_peers: typing.Dict['KademliaPeer', typing.Set['KademliaPeer']] = defaultdict(set)
|
||||
|
||||
async def send_probe(self, peer: 'KademliaPeer') -> FindValueResponse:
|
||||
log.debug("probe %s:%d (%s) for VALUE %s",
|
||||
peer.address, peer.udp_port, peer.node_id.hex()[:8], self.key.hex()[:8])
|
||||
page = self.peer_pages[peer]
|
||||
response = await self.protocol.get_rpc_peer(peer).find_value(self.key, page=page)
|
||||
parsed = FindValueResponse(self.key, response)
|
||||
@@ -347,7 +349,6 @@ class IterativeValueFinder(IterativeFinder):
|
||||
already_known + len(parsed.found_compact_addresses))
|
||||
if len(self.discovered_peers[peer]) != already_known + len(parsed.found_compact_addresses):
|
||||
log.warning("misbehaving peer %s:%i returned duplicate peers for blob", peer.address, peer.udp_port)
|
||||
parsed.found_compact_addresses.clear()
|
||||
elif len(parsed.found_compact_addresses) >= constants.K and self.peer_pages[peer] < parsed.pages:
|
||||
# the peer returned a full page and indicates it has more
|
||||
self.peer_pages[peer] += 1
|
||||
|
||||
@@ -3,13 +3,16 @@ import socket
|
||||
import functools
|
||||
import hashlib
|
||||
import asyncio
|
||||
import time
|
||||
import typing
|
||||
import binascii
|
||||
import random
|
||||
from asyncio.protocols import DatagramProtocol
|
||||
from asyncio.transports import DatagramTransport
|
||||
|
||||
from prometheus_client import Gauge, Counter, Histogram
|
||||
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.serialization.bencoding import DecodeError
|
||||
from lbry.dht.serialization.datagram import decode_datagram, ErrorDatagram, ResponseDatagram, RequestDatagram
|
||||
from lbry.dht.serialization.datagram import RESPONSE_TYPE, ERROR_TYPE, PAGE_KEY
|
||||
from lbry.dht.error import RemoteException, TransportNotConnected
|
||||
@@ -30,6 +33,11 @@ OLD_PROTOCOL_ERRORS = {
|
||||
|
||||
|
||||
class KademliaRPC:
|
||||
stored_blob_metric = Gauge(
|
||||
"stored_blobs", "Number of blobs announced by other peers", namespace="dht_node",
|
||||
labelnames=("scope",),
|
||||
)
|
||||
|
||||
def __init__(self, protocol: 'KademliaProtocol', loop: asyncio.AbstractEventLoop, peer_port: int = 3333):
|
||||
self.protocol = protocol
|
||||
self.loop = loop
|
||||
@@ -61,6 +69,7 @@ class KademliaRPC:
|
||||
self.protocol.data_store.add_peer_to_blob(
|
||||
rpc_contact, blob_hash
|
||||
)
|
||||
self.stored_blob_metric.labels("global").set(len(self.protocol.data_store))
|
||||
return b'OK'
|
||||
|
||||
def find_node(self, rpc_contact: 'KademliaPeer', key: bytes) -> typing.List[typing.Tuple[bytes, str, int]]:
|
||||
@@ -96,7 +105,7 @@ class KademliaRPC:
|
||||
if not rpc_contact.tcp_port or peer.compact_address_tcp() != rpc_contact.compact_address_tcp()
|
||||
]
|
||||
# if we don't have k storing peers to return and we have this hash locally, include our contact information
|
||||
if len(peers) < constants.K and binascii.hexlify(key).decode() in self.protocol.data_store.completed_blobs:
|
||||
if len(peers) < constants.K and key.hex() in self.protocol.data_store.completed_blobs:
|
||||
peers.append(self.compact_address())
|
||||
if not peers:
|
||||
response[PAGE_KEY] = 0
|
||||
@@ -259,6 +268,30 @@ class PingQueue:
|
||||
|
||||
|
||||
class KademliaProtocol(DatagramProtocol):
|
||||
request_sent_metric = Counter(
|
||||
"request_sent", "Number of requests send from DHT RPC protocol", namespace="dht_node",
|
||||
labelnames=("method",),
|
||||
)
|
||||
request_success_metric = Counter(
|
||||
"request_success", "Number of successful requests", namespace="dht_node",
|
||||
labelnames=("method",),
|
||||
)
|
||||
request_error_metric = Counter(
|
||||
"request_error", "Number of errors returned from request to other peers", namespace="dht_node",
|
||||
labelnames=("method",),
|
||||
)
|
||||
HISTOGRAM_BUCKETS = (
|
||||
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 3.0, 3.5, 4.0, 4.50, 5.0, 5.50, 6.0, float('inf')
|
||||
)
|
||||
response_time_metric = Histogram(
|
||||
"response_time", "Response times of DHT RPC requests", namespace="dht_node", buckets=HISTOGRAM_BUCKETS,
|
||||
labelnames=("method",)
|
||||
)
|
||||
received_request_metric = Counter(
|
||||
"received_request", "Number of received DHT RPC requests", namespace="dht_node",
|
||||
labelnames=("method",),
|
||||
)
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', node_id: bytes, external_ip: str,
|
||||
udp_port: int, peer_port: int, rpc_timeout: float = constants.RPC_TIMEOUT,
|
||||
split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX):
|
||||
@@ -414,8 +447,8 @@ class KademliaProtocol(DatagramProtocol):
|
||||
self._wakeup_routing_task.clear()
|
||||
|
||||
def _handle_rpc(self, sender_contact: 'KademliaPeer', message: RequestDatagram):
|
||||
assert sender_contact.node_id != self.node_id, (binascii.hexlify(sender_contact.node_id)[:8].decode(),
|
||||
binascii.hexlify(self.node_id)[:8].decode())
|
||||
assert sender_contact.node_id != self.node_id, (sender_contact.node_id.hex()[:8],
|
||||
self.node_id.hex()[:8])
|
||||
method = message.method
|
||||
if method not in [b'ping', b'store', b'findNode', b'findValue']:
|
||||
raise AttributeError('Invalid method: %s' % message.method.decode())
|
||||
@@ -447,11 +480,16 @@ class KademliaProtocol(DatagramProtocol):
|
||||
|
||||
def handle_request_datagram(self, address: typing.Tuple[str, int], request_datagram: RequestDatagram):
|
||||
# This is an RPC method request
|
||||
self.received_request_metric.labels(method=request_datagram.method).inc()
|
||||
self.peer_manager.report_last_requested(address[0], address[1])
|
||||
try:
|
||||
peer = self.routing_table.get_peer(request_datagram.node_id)
|
||||
except IndexError:
|
||||
peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
try:
|
||||
peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
except ValueError as err:
|
||||
log.warning("error replying to %s: %s", address[0], str(err))
|
||||
return
|
||||
try:
|
||||
self._handle_rpc(peer, request_datagram)
|
||||
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
|
||||
@@ -551,12 +589,12 @@ class KademliaProtocol(DatagramProtocol):
|
||||
address[0], address[1], OLD_PROTOCOL_ERRORS[error_datagram.response]
|
||||
)
|
||||
|
||||
def datagram_received(self, datagram: bytes, address: typing.Tuple[str, int]) -> None: # pylint: disable=arguments-differ
|
||||
def datagram_received(self, datagram: bytes, address: typing.Tuple[str, int]) -> None: # pylint: disable=arguments-renamed
|
||||
try:
|
||||
message = decode_datagram(datagram)
|
||||
except (ValueError, TypeError):
|
||||
except (ValueError, TypeError, DecodeError):
|
||||
self.peer_manager.report_failure(address[0], address[1])
|
||||
log.warning("Couldn't decode dht datagram from %s: %s", address, binascii.hexlify(datagram).decode())
|
||||
log.warning("Couldn't decode dht datagram from %s: %s", address, datagram.hex())
|
||||
return
|
||||
|
||||
if isinstance(message, RequestDatagram):
|
||||
@@ -571,14 +609,19 @@ class KademliaProtocol(DatagramProtocol):
|
||||
self._send(peer, request)
|
||||
response_fut = self.sent_messages[request.rpc_id][1]
|
||||
try:
|
||||
self.request_sent_metric.labels(method=request.method).inc()
|
||||
start = time.perf_counter()
|
||||
response = await asyncio.wait_for(response_fut, self.rpc_timeout)
|
||||
self.response_time_metric.labels(method=request.method).observe(time.perf_counter() - start)
|
||||
self.peer_manager.report_last_replied(peer.address, peer.udp_port)
|
||||
self.request_success_metric.labels(method=request.method).inc()
|
||||
return response
|
||||
except asyncio.CancelledError:
|
||||
if not response_fut.done():
|
||||
response_fut.cancel()
|
||||
raise
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
self.request_error_metric.labels(method=request.method).inc()
|
||||
self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
if self.peer_manager.peer_is_good(peer) is False:
|
||||
self.remove_peer(peer)
|
||||
@@ -598,7 +641,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
if len(data) > constants.MSG_SIZE_LIMIT:
|
||||
log.warning("cannot send datagram larger than %i bytes (packet is %i bytes)",
|
||||
constants.MSG_SIZE_LIMIT, len(data))
|
||||
log.debug("Packet is too large to send: %s", binascii.hexlify(data[:3500]).decode())
|
||||
log.debug("Packet is too large to send: %s", data[:3500].hex())
|
||||
raise ValueError(
|
||||
f"cannot send datagram larger than {constants.MSG_SIZE_LIMIT} bytes (packet is {len(data)} bytes)"
|
||||
)
|
||||
@@ -658,13 +701,13 @@ class KademliaProtocol(DatagramProtocol):
|
||||
res = await self.get_rpc_peer(peer).store(hash_value)
|
||||
if res != b"OK":
|
||||
raise ValueError(res)
|
||||
log.debug("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
log.debug("Stored %s to %s", hash_value.hex()[:8], peer)
|
||||
return peer.node_id, True
|
||||
|
||||
try:
|
||||
return await __store()
|
||||
except asyncio.TimeoutError:
|
||||
log.debug("Timeout while storing blob_hash %s at %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
log.debug("Timeout while storing blob_hash %s at %s", hash_value.hex()[:8], peer)
|
||||
return peer.node_id, False
|
||||
except ValueError as err:
|
||||
log.error("Unexpected response: %s", err)
|
||||
|
||||
@@ -4,6 +4,8 @@ import logging
|
||||
import typing
|
||||
import itertools
|
||||
|
||||
from prometheus_client import Gauge
|
||||
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.protocol.distance import Distance
|
||||
if typing.TYPE_CHECKING:
|
||||
@@ -13,8 +15,17 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class KBucket:
|
||||
""" Description - later
|
||||
"""
|
||||
Kademlia K-bucket implementation.
|
||||
"""
|
||||
peer_in_routing_table_metric = Gauge(
|
||||
"peers_in_routing_table", "Number of peers on routing table", namespace="dht_node",
|
||||
labelnames=("scope",)
|
||||
)
|
||||
peer_with_x_bit_colliding_metric = Gauge(
|
||||
"peer_x_bit_colliding", "Number of peers with at least X bits colliding with this node id",
|
||||
namespace="dht_node", labelnames=("amount",)
|
||||
)
|
||||
|
||||
def __init__(self, peer_manager: 'PeerManager', range_min: int, range_max: int, node_id: bytes):
|
||||
"""
|
||||
@@ -50,7 +61,7 @@ class KBucket:
|
||||
self.peers.append(peer)
|
||||
return True
|
||||
else:
|
||||
for i in range(len(self.peers)):
|
||||
for i, _ in enumerate(self.peers):
|
||||
local_peer = self.peers[i]
|
||||
if local_peer.node_id == peer.node_id:
|
||||
self.peers.remove(local_peer)
|
||||
@@ -58,6 +69,10 @@ class KBucket:
|
||||
return True
|
||||
if len(self.peers) < constants.K:
|
||||
self.peers.append(peer)
|
||||
self.peer_in_routing_table_metric.labels("global").inc()
|
||||
if peer.node_id[0] == self._node_id[0]:
|
||||
bits_colliding = 8 - (peer.node_id[1] ^ self._node_id[1]).bit_length()
|
||||
self.peer_with_x_bit_colliding_metric.labels(amount=(bits_colliding + 8)).inc()
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
@@ -124,6 +139,10 @@ class KBucket:
|
||||
|
||||
def remove_peer(self, peer: 'KademliaPeer') -> None:
|
||||
self.peers.remove(peer)
|
||||
self.peer_in_routing_table_metric.labels("global").dec()
|
||||
if peer.node_id[0] == self._node_id[0]:
|
||||
bits_colliding = 8 - (peer.node_id[1] ^ self._node_id[1]).bit_length()
|
||||
self.peer_with_x_bit_colliding_metric.labels(amount=(bits_colliding + 8)).dec()
|
||||
|
||||
def key_in_range(self, key: bytes) -> bool:
|
||||
""" Tests whether the specified key (i.e. node ID) is in the range
|
||||
@@ -162,6 +181,10 @@ class TreeRoutingTable:
|
||||
ping RPC-based k-bucket eviction algorithm described in section 2.2 of
|
||||
that paper.
|
||||
"""
|
||||
bucket_in_routing_table_metric = Gauge(
|
||||
"buckets_in_routing_table", "Number of buckets on routing table", namespace="dht_node",
|
||||
labelnames=("scope",)
|
||||
)
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', parent_node_id: bytes,
|
||||
split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX):
|
||||
@@ -279,6 +302,7 @@ class TreeRoutingTable:
|
||||
# ...and remove them from the old bucket
|
||||
for contact in new_bucket.peers:
|
||||
old_bucket.remove_peer(contact)
|
||||
self.bucket_in_routing_table_metric.labels("global").set(len(self.buckets))
|
||||
|
||||
def join_buckets(self):
|
||||
if len(self.buckets) == 1:
|
||||
@@ -302,6 +326,7 @@ class TreeRoutingTable:
|
||||
elif can_go_higher:
|
||||
self.buckets[bucket_index_to_pop + 1].range_min = bucket.range_min
|
||||
self.buckets.remove(bucket)
|
||||
self.bucket_in_routing_table_metric.labels("global").set(len(self.buckets))
|
||||
return self.join_buckets()
|
||||
|
||||
def contact_in_routing_table(self, address_tuple: typing.Tuple[str, int]) -> bool:
|
||||
|
||||
@@ -144,7 +144,7 @@ class ErrorDatagram(KademliaDatagramBase):
|
||||
self.response = response.decode()
|
||||
|
||||
|
||||
def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]:
|
||||
def _decode_datagram(datagram: bytes):
|
||||
msg_types = {
|
||||
REQUEST_TYPE: RequestDatagram,
|
||||
RESPONSE_TYPE: ResponseDatagram,
|
||||
@@ -152,26 +152,36 @@ def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDa
|
||||
}
|
||||
|
||||
primitive: typing.Dict = bdecode(datagram)
|
||||
if primitive[0] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
|
||||
datagram_type = primitive[0] # pylint: disable=unsubscriptable-object
|
||||
|
||||
converted = {
|
||||
str(k).encode() if not isinstance(k, bytes) else k: v for k, v in primitive.items()
|
||||
}
|
||||
|
||||
if converted[b'0'] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
|
||||
datagram_type = converted[b'0'] # pylint: disable=unsubscriptable-object
|
||||
else:
|
||||
raise ValueError("invalid datagram type")
|
||||
datagram_class = msg_types[datagram_type]
|
||||
decoded = {
|
||||
k: primitive[i] # pylint: disable=unsubscriptable-object
|
||||
k: converted[str(i).encode()] # pylint: disable=unsubscriptable-object
|
||||
for i, k in enumerate(datagram_class.required_fields)
|
||||
if i in primitive # pylint: disable=unsupported-membership-test
|
||||
if str(i).encode() in converted # pylint: disable=unsupported-membership-test
|
||||
}
|
||||
for i, _ in enumerate(OPTIONAL_FIELDS):
|
||||
if i + OPTIONAL_ARG_OFFSET in primitive:
|
||||
decoded[i + OPTIONAL_ARG_OFFSET] = primitive[i + OPTIONAL_ARG_OFFSET]
|
||||
if str(i + OPTIONAL_ARG_OFFSET).encode() in converted:
|
||||
decoded[i + OPTIONAL_ARG_OFFSET] = converted[str(i + OPTIONAL_ARG_OFFSET).encode()]
|
||||
return decoded, datagram_class
|
||||
|
||||
|
||||
def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]:
|
||||
decoded, datagram_class = _decode_datagram(datagram)
|
||||
return datagram_class(**decoded)
|
||||
|
||||
|
||||
def make_compact_ip(address: str) -> bytearray:
|
||||
compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), address.split('.'), bytearray())
|
||||
if len(compact_ip) != 4:
|
||||
raise ValueError(f"invalid IPv4 length")
|
||||
raise ValueError("invalid IPv4 length")
|
||||
return compact_ip
|
||||
|
||||
|
||||
@@ -180,7 +190,7 @@ def make_compact_address(node_id: bytes, address: str, port: int) -> bytearray:
|
||||
if not 0 < port < 65536:
|
||||
raise ValueError(f'Invalid port: {port}')
|
||||
if len(node_id) != constants.HASH_BITS // 8:
|
||||
raise ValueError(f"invalid node node_id length")
|
||||
raise ValueError("invalid node node_id length")
|
||||
return compact_ip + port.to_bytes(2, 'big') + node_id
|
||||
|
||||
|
||||
@@ -191,5 +201,5 @@ def decode_compact_address(compact_address: bytes) -> typing.Tuple[bytes, str, i
|
||||
if not 0 < port < 65536:
|
||||
raise ValueError(f'Invalid port: {port}')
|
||||
if len(node_id) != constants.HASH_BITS // 8:
|
||||
raise ValueError(f"invalid node node_id length")
|
||||
raise ValueError("invalid node node_id length")
|
||||
return node_id, address, port
|
||||
+13
-1
@@ -34,6 +34,11 @@ Code | Name | Message
|
||||
**11x** | InputValue(ValueError) | Invalid argument value provided to command.
|
||||
111 | GenericInputValue | The value '{value}' for argument '{argument}' is not valid.
|
||||
112 | InputValueIsNone | None or null is not valid value for argument '{argument}'.
|
||||
113 | ConflictingInputValue | Only '{first_argument}' or '{second_argument}' is allowed, not both.
|
||||
114 | InputStringIsBlank | {argument} cannot be blank.
|
||||
115 | EmptyPublishedFile | Cannot publish empty file: {file_path}
|
||||
116 | MissingPublishedFile | File does not exist: {file_path}
|
||||
117 | InvalidStreamURL | Invalid LBRY stream URL: '{url}' -- When an URL cannot be downloaded, such as '@Channel/' or a collection
|
||||
**2xx** | Configuration | Configuration errors.
|
||||
201 | ConfigWrite | Cannot write configuration file '{path}'. -- When writing the default config fails on startup, such as due to permission issues.
|
||||
202 | ConfigRead | Cannot find provided configuration file '{path}'. -- Can't open the config file user provided via command line args.
|
||||
@@ -51,15 +56,22 @@ Code | Name | Message
|
||||
405 | ChannelKeyNotFound | Channel signing key not found.
|
||||
406 | ChannelKeyInvalid | Channel signing key is out of date. -- For example, channel was updated but you don't have the updated key.
|
||||
407 | DataDownload | Failed to download blob. *generic*
|
||||
408 | PrivateKeyNotFound | Couldn't find private key for {key} '{value}'.
|
||||
410 | Resolve | Failed to resolve '{url}'.
|
||||
411 | ResolveTimeout | Failed to resolve '{url}' within the timeout.
|
||||
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{claim_id(censor_hash)}'.
|
||||
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{censor_id}'.
|
||||
420 | KeyFeeAboveMaxAllowed | {message}
|
||||
421 | InvalidPassword | Password is invalid.
|
||||
422 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version.
|
||||
423 | TooManyClaimSearchParameters | {key} cant have more than {limit} items.
|
||||
424 | AlreadyPurchased | You already have a purchase for claim_id '{claim_id_hex}'. Use --allow-duplicate-purchase flag to override.
|
||||
431 | ServerPaymentInvalidAddress | Invalid address from wallet server: '{address}' - skipping payment round.
|
||||
432 | ServerPaymentWalletLocked | Cannot spend funds with locked wallet, skipping payment round.
|
||||
433 | ServerPaymentFeeAboveMaxAllowed | Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.
|
||||
434 | WalletNotLoaded | Wallet {wallet_id} is not loaded.
|
||||
435 | WalletAlreadyLoaded | Wallet {wallet_path} is already loaded.
|
||||
436 | WalletNotFound | Wallet not found at {wallet_path}.
|
||||
437 | WalletAlreadyExists | Wallet {wallet_path} already exists, use `wallet_add` to load it.
|
||||
**5xx** | Blob | **Blobs**
|
||||
500 | BlobNotFound | Blob not found.
|
||||
501 | BlobPermissionDenied | Permission denied to read blob.
|
||||
|
||||
+97
-3
@@ -76,6 +76,45 @@ class InputValueIsNoneError(InputValueError):
|
||||
super().__init__(f"None or null is not valid value for argument '{argument}'.")
|
||||
|
||||
|
||||
class ConflictingInputValueError(InputValueError):
|
||||
|
||||
def __init__(self, first_argument, second_argument):
|
||||
self.first_argument = first_argument
|
||||
self.second_argument = second_argument
|
||||
super().__init__(f"Only '{first_argument}' or '{second_argument}' is allowed, not both.")
|
||||
|
||||
|
||||
class InputStringIsBlankError(InputValueError):
|
||||
|
||||
def __init__(self, argument):
|
||||
self.argument = argument
|
||||
super().__init__(f"{argument} cannot be blank.")
|
||||
|
||||
|
||||
class EmptyPublishedFileError(InputValueError):
|
||||
|
||||
def __init__(self, file_path):
|
||||
self.file_path = file_path
|
||||
super().__init__(f"Cannot publish empty file: {file_path}")
|
||||
|
||||
|
||||
class MissingPublishedFileError(InputValueError):
|
||||
|
||||
def __init__(self, file_path):
|
||||
self.file_path = file_path
|
||||
super().__init__(f"File does not exist: {file_path}")
|
||||
|
||||
|
||||
class InvalidStreamURLError(InputValueError):
|
||||
"""
|
||||
When an URL cannot be downloaded, such as '@Channel/' or a collection
|
||||
"""
|
||||
|
||||
def __init__(self, url):
|
||||
self.url = url
|
||||
super().__init__(f"Invalid LBRY stream URL: '{url}'")
|
||||
|
||||
|
||||
class ConfigurationError(BaseError):
|
||||
"""
|
||||
Configuration errors.
|
||||
@@ -199,6 +238,14 @@ class DataDownloadError(WalletError):
|
||||
super().__init__("Failed to download blob. *generic*")
|
||||
|
||||
|
||||
class PrivateKeyNotFoundError(WalletError):
|
||||
|
||||
def __init__(self, key, value):
|
||||
self.key = key
|
||||
self.value = value
|
||||
super().__init__(f"Couldn't find private key for {key} '{value}'.")
|
||||
|
||||
|
||||
class ResolveError(WalletError):
|
||||
|
||||
def __init__(self, url):
|
||||
@@ -215,10 +262,11 @@ class ResolveTimeoutError(WalletError):
|
||||
|
||||
class ResolveCensoredError(WalletError):
|
||||
|
||||
def __init__(self, url, censor_hash):
|
||||
def __init__(self, url, censor_id, censor_row):
|
||||
self.url = url
|
||||
self.censor_hash = censor_hash
|
||||
super().__init__(f"Resolve of '{url}' was censored by channel with claim id '{claim_id(censor_hash)}'.")
|
||||
self.censor_id = censor_id
|
||||
self.censor_row = censor_row
|
||||
super().__init__(f"Resolve of '{url}' was censored by channel with claim id '{censor_id}'.")
|
||||
|
||||
|
||||
class KeyFeeAboveMaxAllowedError(WalletError):
|
||||
@@ -242,6 +290,24 @@ class IncompatibleWalletServerError(WalletError):
|
||||
super().__init__(f"'{server}:{port}' has an incompatibly old version.")
|
||||
|
||||
|
||||
class TooManyClaimSearchParametersError(WalletError):
|
||||
|
||||
def __init__(self, key, limit):
|
||||
self.key = key
|
||||
self.limit = limit
|
||||
super().__init__(f"{key} cant have more than {limit} items.")
|
||||
|
||||
|
||||
class AlreadyPurchasedError(WalletError):
|
||||
"""
|
||||
allow-duplicate-purchase flag to override.
|
||||
"""
|
||||
|
||||
def __init__(self, claim_id_hex):
|
||||
self.claim_id_hex = claim_id_hex
|
||||
super().__init__(f"You already have a purchase for claim_id '{claim_id_hex}'. Use")
|
||||
|
||||
|
||||
class ServerPaymentInvalidAddressError(WalletError):
|
||||
|
||||
def __init__(self, address):
|
||||
@@ -263,6 +329,34 @@ class ServerPaymentFeeAboveMaxAllowedError(WalletError):
|
||||
super().__init__(f"Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.")
|
||||
|
||||
|
||||
class WalletNotLoadedError(WalletError):
|
||||
|
||||
def __init__(self, wallet_id):
|
||||
self.wallet_id = wallet_id
|
||||
super().__init__(f"Wallet {wallet_id} is not loaded.")
|
||||
|
||||
|
||||
class WalletAlreadyLoadedError(WalletError):
|
||||
|
||||
def __init__(self, wallet_path):
|
||||
self.wallet_path = wallet_path
|
||||
super().__init__(f"Wallet {wallet_path} is already loaded.")
|
||||
|
||||
|
||||
class WalletNotFoundError(WalletError):
|
||||
|
||||
def __init__(self, wallet_path):
|
||||
self.wallet_path = wallet_path
|
||||
super().__init__(f"Wallet not found at {wallet_path}.")
|
||||
|
||||
|
||||
class WalletAlreadyExistsError(WalletError):
|
||||
|
||||
def __init__(self, wallet_path):
|
||||
self.wallet_path = wallet_path
|
||||
super().__init__(f"Wallet {wallet_path} already exists, use `wallet_add` to load it.")
|
||||
|
||||
|
||||
class BlobError(BaseError):
|
||||
"""
|
||||
**Blobs**
|
||||
|
||||
@@ -63,7 +63,7 @@ class ErrorClass:
|
||||
@staticmethod
|
||||
def get_fields(args):
|
||||
if len(args) > 1:
|
||||
return f''.join(f'\n{INDENT*2}self.{field} = {field}' for field in args[1:])
|
||||
return ''.join(f'\n{INDENT*2}self.{field} = {field}' for field in args[1:])
|
||||
return ''
|
||||
|
||||
@staticmethod
|
||||
|
||||
+4
-10
@@ -3,7 +3,6 @@ import sys
|
||||
import shutil
|
||||
import signal
|
||||
import pathlib
|
||||
import platform
|
||||
import json
|
||||
import asyncio
|
||||
import argparse
|
||||
@@ -15,7 +14,6 @@ from aiohttp.web import GracefulExit
|
||||
from docopt import docopt
|
||||
|
||||
from lbry import __version__ as lbrynet_version
|
||||
from lbry.extras.daemon.loggly_handler import get_loggly_handler
|
||||
from lbry.extras.daemon.daemon import Daemon
|
||||
from lbry.conf import Config, CLIConfig
|
||||
|
||||
@@ -103,7 +101,7 @@ class ArgumentParser(argparse.ArgumentParser):
|
||||
self._optionals.title = 'Options'
|
||||
if group_name is None:
|
||||
self.epilog = (
|
||||
f"Run 'lbrynet COMMAND --help' for more information on a command or group."
|
||||
"Run 'lbrynet COMMAND --help' for more information on a command or group."
|
||||
)
|
||||
else:
|
||||
self.epilog = (
|
||||
@@ -228,6 +226,9 @@ def get_argument_parser():
|
||||
def ensure_directory_exists(path: str):
|
||||
if not os.path.isdir(path):
|
||||
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
|
||||
use_effective_ids = os.access in os.supports_effective_ids
|
||||
if not os.access(path, os.W_OK, effective_ids=use_effective_ids):
|
||||
raise PermissionError(f"The following directory is not writable: {path}")
|
||||
|
||||
|
||||
LOG_MODULES = 'lbry', 'aioupnp'
|
||||
@@ -256,15 +257,8 @@ def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config
|
||||
else:
|
||||
logger.getChild('lbry').setLevel(logging.DEBUG)
|
||||
|
||||
loggly_handler = get_loggly_handler(conf)
|
||||
loggly_handler.setLevel(logging.ERROR)
|
||||
logger.getChild('lbry').addHandler(loggly_handler)
|
||||
|
||||
|
||||
def run_daemon(args: argparse.Namespace, conf: Config):
|
||||
if sys.version_info < (3, 8) and platform.system() == "Windows":
|
||||
# TODO: remove after we move to requiring Python 3.8
|
||||
asyncio.set_event_loop(asyncio.ProactorEventLoop())
|
||||
loop = asyncio.get_event_loop()
|
||||
if args.verbose is not None:
|
||||
loop.set_debug(True)
|
||||
|
||||
@@ -18,6 +18,7 @@ DOWNLOAD_STARTED = 'Download Started'
|
||||
DOWNLOAD_ERRORED = 'Download Errored'
|
||||
DOWNLOAD_FINISHED = 'Download Finished'
|
||||
HEARTBEAT = 'Heartbeat'
|
||||
DISK_SPACE = 'Disk Space'
|
||||
CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon
|
||||
NEW_CHANNEL = 'New Channel'
|
||||
CREDITS_SENT = 'Credits Sent'
|
||||
@@ -66,7 +67,7 @@ def _download_properties(conf: Config, external_ip: str, resolve_duration: float
|
||||
"node_rpc_timeout": conf.node_rpc_timeout,
|
||||
"peer_connect_timeout": conf.peer_connect_timeout,
|
||||
"blob_download_timeout": conf.blob_download_timeout,
|
||||
"use_fixed_peers": len(conf.reflector_servers) > 0,
|
||||
"use_fixed_peers": len(conf.fixed_peers) > 0,
|
||||
"fixed_peer_delay": fixed_peer_delay,
|
||||
"added_fixed_peers": added_fixed_peers,
|
||||
"active_peer_count": active_peer_count,
|
||||
@@ -132,7 +133,7 @@ class AnalyticsManager:
|
||||
async def run(self):
|
||||
while True:
|
||||
if self.enabled:
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
await self._send_heartbeat()
|
||||
await asyncio.sleep(1800)
|
||||
|
||||
@@ -169,6 +170,15 @@ class AnalyticsManager:
|
||||
})
|
||||
)
|
||||
|
||||
async def send_disk_space_used(self, storage_used, storage_limit, is_from_network_quota):
|
||||
await self.track(
|
||||
self._event(DISK_SPACE, {
|
||||
'used': storage_used,
|
||||
'limit': storage_limit,
|
||||
'from_network_quota': is_from_network_quota
|
||||
})
|
||||
)
|
||||
|
||||
async def send_server_startup(self):
|
||||
await self.track(self._event(SERVER_STARTUP))
|
||||
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
import logging
|
||||
import time
|
||||
import hashlib
|
||||
import binascii
|
||||
|
||||
import ecdsa
|
||||
from lbry import utils
|
||||
from lbry.crypto.hash import sha256
|
||||
from lbry.wallet.transaction import Output
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_encoded_signature(signature):
|
||||
signature = signature.encode() if isinstance(signature, str) else signature
|
||||
r = int(signature[:int(len(signature) / 2)], 16)
|
||||
s = int(signature[int(len(signature) / 2):], 16)
|
||||
return ecdsa.util.sigencode_der(r, s, len(signature) * 4)
|
||||
|
||||
|
||||
def cid2hash(claim_id: str) -> bytes:
|
||||
return binascii.unhexlify(claim_id.encode())[::-1]
|
||||
|
||||
|
||||
def is_comment_signed_by_channel(comment: dict, channel: Output, abandon=False):
|
||||
if isinstance(channel, Output):
|
||||
try:
|
||||
signing_field = comment['comment_id'] if abandon else comment['comment']
|
||||
pieces = [
|
||||
comment['signing_ts'].encode(),
|
||||
cid2hash(comment['channel_id']),
|
||||
signing_field.encode()
|
||||
]
|
||||
return Output.is_signature_valid(
|
||||
get_encoded_signature(comment['signature']),
|
||||
sha256(b''.join(pieces)),
|
||||
channel.claim.channel.public_key_bytes
|
||||
)
|
||||
except KeyError:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def sign_comment(comment: dict, channel: Output, abandon=False):
|
||||
timestamp = str(int(time.time()))
|
||||
signing_field = comment['comment_id'] if abandon else comment['comment']
|
||||
pieces = [timestamp.encode(), channel.claim_hash, signing_field.encode()]
|
||||
digest = sha256(b''.join(pieces))
|
||||
signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
|
||||
comment.update({
|
||||
'signature': binascii.hexlify(signature).decode(),
|
||||
'signing_ts': timestamp
|
||||
})
|
||||
|
||||
|
||||
async def jsonrpc_post(url: str, method: str, params: dict = None, **kwargs) -> any:
|
||||
params = params or {}
|
||||
params.update(kwargs)
|
||||
json_body = {'jsonrpc': '2.0', 'id': None, 'method': method, 'params': params}
|
||||
async with utils.aiohttp_request('POST', url, json=json_body) as response:
|
||||
try:
|
||||
result = await response.json()
|
||||
return result['result'] if 'result' in result else result
|
||||
except Exception as cte:
|
||||
log.exception('Unable to decode response from server: %s', cte)
|
||||
return await response.text()
|
||||
@@ -158,11 +158,14 @@ class ComponentManager:
|
||||
for component in self.components
|
||||
}
|
||||
|
||||
def get_component(self, component_name):
|
||||
def get_actual_component(self, component_name):
|
||||
for component in self.components:
|
||||
if component.component_name == component_name:
|
||||
return component.component
|
||||
return component
|
||||
raise NameError(component_name)
|
||||
|
||||
def get_component(self, component_name):
|
||||
return self.get_actual_component(component_name).component
|
||||
|
||||
def has_component(self, component_name):
|
||||
return any(component for component in self.components if component_name == component.component_name)
|
||||
@@ -4,6 +4,7 @@ import asyncio
|
||||
import logging
|
||||
import binascii
|
||||
import typing
|
||||
|
||||
import base58
|
||||
|
||||
from aioupnp import __version__ as aioupnp_version
|
||||
@@ -15,13 +16,21 @@ from lbry.dht.node import Node
|
||||
from lbry.dht.peer import is_valid_public_ipv4
|
||||
from lbry.dht.blob_announcer import BlobAnnouncer
|
||||
from lbry.blob.blob_manager import BlobManager
|
||||
from lbry.blob.disk_space_manager import DiskSpaceManager
|
||||
from lbry.blob_exchange.server import BlobServer
|
||||
from lbry.stream.background_downloader import BackgroundDownloader
|
||||
from lbry.stream.stream_manager import StreamManager
|
||||
from lbry.file.file_manager import FileManager
|
||||
from lbry.extras.daemon.component import Component
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from lbry.torrent.torrent_manager import TorrentManager
|
||||
from lbry.wallet import WalletManager
|
||||
from lbry.wallet.usage_payment import WalletServerPayer
|
||||
try:
|
||||
from lbry.torrent.session import TorrentSession
|
||||
except ImportError:
|
||||
TorrentSession = None
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -33,10 +42,13 @@ WALLET_COMPONENT = "wallet"
|
||||
WALLET_SERVER_PAYMENTS_COMPONENT = "wallet_server_payments"
|
||||
DHT_COMPONENT = "dht"
|
||||
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
|
||||
STREAM_MANAGER_COMPONENT = "stream_manager"
|
||||
FILE_MANAGER_COMPONENT = "file_manager"
|
||||
DISK_SPACE_COMPONENT = "disk_space"
|
||||
BACKGROUND_DOWNLOADER_COMPONENT = "background_downloader"
|
||||
PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
|
||||
UPNP_COMPONENT = "upnp"
|
||||
EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager"
|
||||
LIBTORRENT_COMPONENT = "libtorrent_component"
|
||||
|
||||
|
||||
class DatabaseComponent(Component):
|
||||
@@ -52,7 +64,7 @@ class DatabaseComponent(Component):
|
||||
|
||||
@staticmethod
|
||||
def get_current_db_revision():
|
||||
return 14
|
||||
return 15
|
||||
|
||||
@property
|
||||
def revision_filename(self):
|
||||
@@ -112,13 +124,14 @@ class WalletComponent(Component):
|
||||
async def get_status(self):
|
||||
if self.wallet_manager is None:
|
||||
return
|
||||
session_pool = self.wallet_manager.ledger.network.session_pool
|
||||
sessions = session_pool.sessions
|
||||
is_connected = self.wallet_manager.ledger.network.is_connected
|
||||
sessions = []
|
||||
connected = None
|
||||
if self.wallet_manager.ledger.network.client:
|
||||
addr_and_port = self.wallet_manager.ledger.network.client.server_address_and_port
|
||||
if addr_and_port:
|
||||
connected = f"{addr_and_port[0]}:{addr_and_port[1]}"
|
||||
if is_connected:
|
||||
addr, port = self.wallet_manager.ledger.network.client.server
|
||||
connected = f"{addr}:{port}"
|
||||
sessions.append(self.wallet_manager.ledger.network.client)
|
||||
|
||||
result = {
|
||||
'connected': connected,
|
||||
'connected_features': self.wallet_manager.ledger.network.server_features,
|
||||
@@ -130,8 +143,8 @@ class WalletComponent(Component):
|
||||
'availability': session.available,
|
||||
} for session in sessions
|
||||
],
|
||||
'known_servers': len(sessions),
|
||||
'available_servers': len(list(session_pool.available_sessions))
|
||||
'known_servers': len(self.wallet_manager.ledger.network.known_hubs),
|
||||
'available_servers': 1 if is_connected else 0
|
||||
}
|
||||
|
||||
if self.wallet_manager.ledger.network.remote_height:
|
||||
@@ -143,7 +156,7 @@ class WalletComponent(Component):
|
||||
progress = min(max(math.ceil(float(download_height) / float(target_height) * 100), 0), 100)
|
||||
else:
|
||||
progress = 100
|
||||
best_hash = self.wallet_manager.get_best_blockhash()
|
||||
best_hash = await self.wallet_manager.get_best_blockhash()
|
||||
result.update({
|
||||
'headers_synchronization_progress': progress,
|
||||
'blocks': max(local_height, 0),
|
||||
@@ -267,7 +280,7 @@ class DHTComponent(Component):
|
||||
external_ip = upnp_component.external_ip
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
if not external_ip:
|
||||
external_ip = await utils.get_external_ip()
|
||||
external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
if not external_ip:
|
||||
log.warning("failed to get external ip")
|
||||
|
||||
@@ -319,23 +332,23 @@ class HashAnnouncerComponent(Component):
|
||||
}
|
||||
|
||||
|
||||
class StreamManagerComponent(Component):
|
||||
component_name = STREAM_MANAGER_COMPONENT
|
||||
class FileManagerComponent(Component):
|
||||
component_name = FILE_MANAGER_COMPONENT
|
||||
depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.stream_manager: typing.Optional[StreamManager] = None
|
||||
self.file_manager: typing.Optional[FileManager] = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[StreamManager]:
|
||||
return self.stream_manager
|
||||
def component(self) -> typing.Optional[FileManager]:
|
||||
return self.file_manager
|
||||
|
||||
async def get_status(self):
|
||||
if not self.stream_manager:
|
||||
if not self.file_manager:
|
||||
return
|
||||
return {
|
||||
'managed_files': len(self.stream_manager.streams),
|
||||
'managed_files': len(self.file_manager.get_filtered()),
|
||||
}
|
||||
|
||||
async def start(self):
|
||||
@@ -344,16 +357,147 @@ class StreamManagerComponent(Component):
|
||||
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
node = self.component_manager.get_component(DHT_COMPONENT) \
|
||||
if self.component_manager.has_component(DHT_COMPONENT) else None
|
||||
try:
|
||||
torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT) if TorrentSession else None
|
||||
except NameError:
|
||||
torrent = None
|
||||
log.info('Starting the file manager')
|
||||
loop = asyncio.get_event_loop()
|
||||
self.stream_manager = StreamManager(
|
||||
loop, self.conf, blob_manager, wallet, storage, node, self.component_manager.analytics_manager
|
||||
self.file_manager = FileManager(
|
||||
loop, self.conf, wallet, storage, self.component_manager.analytics_manager
|
||||
)
|
||||
await self.stream_manager.start()
|
||||
self.file_manager.source_managers['stream'] = StreamManager(
|
||||
loop, self.conf, blob_manager, wallet, storage, node,
|
||||
)
|
||||
if TorrentSession and LIBTORRENT_COMPONENT not in self.conf.components_to_skip:
|
||||
self.file_manager.source_managers['torrent'] = TorrentManager(
|
||||
loop, self.conf, torrent, storage, self.component_manager.analytics_manager
|
||||
)
|
||||
await self.file_manager.start()
|
||||
log.info('Done setting up file manager')
|
||||
|
||||
async def stop(self):
|
||||
self.stream_manager.stop()
|
||||
self.file_manager.stop()
|
||||
|
||||
|
||||
class BackgroundDownloaderComponent(Component):
|
||||
component_name = BACKGROUND_DOWNLOADER_COMPONENT
|
||||
depends_on = [DATABASE_COMPONENT, BLOB_COMPONENT, DISK_SPACE_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.background_task: typing.Optional[asyncio.Task] = None
|
||||
self.download_loop_delay_seconds = 60
|
||||
self.ongoing_download: typing.Optional[asyncio.Task] = None
|
||||
self.space_manager: typing.Optional[DiskSpaceManager] = None
|
||||
self.blob_manager: typing.Optional[BlobManager] = None
|
||||
self.background_downloader: typing.Optional[BackgroundDownloader] = None
|
||||
self.dht_node: typing.Optional[Node] = None
|
||||
self.space_available: typing.Optional[int] = None
|
||||
|
||||
@property
|
||||
def is_busy(self):
|
||||
return bool(self.ongoing_download and not self.ongoing_download.done())
|
||||
|
||||
@property
|
||||
def component(self) -> 'BackgroundDownloaderComponent':
|
||||
return self
|
||||
|
||||
async def get_status(self):
|
||||
return {'running': self.background_task is not None and not self.background_task.done(),
|
||||
'available_free_space_mb': self.space_available,
|
||||
'ongoing_download': self.is_busy}
|
||||
|
||||
async def download_blobs_in_background(self):
|
||||
while True:
|
||||
self.space_available = await self.space_manager.get_free_space_mb(True)
|
||||
if not self.is_busy and self.space_available > 10:
|
||||
blob_hash = next((key.hex() for key in self.dht_node.stored_blob_hashes if
|
||||
key.hex() not in self.blob_manager.completed_blob_hashes), None)
|
||||
if blob_hash:
|
||||
self.ongoing_download = asyncio.create_task(self.background_downloader.download_blobs(blob_hash))
|
||||
await asyncio.sleep(self.download_loop_delay_seconds)
|
||||
|
||||
async def start(self):
|
||||
self.space_manager: DiskSpaceManager = self.component_manager.get_component(DISK_SPACE_COMPONENT)
|
||||
if not self.component_manager.has_component(DHT_COMPONENT):
|
||||
return
|
||||
self.dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
self.blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
self.background_downloader = BackgroundDownloader(self.conf, storage, self.blob_manager, self.dht_node)
|
||||
self.background_task = asyncio.create_task(self.download_blobs_in_background())
|
||||
|
||||
async def stop(self):
|
||||
if self.ongoing_download and not self.ongoing_download.done():
|
||||
self.ongoing_download.cancel()
|
||||
if self.background_task:
|
||||
self.background_task.cancel()
|
||||
|
||||
|
||||
class DiskSpaceComponent(Component):
|
||||
component_name = DISK_SPACE_COMPONENT
|
||||
depends_on = [DATABASE_COMPONENT, BLOB_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.disk_space_manager: typing.Optional[DiskSpaceManager] = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[DiskSpaceManager]:
|
||||
return self.disk_space_manager
|
||||
|
||||
async def get_status(self):
|
||||
if self.disk_space_manager:
|
||||
space_used = await self.disk_space_manager.get_space_used_mb(cached=True)
|
||||
return {
|
||||
'total_used_mb': space_used['total'],
|
||||
'published_blobs_storage_used_mb': space_used['private_storage'],
|
||||
'content_blobs_storage_used_mb': space_used['content_storage'],
|
||||
'seed_blobs_storage_used_mb': space_used['network_storage'],
|
||||
'running': self.disk_space_manager.running,
|
||||
}
|
||||
return {'space_used': '0', 'network_seeding_space_used': '0', 'running': False}
|
||||
|
||||
async def start(self):
|
||||
db = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
self.disk_space_manager = DiskSpaceManager(
|
||||
self.conf, db, blob_manager,
|
||||
analytics=self.component_manager.analytics_manager
|
||||
)
|
||||
await self.disk_space_manager.start()
|
||||
|
||||
async def stop(self):
|
||||
await self.disk_space_manager.stop()
|
||||
|
||||
|
||||
class TorrentComponent(Component):
|
||||
component_name = LIBTORRENT_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.torrent_session = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[TorrentSession]:
|
||||
return self.torrent_session
|
||||
|
||||
async def get_status(self):
|
||||
if not self.torrent_session:
|
||||
return
|
||||
return {
|
||||
'running': True, # TODO: what to return here?
|
||||
}
|
||||
|
||||
async def start(self):
|
||||
if TorrentSession:
|
||||
self.torrent_session = TorrentSession(asyncio.get_event_loop(), None)
|
||||
await self.torrent_session.bind() # TODO: specify host/port
|
||||
|
||||
async def stop(self):
|
||||
if self.torrent_session:
|
||||
await self.torrent_session.pause()
|
||||
|
||||
|
||||
class PeerProtocolServerComponent(Component):
|
||||
@@ -429,11 +573,15 @@ class UPnPComponent(Component):
|
||||
pass
|
||||
if external_ip and not is_valid_public_ipv4(external_ip):
|
||||
log.warning("UPnP returned a private/reserved ip - %s, checking lbry.com fallback", external_ip)
|
||||
external_ip = await utils.get_external_ip()
|
||||
external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
if self.external_ip and self.external_ip != external_ip:
|
||||
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
|
||||
if external_ip:
|
||||
self.external_ip = external_ip
|
||||
dht_component = self.component_manager.get_component(DHT_COMPONENT)
|
||||
if dht_component:
|
||||
dht_node = dht_component.component
|
||||
dht_node.protocol.external_ip = external_ip
|
||||
# assert self.external_ip is not None # TODO: handle going/starting offline
|
||||
|
||||
if not self.upnp_redirects and self.upnp: # setup missing redirects
|
||||
@@ -487,13 +635,15 @@ class UPnPComponent(Component):
|
||||
async def start(self):
|
||||
log.info("detecting external ip")
|
||||
if not self.use_upnp:
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
return
|
||||
success = False
|
||||
await self._maintain_redirects()
|
||||
if self.upnp:
|
||||
if not self.upnp_redirects and not all([x in self.component_manager.skip_components for x in
|
||||
(DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT)]):
|
||||
if not self.upnp_redirects and not all(
|
||||
x in self.component_manager.skip_components
|
||||
for x in (DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT)
|
||||
):
|
||||
log.error("failed to setup upnp")
|
||||
else:
|
||||
success = True
|
||||
@@ -502,9 +652,9 @@ class UPnPComponent(Component):
|
||||
else:
|
||||
log.error("failed to setup upnp")
|
||||
if not self.external_ip:
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
self.external_ip, probed_url = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
if self.external_ip:
|
||||
log.info("detected external ip using lbry.com fallback")
|
||||
log.info("detected external ip using %s fallback", probed_url)
|
||||
if self.component_manager.analytics_manager:
|
||||
self.component_manager.loop.create_task(
|
||||
self.component_manager.analytics_manager.send_upnp_setup_success_fail(
|
||||
|
||||
+1139
-600
File diff suppressed because it is too large.
Load diff
@@ -2,9 +2,10 @@ import json
|
||||
import time
|
||||
import asyncio
|
||||
import logging
|
||||
from statistics import median
|
||||
from decimal import Decimal
|
||||
from typing import Optional, Iterable, Type
|
||||
from aiohttp.client_exceptions import ContentTypeError
|
||||
from aiohttp.client_exceptions import ContentTypeError, ClientConnectionError
|
||||
from lbry.error import InvalidExchangeRateResponseError, CurrencyConversionError
|
||||
from lbry.utils import aiohttp_request
|
||||
from lbry.wallet.dewies import lbc_to_dewies
|
||||
@@ -58,9 +59,12 @@ class MarketFeed:
|
||||
raise NotImplementedError()
|
||||
|
||||
async def get_response(self):
|
||||
async with aiohttp_request('get', self.url, params=self.params, timeout=self.request_timeout) as response:
|
||||
async with aiohttp_request(
|
||||
'get', self.url, params=self.params,
|
||||
timeout=self.request_timeout, headers={"User-Agent": "lbrynet"}
|
||||
) as response:
|
||||
try:
|
||||
self._last_response = await response.json()
|
||||
self._last_response = await response.json(content_type=None)
|
||||
except ContentTypeError as e:
|
||||
self._last_response = {}
|
||||
log.warning("Could not parse exchange rate response from %s: %s", self.name, e.message)
|
||||
@@ -75,18 +79,23 @@ class MarketFeed:
|
||||
log.debug("Saving rate update %f for %s from %s", rate, self.market, self.name)
|
||||
self.rate = ExchangeRate(self.market, rate, int(time.time()))
|
||||
self.last_check = time.time()
|
||||
self.event.set()
|
||||
return self.rate
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("Timed out fetching exchange rate from %s.", self.name)
|
||||
except json.JSONDecodeError as e:
|
||||
log.warning("Could not parse exchange rate response from %s: %s", self.name, e.doc)
|
||||
msg = e.doc if '<html>' not in e.doc else 'unexpected content type.'
|
||||
log.warning("Could not parse exchange rate response from %s: %s", self.name, msg)
|
||||
log.debug(e.doc)
|
||||
except InvalidExchangeRateResponseError as e:
|
||||
log.warning(str(e))
|
||||
except ClientConnectionError as e:
|
||||
log.warning("Error trying to connect to exchange rate %s: %s", self.name, str(e))
|
||||
except Exception as e:
|
||||
log.exception("Exchange rate error (%s from %s):", self.market, self.name)
|
||||
finally:
|
||||
self.event.set()
|
||||
|
||||
async def keep_updated(self):
|
||||
while True:
|
||||
@@ -104,70 +113,92 @@ class MarketFeed:
|
||||
self.event.clear()
|
||||
|
||||
|
||||
class BittrexFeed(MarketFeed):
|
||||
class BaseBittrexFeed(MarketFeed):
|
||||
name = "Bittrex"
|
||||
market = "BTCLBC"
|
||||
url = "https://bittrex.com/api/v1.1/public/getmarkethistory"
|
||||
params = {'market': 'BTC-LBC', 'count': 50}
|
||||
market = None
|
||||
url = None
|
||||
fee = 0.0025
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'lastTradeRate' not in json_response:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return 1.0 / float(json_response['lastTradeRate'])
|
||||
|
||||
|
||||
class BittrexBTCFeed(BaseBittrexFeed):
|
||||
market = "BTCLBC"
|
||||
url = "https://api.bittrex.com/v3/markets/LBC-BTC/ticker"
|
||||
|
||||
|
||||
class BittrexUSDFeed(BaseBittrexFeed):
|
||||
market = "USDLBC"
|
||||
url = "https://api.bittrex.com/v3/markets/LBC-USD/ticker"
|
||||
|
||||
|
||||
class BaseCoinExFeed(MarketFeed):
|
||||
name = "CoinEx"
|
||||
market = None
|
||||
url = None
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'data' not in json_response or \
|
||||
'ticker' not in json_response['data'] or \
|
||||
'last' not in json_response['data']['ticker']:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return 1.0 / float(json_response['data']['ticker']['last'])
|
||||
|
||||
|
||||
class CoinExBTCFeed(BaseCoinExFeed):
|
||||
market = "BTCLBC"
|
||||
url = "https://api.coinex.com/v1/market/ticker?market=LBCBTC"
|
||||
|
||||
|
||||
class CoinExUSDFeed(BaseCoinExFeed):
|
||||
market = "USDLBC"
|
||||
url = "https://api.coinex.com/v1/market/ticker?market=LBCUSDT"
|
||||
|
||||
|
||||
class BaseHotbitFeed(MarketFeed):
|
||||
name = "hotbit"
|
||||
market = None
|
||||
url = "https://api.hotbit.io/api/v1/market.last"
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'result' not in json_response:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
trades = json_response['result']
|
||||
if len(trades) == 0:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'trades not found')
|
||||
totals = sum([i['Total'] for i in trades])
|
||||
qtys = sum([i['Quantity'] for i in trades])
|
||||
if totals <= 0 or qtys <= 0:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'quantities were not positive')
|
||||
vwap = totals / qtys
|
||||
return float(1.0 / vwap)
|
||||
return 1.0 / float(json_response['result'])
|
||||
|
||||
|
||||
class LBRYFeed(MarketFeed):
|
||||
name = "lbry.com"
|
||||
class HotbitBTCFeed(BaseHotbitFeed):
|
||||
market = "BTCLBC"
|
||||
url = "https://api.lbry.com/lbc/exchange_rate"
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'data' not in json_response:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return 1.0 / json_response['data']['lbc_btc']
|
||||
params = {"market": "LBC/BTC"}
|
||||
|
||||
|
||||
class LBRYBTCFeed(LBRYFeed):
|
||||
market = "USDBTC"
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'data' not in json_response:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return 1.0 / json_response['data']['btc_usd']
|
||||
class HotbitUSDFeed(BaseHotbitFeed):
|
||||
market = "USDLBC"
|
||||
params = {"market": "LBC/USDT"}
|
||||
|
||||
|
||||
class CryptonatorFeed(MarketFeed):
|
||||
name = "cryptonator.com"
|
||||
class UPbitBTCFeed(MarketFeed):
|
||||
name = "UPbit"
|
||||
market = "BTCLBC"
|
||||
url = "https://api.cryptonator.com/api/ticker/btc-lbc"
|
||||
url = "https://api.upbit.com/v1/ticker"
|
||||
params = {"markets": "BTC-LBC"}
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \
|
||||
'success' not in json_response or json_response['success'] is not True:
|
||||
if "error" in json_response or len(json_response) != 1 or 'trade_price' not in json_response[0]:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return float(json_response['ticker']['price'])
|
||||
|
||||
|
||||
class CryptonatorBTCFeed(CryptonatorFeed):
|
||||
market = "USDBTC"
|
||||
url = "https://api.cryptonator.com/api/ticker/usd-btc"
|
||||
return 1.0 / float(json_response[0]['trade_price'])
|
||||
|
||||
|
||||
FEEDS: Iterable[Type[MarketFeed]] = (
|
||||
LBRYFeed,
|
||||
LBRYBTCFeed,
|
||||
BittrexFeed,
|
||||
# CryptonatorFeed,
|
||||
# CryptonatorBTCFeed,
|
||||
BittrexBTCFeed,
|
||||
BittrexUSDFeed,
|
||||
CoinExBTCFeed,
|
||||
CoinExUSDFeed,
|
||||
HotbitBTCFeed,
|
||||
HotbitUSDFeed,
|
||||
UPbitBTCFeed,
|
||||
)
|
||||
|
||||
|
||||
@@ -191,20 +222,23 @@ class ExchangeRateManager:
|
||||
source.stop()
|
||||
|
||||
def convert_currency(self, from_currency, to_currency, amount):
|
||||
rates = [market.rate for market in self.market_feeds]
|
||||
log.debug("Converting %f %s to %s, rates: %s", amount, from_currency, to_currency, rates)
|
||||
log.debug(
|
||||
"Converting %f %s to %s, rates: %s",
|
||||
amount, from_currency, to_currency,
|
||||
[market.rate for market in self.market_feeds]
|
||||
)
|
||||
if from_currency == to_currency:
|
||||
return round(amount, 8)
|
||||
|
||||
rates = []
|
||||
for market in self.market_feeds:
|
||||
if (market.has_rate and market.is_online and
|
||||
market.rate.currency_pair == (from_currency, to_currency)):
|
||||
return round(amount * Decimal(market.rate.spot), 8)
|
||||
for market in self.market_feeds:
|
||||
if (market.has_rate and market.is_online and
|
||||
market.rate.currency_pair[0] == from_currency):
|
||||
return round(self.convert_currency(
|
||||
market.rate.currency_pair[1], to_currency, amount * Decimal(market.rate.spot)), 8)
|
||||
rates.append(market.rate.spot)
|
||||
|
||||
if rates:
|
||||
return round(amount * Decimal(median(rates)), 8)
|
||||
|
||||
raise CurrencyConversionError(
|
||||
f'Unable to convert {amount} from {from_currency} to {to_currency}')
|
||||
|
||||
|
||||
@@ -6,9 +6,11 @@ from json import JSONEncoder
|
||||
|
||||
from google.protobuf.message import DecodeError
|
||||
|
||||
from lbry.schema.claim import Claim
|
||||
from scribe.schema.claim import Claim
|
||||
from scribe.schema.support import Support
|
||||
from scribe.schema.bip32 import PublicKey
|
||||
from lbry.torrent.torrent_manager import TorrentSource
|
||||
from lbry.wallet import Wallet, Ledger, Account, Transaction, Output
|
||||
from lbry.wallet.bip32 import PubKey
|
||||
from lbry.wallet.dewies import dewies_to_lbc
|
||||
from lbry.stream.managed_stream import ManagedStream
|
||||
|
||||
@@ -25,6 +27,8 @@ def encode_txo_doc():
|
||||
'address': "address of who can spend the txo",
|
||||
'confirmations': "number of confirmed blocks",
|
||||
'is_change': "payment to change address, only available when it can be determined",
|
||||
'is_received': "true if txo was sent from external account to this account",
|
||||
'is_spent': "true if txo is spent",
|
||||
'is_mine': "payment to one of your accounts, only available when it can be determined",
|
||||
'type': "one of 'claim', 'support' or 'purchase'",
|
||||
'name': "when type is 'claim' or 'support', this is the claim name",
|
||||
@@ -106,7 +110,9 @@ def encode_file_doc():
|
||||
'metadata': '(dict) None if claim is not found else the claim metadata',
|
||||
'channel_claim_id': '(str) None if claim is not found or not signed',
|
||||
'channel_name': '(str) None if claim is not found or not signed',
|
||||
'claim_name': '(str) None if claim is not found else the claim name'
|
||||
'claim_name': '(str) None if claim is not found else the claim name',
|
||||
'reflector_progress': '(int) reflector upload progress, 0 to 100',
|
||||
'uploading_to_reflector': '(bool) set to True when currently uploading to reflector'
|
||||
}
|
||||
|
||||
|
||||
@@ -117,12 +123,12 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
self.ledger = ledger
|
||||
self.include_protobuf = include_protobuf
|
||||
|
||||
def default(self, obj): # pylint: disable=method-hidden,arguments-differ,too-many-return-statements
|
||||
def default(self, obj): # pylint: disable=method-hidden,arguments-renamed,too-many-return-statements
|
||||
if isinstance(obj, Account):
|
||||
return self.encode_account(obj)
|
||||
if isinstance(obj, Wallet):
|
||||
return self.encode_wallet(obj)
|
||||
if isinstance(obj, ManagedStream):
|
||||
if isinstance(obj, (ManagedStream, TorrentSource)):
|
||||
return self.encode_file(obj)
|
||||
if isinstance(obj, Transaction):
|
||||
return self.encode_transaction(obj)
|
||||
@@ -130,7 +136,9 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
return self.encode_output(obj)
|
||||
if isinstance(obj, Claim):
|
||||
return self.encode_claim(obj)
|
||||
if isinstance(obj, PubKey):
|
||||
if isinstance(obj, Support):
|
||||
return obj.to_dict()
|
||||
if isinstance(obj, PublicKey):
|
||||
return obj.extended_key_string()
|
||||
if isinstance(obj, datetime):
|
||||
return obj.strftime("%Y%m%dT%H:%M:%S")
|
||||
@@ -164,12 +172,22 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
'amount': dewies_to_lbc(txo.amount),
|
||||
'address': txo.get_address(self.ledger) if txo.has_address else None,
|
||||
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
|
||||
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None
|
||||
'timestamp': self.ledger.headers.estimated_timestamp(tx_height)
|
||||
}
|
||||
if txo.is_change is not None:
|
||||
output['is_change'] = txo.is_change
|
||||
if txo.is_my_account is not None:
|
||||
output['is_mine'] = txo.is_my_account
|
||||
if txo.is_spent is not None:
|
||||
output['is_spent'] = txo.is_spent
|
||||
if txo.is_my_output is not None:
|
||||
output['is_my_output'] = txo.is_my_output
|
||||
if txo.is_my_input is not None:
|
||||
output['is_my_input'] = txo.is_my_input
|
||||
if txo.sent_supports is not None:
|
||||
output['sent_supports'] = dewies_to_lbc(txo.sent_supports)
|
||||
if txo.sent_tips is not None:
|
||||
output['sent_tips'] = dewies_to_lbc(txo.sent_tips)
|
||||
if txo.received_tips is not None:
|
||||
output['received_tips'] = dewies_to_lbc(txo.received_tips)
|
||||
if txo.is_internal_transfer is not None:
|
||||
output['is_internal_transfer'] = txo.is_internal_transfer
|
||||
|
||||
if txo.script.is_claim_name:
|
||||
output['type'] = 'claim'
|
||||
@@ -205,25 +223,26 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
output['claims'] = [self.encode_output(o) for o in txo.claims]
|
||||
if txo.reposted_claim is not None:
|
||||
output['reposted_claim'] = self.encode_output(txo.reposted_claim)
|
||||
if txo.script.is_claim_name or txo.script.is_update_claim:
|
||||
try:
|
||||
output['value'] = txo.claim
|
||||
if txo.script.is_claim_name or txo.script.is_update_claim or txo.script.is_support_claim_data:
|
||||
try:
|
||||
output['value'] = txo.signable
|
||||
if self.include_protobuf:
|
||||
output['protobuf'] = hexlify(txo.signable.to_bytes())
|
||||
if txo.purchase_receipt is not None:
|
||||
output['purchase_receipt'] = self.encode_output(txo.purchase_receipt)
|
||||
if txo.script.is_claim_name or txo.script.is_update_claim:
|
||||
output['value_type'] = txo.claim.claim_type
|
||||
if self.include_protobuf:
|
||||
output['protobuf'] = hexlify(txo.claim.to_bytes())
|
||||
if txo.purchase_receipt is not None:
|
||||
output['purchase_receipt'] = self.encode_output(txo.purchase_receipt)
|
||||
if txo.claim.is_channel:
|
||||
output['has_signing_key'] = txo.has_private_key
|
||||
if check_signature and txo.claim.is_signed:
|
||||
if txo.channel is not None:
|
||||
output['signing_channel'] = self.encode_output(txo.channel)
|
||||
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
|
||||
else:
|
||||
output['signing_channel'] = {'channel_id': txo.claim.signing_channel_id}
|
||||
output['is_channel_signature_valid'] = False
|
||||
except DecodeError:
|
||||
pass
|
||||
if check_signature and txo.signable.is_signed:
|
||||
if txo.channel is not None:
|
||||
output['signing_channel'] = self.encode_output(txo.channel)
|
||||
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
|
||||
else:
|
||||
output['signing_channel'] = {'channel_id': txo.signable.signing_channel_id}
|
||||
output['is_channel_signature_valid'] = False
|
||||
except DecodeError:
|
||||
pass
|
||||
return output
|
||||
|
||||
def encode_claim_meta(self, meta):
|
||||
@@ -232,7 +251,7 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
if isinstance(value, int):
|
||||
meta[key] = dewies_to_lbc(value)
|
||||
if 0 < meta.get('creation_height', 0) <= self.ledger.headers.height:
|
||||
meta['creation_timestamp'] = self.ledger.headers[meta['creation_height']]['timestamp']
|
||||
meta['creation_timestamp'] = self.ledger.headers.estimated_timestamp(meta['creation_height'])
|
||||
return meta
|
||||
|
||||
def encode_input(self, txi):
|
||||
@@ -259,26 +278,32 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
output_exists = managed_stream.output_file_exists
|
||||
tx_height = managed_stream.stream_claim_info.height
|
||||
best_height = self.ledger.headers.height
|
||||
return {
|
||||
'streaming_url': managed_stream.stream_url,
|
||||
is_stream = hasattr(managed_stream, 'stream_hash')
|
||||
if is_stream:
|
||||
total_bytes_lower_bound = managed_stream.descriptor.lower_bound_decrypted_length()
|
||||
total_bytes = managed_stream.descriptor.upper_bound_decrypted_length()
|
||||
else:
|
||||
total_bytes_lower_bound = total_bytes = managed_stream.torrent_length
|
||||
result = {
|
||||
'streaming_url': None,
|
||||
'completed': managed_stream.completed,
|
||||
'file_name': managed_stream.file_name if output_exists else None,
|
||||
'download_directory': managed_stream.download_directory if output_exists else None,
|
||||
'download_path': managed_stream.full_path if output_exists else None,
|
||||
'file_name': None,
|
||||
'download_directory': None,
|
||||
'download_path': None,
|
||||
'points_paid': 0.0,
|
||||
'stopped': not managed_stream.running,
|
||||
'stream_hash': managed_stream.stream_hash,
|
||||
'stream_name': managed_stream.descriptor.stream_name,
|
||||
'suggested_file_name': managed_stream.descriptor.suggested_file_name,
|
||||
'sd_hash': managed_stream.descriptor.sd_hash,
|
||||
'mime_type': managed_stream.mime_type,
|
||||
'key': managed_stream.descriptor.key,
|
||||
'total_bytes_lower_bound': managed_stream.descriptor.lower_bound_decrypted_length(),
|
||||
'total_bytes': managed_stream.descriptor.upper_bound_decrypted_length(),
|
||||
'stream_hash': None,
|
||||
'stream_name': None,
|
||||
'suggested_file_name': None,
|
||||
'sd_hash': None,
|
||||
'mime_type': None,
|
||||
'key': None,
|
||||
'total_bytes_lower_bound': total_bytes_lower_bound,
|
||||
'total_bytes': total_bytes,
|
||||
'written_bytes': managed_stream.written_bytes,
|
||||
'blobs_completed': managed_stream.blobs_completed,
|
||||
'blobs_in_stream': managed_stream.blobs_in_stream,
|
||||
'blobs_remaining': managed_stream.blobs_remaining,
|
||||
'blobs_completed': None,
|
||||
'blobs_in_stream': None,
|
||||
'blobs_remaining': None,
|
||||
'status': managed_stream.status,
|
||||
'claim_id': managed_stream.claim_id,
|
||||
'txid': managed_stream.txid,
|
||||
@@ -294,9 +319,38 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
'added_on': managed_stream.added_on,
|
||||
'height': tx_height,
|
||||
'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height,
|
||||
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None,
|
||||
'is_fully_reflected': managed_stream.is_fully_reflected
|
||||
'timestamp': self.ledger.headers.estimated_timestamp(tx_height),
|
||||
'is_fully_reflected': False,
|
||||
'reflector_progress': False,
|
||||
'uploading_to_reflector': False
|
||||
}
|
||||
if is_stream:
|
||||
result.update({
|
||||
'streaming_url': managed_stream.stream_url,
|
||||
'stream_hash': managed_stream.stream_hash,
|
||||
'stream_name': managed_stream.descriptor.stream_name,
|
||||
'suggested_file_name': managed_stream.descriptor.suggested_file_name,
|
||||
'sd_hash': managed_stream.descriptor.sd_hash,
|
||||
'mime_type': managed_stream.mime_type,
|
||||
'key': managed_stream.descriptor.key,
|
||||
'blobs_completed': managed_stream.blobs_completed,
|
||||
'blobs_in_stream': managed_stream.blobs_in_stream,
|
||||
'blobs_remaining': managed_stream.blobs_remaining,
|
||||
'is_fully_reflected': managed_stream.is_fully_reflected,
|
||||
'reflector_progress': managed_stream.reflector_progress,
|
||||
'uploading_to_reflector': managed_stream.uploading_to_reflector
|
||||
})
|
||||
else:
|
||||
result.update({
|
||||
'streaming_url': f'file://{managed_stream.full_path}',
|
||||
})
|
||||
if output_exists:
|
||||
result.update({
|
||||
'file_name': managed_stream.file_name,
|
||||
'download_directory': managed_stream.download_directory,
|
||||
'download_path': managed_stream.full_path,
|
||||
})
|
||||
return result
|
||||
|
||||
def encode_claim(self, claim):
|
||||
encoded = getattr(claim, claim.claim_type).to_dict()
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging.handlers
|
||||
import traceback
|
||||
|
||||
import typing
|
||||
from aiohttp.client_exceptions import ClientError
|
||||
import aiohttp
|
||||
from lbry import utils, __version__
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
|
||||
LOGGLY_TOKEN = 'BQEzZmMzLJHgAGxkBF00LGD0YGuyATVgAmqxAQEuAQZ2BQH4'
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""Format log records using json serialization"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__()
|
||||
self.attributes = kwargs
|
||||
|
||||
def format(self, record):
|
||||
data = {
|
||||
'loggerName': record.name,
|
||||
'asciTime': self.formatTime(record),
|
||||
'fileName': record.filename,
|
||||
'functionName': record.funcName,
|
||||
'levelNo': record.levelno,
|
||||
'lineNo': record.lineno,
|
||||
'levelName': record.levelname,
|
||||
'message': record.getMessage(),
|
||||
}
|
||||
data.update(self.attributes)
|
||||
if record.exc_info:
|
||||
data['exc_info'] = self.formatException(record.exc_info)
|
||||
return json.dumps(data)
|
||||
|
||||
|
||||
class HTTPSLogglyHandler(logging.Handler):
|
||||
def __init__(self, loggly_token: str, config: 'Config'):
|
||||
super().__init__()
|
||||
self.cookies = {}
|
||||
self.url = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}".format(
|
||||
token=utils.deobfuscate(loggly_token), tag='lbrynet-' + __version__
|
||||
)
|
||||
self._loop = asyncio.get_event_loop()
|
||||
self._session = aiohttp.ClientSession()
|
||||
self._config = config
|
||||
|
||||
@property
|
||||
def enabled(self):
|
||||
return self._config.share_usage_data
|
||||
|
||||
@staticmethod
|
||||
def get_full_message(record):
|
||||
if record.exc_info:
|
||||
return '\n'.join(traceback.format_exception(*record.exc_info))
|
||||
else:
|
||||
return record.getMessage()
|
||||
|
||||
async def _emit(self, record, retry=True):
|
||||
data = self.format(record).encode()
|
||||
try:
|
||||
async with self._session.post(self.url, data=data,
|
||||
cookies=self.cookies) as response:
|
||||
self.cookies.update(response.cookies)
|
||||
except ClientError:
|
||||
if self._loop.is_running() and retry and self.enabled:
|
||||
await self._session.close()
|
||||
self._session = aiohttp.ClientSession()
|
||||
return await self._emit(record, retry=False)
|
||||
|
||||
def emit(self, record):
|
||||
if not self.enabled:
|
||||
return
|
||||
try:
|
||||
asyncio.ensure_future(self._emit(record), loop=self._loop)
|
||||
except RuntimeError: # TODO: use a second loop
|
||||
print(f"\nfailed to send traceback to loggly, please file an issue with the following traceback:\n"
|
||||
f"{self.format(record)}")
|
||||
|
||||
def close(self):
|
||||
super().close()
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(self._session.close())
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
|
||||
def get_loggly_handler(config):
|
||||
handler = HTTPSLogglyHandler(LOGGLY_TOKEN, config=config)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
return handler
|
||||
@@ -35,6 +35,10 @@ def migrate_db(conf, start, end):
|
||||
from .migrate12to13 import do_migration
|
||||
elif current == 13:
|
||||
from .migrate13to14 import do_migration
|
||||
elif current == 14:
|
||||
from .migrate14to15 import do_migration
|
||||
elif current == 15:
|
||||
from .migrate15to16 import do_migration
|
||||
else:
|
||||
raise Exception(f"DB migration of version {current} to {current+1} is not available")
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
import os
|
||||
import sqlite3
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
cursor.executescript("""
|
||||
alter table blob add column added_on integer not null default 0;
|
||||
alter table blob add column is_mine integer not null default 1;
|
||||
""")
|
||||
|
||||
connection.commit()
|
||||
connection.close()
|
||||
@@ -0,0 +1,17 @@
|
||||
import os
|
||||
import sqlite3
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
cursor.executescript("""
|
||||
update blob set should_announce=0
|
||||
where should_announce=1 and
|
||||
blob.blob_hash in (select stream_blob.blob_hash from stream_blob where position=0);
|
||||
""")
|
||||
|
||||
connection.commit()
|
||||
connection.close()
|
||||
@@ -3,7 +3,7 @@ import os
|
||||
import json
|
||||
import logging
|
||||
from binascii import hexlify
|
||||
from lbry.schema.claim import Claim
|
||||
from scribe.schema.claim import Claim
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
import logging
|
||||
from aiohttp import web
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def ensure_request_allowed(request, conf):
|
||||
if is_request_allowed(request, conf):
|
||||
return
|
||||
if conf.allowed_origin:
|
||||
log.warning(
|
||||
"API requests with Origin '%s' are not allowed, "
|
||||
"configuration 'allowed_origin' limits requests to: '%s'",
|
||||
request.headers.get('Origin'), conf.allowed_origin
|
||||
)
|
||||
else:
|
||||
log.warning(
|
||||
"API requests with Origin '%s' are not allowed, "
|
||||
"update configuration 'allowed_origin' to enable this origin.",
|
||||
request.headers.get('Origin')
|
||||
)
|
||||
raise web.HTTPForbidden()
|
||||
|
||||
|
||||
def is_request_allowed(request, conf) -> bool:
|
||||
origin = request.headers.get('Origin')
|
||||
return (
|
||||
origin is None or
|
||||
origin == conf.allowed_origin or
|
||||
conf.allowed_origin == '*'
|
||||
)
|
||||
+111
-15
@@ -9,8 +9,8 @@ from typing import Optional
|
||||
from lbry.wallet import SQLiteMixin
|
||||
from lbry.conf import Config
|
||||
from lbry.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
||||
from lbry.wallet.transaction import Transaction
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.wallet.transaction import Transaction, Output
|
||||
from scribe.schema.claim import Claim
|
||||
from lbry.dht.constants import DATA_EXPIRATION
|
||||
from lbry.blob.blob_info import BlobInfo
|
||||
|
||||
@@ -170,8 +170,8 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
|
||||
def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'):
|
||||
# add all blobs, except the last one, which is empty
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
((blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
((blob.blob_hash, blob.length, 0, 0, "pending", 0, 0, blob.added_on, blob.is_mine)
|
||||
for blob in (descriptor.blobs[:-1] if len(descriptor.blobs) > 1 else descriptor.blobs) + [sd_blob])
|
||||
).fetchall()
|
||||
# associate the blobs to the stream
|
||||
@@ -187,8 +187,8 @@ def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descripto
|
||||
).fetchall()
|
||||
# ensure should_announce is set regardless if insert was ignored
|
||||
transaction.execute(
|
||||
"update blob set should_announce=1 where blob_hash in (?, ?)",
|
||||
(sd_blob.blob_hash, descriptor.blobs[0].blob_hash,)
|
||||
"update blob set should_announce=1 where blob_hash in (?)",
|
||||
(sd_blob.blob_hash,)
|
||||
).fetchall()
|
||||
|
||||
|
||||
@@ -242,7 +242,9 @@ class SQLiteStorage(SQLiteMixin):
|
||||
should_announce integer not null default 0,
|
||||
status text not null,
|
||||
last_announced_time integer,
|
||||
single_announce integer
|
||||
single_announce integer,
|
||||
added_on integer not null,
|
||||
is_mine integer not null default 0
|
||||
);
|
||||
|
||||
create table if not exists stream (
|
||||
@@ -335,6 +337,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
tcp_port integer,
|
||||
unique (address, udp_port)
|
||||
);
|
||||
create index if not exists blob_data on blob(blob_hash, blob_length, is_mine);
|
||||
"""
|
||||
|
||||
def __init__(self, conf: Config, path, loop=None, time_getter: typing.Optional[typing.Callable[[], float]] = None):
|
||||
@@ -356,19 +359,19 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
# # # # # # # # # blob functions # # # # # # # # #
|
||||
|
||||
async def add_blobs(self, *blob_hashes_and_lengths: typing.Tuple[str, int], finished=False):
|
||||
async def add_blobs(self, *blob_hashes_and_lengths: typing.Tuple[str, int, int, int], finished=False):
|
||||
def _add_blobs(transaction: sqlite3.Connection):
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
(blob_hash, length, 0, 0, "pending" if not finished else "finished", 0, 0)
|
||||
for blob_hash, length in blob_hashes_and_lengths
|
||||
(blob_hash, length, 0, 0, "pending" if not finished else "finished", 0, 0, added_on, is_mine)
|
||||
for blob_hash, length, added_on, is_mine in blob_hashes_and_lengths
|
||||
)
|
||||
).fetchall()
|
||||
if finished:
|
||||
transaction.executemany(
|
||||
"update blob set status='finished' where blob.blob_hash=?", (
|
||||
(blob_hash, ) for blob_hash, _ in blob_hashes_and_lengths
|
||||
(blob_hash, ) for blob_hash, _, _, _ in blob_hashes_and_lengths
|
||||
)
|
||||
).fetchall()
|
||||
return await self.db.run(_add_blobs)
|
||||
@@ -378,6 +381,11 @@ class SQLiteStorage(SQLiteMixin):
|
||||
"select status from blob where blob_hash=?", blob_hash
|
||||
)
|
||||
|
||||
def set_announce(self, *blob_hashes):
|
||||
return self.db.execute_fetchall(
|
||||
"update blob set should_announce=1 where blob_hash in (?, ?)", blob_hashes
|
||||
)
|
||||
|
||||
def update_last_announced_blobs(self, blob_hashes: typing.List[str]):
|
||||
def _update_last_announced_blobs(transaction: sqlite3.Connection):
|
||||
last_announced = self.time_getter()
|
||||
@@ -435,6 +443,59 @@ class SQLiteStorage(SQLiteMixin):
|
||||
def get_all_blob_hashes(self):
|
||||
return self.run_and_return_list("select blob_hash from blob")
|
||||
|
||||
async def get_stored_blobs(self, is_mine: bool, is_network_blob=False):
|
||||
is_mine = 1 if is_mine else 0
|
||||
if is_network_blob:
|
||||
return await self.db.execute_fetchall(
|
||||
"select blob.blob_hash, blob.blob_length, blob.added_on "
|
||||
"from blob left join stream_blob using (blob_hash) "
|
||||
"where stream_blob.stream_hash is null and blob.is_mine=? order by blob.added_on asc",
|
||||
(is_mine,)
|
||||
)
|
||||
|
||||
sd_blobs = await self.db.execute_fetchall(
|
||||
"select blob.blob_hash, blob.blob_length, blob.added_on "
|
||||
"from blob join stream on blob.blob_hash=stream.sd_hash join file using (stream_hash) "
|
||||
"where blob.is_mine=? order by blob.added_on asc",
|
||||
(is_mine,)
|
||||
)
|
||||
content_blobs = await self.db.execute_fetchall(
|
||||
"select blob.blob_hash, blob.blob_length, blob.added_on "
|
||||
"from blob join stream_blob using (blob_hash) cross join stream using (stream_hash)"
|
||||
"cross join file using (stream_hash) where blob.is_mine=? order by blob.added_on asc, blob.blob_length asc",
|
||||
(is_mine,)
|
||||
)
|
||||
return content_blobs + sd_blobs
|
||||
|
||||
async def get_stored_blob_disk_usage(self):
|
||||
total, network_size, content_size, private_size = await self.db.execute_fetchone("""
|
||||
select coalesce(sum(blob_length), 0) as total,
|
||||
coalesce(sum(case when
|
||||
stream_blob.stream_hash is null
|
||||
then blob_length else 0 end), 0) as network_storage,
|
||||
coalesce(sum(case when
|
||||
stream_blob.blob_hash is not null and is_mine=0
|
||||
then blob_length else 0 end), 0) as content_storage,
|
||||
coalesce(sum(case when
|
||||
is_mine=1
|
||||
then blob_length else 0 end), 0) as private_storage
|
||||
from blob left join stream_blob using (blob_hash)
|
||||
""")
|
||||
return {
|
||||
'network_storage': network_size,
|
||||
'content_storage': content_size,
|
||||
'private_storage': private_size,
|
||||
'total': total
|
||||
}
|
||||
|
||||
async def update_blob_ownership(self, sd_hash, is_mine: bool):
|
||||
is_mine = 1 if is_mine else 0
|
||||
await self.db.execute_fetchall(
|
||||
"update blob set is_mine = ? where blob_hash in ("
|
||||
" select blob_hash from blob natural join stream_blob natural join stream where sd_hash = ?"
|
||||
") OR blob_hash = ?", (is_mine, sd_hash, sd_hash)
|
||||
)
|
||||
|
||||
def sync_missing_blobs(self, blob_files: typing.Set[str]) -> typing.Awaitable[typing.Set[str]]:
|
||||
def _sync_blobs(transaction: sqlite3.Connection) -> typing.Set[str]:
|
||||
finished_blob_hashes = tuple(
|
||||
@@ -570,6 +631,10 @@ class SQLiteStorage(SQLiteMixin):
|
||||
log.debug("update file status %s -> %s", stream_hash, new_status)
|
||||
return self.db.execute_fetchall("update file set status=? where stream_hash=?", (new_status, stream_hash))
|
||||
|
||||
def stop_all_files(self):
|
||||
log.debug("stopping all files")
|
||||
return self.db.execute_fetchall("update file set status=?", ("stopped",))
|
||||
|
||||
async def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: typing.Optional[str],
|
||||
file_name: typing.Optional[str]):
|
||||
if not file_name or not download_dir:
|
||||
@@ -617,7 +682,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
).fetchall()
|
||||
download_dir = binascii.hexlify(self.conf.download_dir.encode()).decode()
|
||||
transaction.executemany(
|
||||
f"update file set download_directory=? where stream_hash=?",
|
||||
"update file set download_directory=? where stream_hash=?",
|
||||
((download_dir, stream_hash) for stream_hash in stream_hashes)
|
||||
).fetchall()
|
||||
await self.db.run_with_foreign_keys_disabled(_recover)
|
||||
@@ -727,6 +792,19 @@ class SQLiteStorage(SQLiteMixin):
|
||||
if claim_id_to_supports:
|
||||
await self.save_supports(claim_id_to_supports)
|
||||
|
||||
def save_claim_from_output(self, ledger, *outputs: Output):
|
||||
return self.save_claims([{
|
||||
"claim_id": output.claim_id,
|
||||
"name": output.claim_name,
|
||||
"amount": dewies_to_lbc(output.amount),
|
||||
"address": output.get_address(ledger),
|
||||
"txid": output.tx_ref.id,
|
||||
"nout": output.position,
|
||||
"value": output.claim,
|
||||
"height": output.tx_ref.height,
|
||||
"claim_sequence": -1,
|
||||
} for output in outputs])
|
||||
|
||||
def save_claims_for_resolve(self, claim_infos):
|
||||
to_save = {}
|
||||
for info in claim_infos:
|
||||
@@ -740,7 +818,8 @@ class SQLiteStorage(SQLiteMixin):
|
||||
return self.save_claims(to_save.values())
|
||||
|
||||
@staticmethod
|
||||
def _save_content_claim(transaction, claim_outpoint, stream_hash):
|
||||
def _save_content_claim(transaction, claim_outpoint, stream_hash=None, bt_infohash=None):
|
||||
assert stream_hash or bt_infohash
|
||||
# get the claim id and serialized metadata
|
||||
claim_info = transaction.execute(
|
||||
"select claim_id, serialized_metadata from claim where claim_outpoint=?", (claim_outpoint,)
|
||||
@@ -788,6 +867,19 @@ class SQLiteStorage(SQLiteMixin):
|
||||
if stream_hash in self.content_claim_callbacks:
|
||||
await self.content_claim_callbacks[stream_hash]()
|
||||
|
||||
async def save_torrent_content_claim(self, bt_infohash, claim_outpoint, length, name):
|
||||
def _save_torrent(transaction):
|
||||
transaction.execute(
|
||||
"insert or replace into torrent values (?, NULL, ?, ?)", (bt_infohash, length, name)
|
||||
).fetchall()
|
||||
transaction.execute(
|
||||
"insert or replace into content_claim values (NULL, ?, ?)", (bt_infohash, claim_outpoint)
|
||||
).fetchall()
|
||||
await self.db.run(_save_torrent)
|
||||
# update corresponding ManagedEncryptedFileDownloader object
|
||||
if bt_infohash in self.content_claim_callbacks:
|
||||
await self.content_claim_callbacks[bt_infohash]()
|
||||
|
||||
async def get_content_claim(self, stream_hash: str, include_supports: typing.Optional[bool] = True) -> typing.Dict:
|
||||
claims = await self.db.run(get_claims_from_stream_hashes, [stream_hash])
|
||||
claim = None
|
||||
@@ -799,6 +891,10 @@ class SQLiteStorage(SQLiteMixin):
|
||||
claim['effective_amount'] = calculate_effective_amount(claim['amount'], supports)
|
||||
return claim
|
||||
|
||||
async def get_content_claim_for_torrent(self, bt_infohash):
|
||||
claims = await self.db.run(get_claims_from_torrent_info_hashes, [bt_infohash])
|
||||
return claims[bt_infohash].as_dict() if claims else None
|
||||
|
||||
# # # # # # # # # reflector functions # # # # # # # # #
|
||||
|
||||
def update_reflected_stream(self, sd_hash, reflector_address, success=True):
|
||||
@@ -830,6 +926,6 @@ class SQLiteStorage(SQLiteMixin):
|
||||
transaction.execute('delete from peer').fetchall()
|
||||
transaction.executemany(
|
||||
'insert into peer(node_id, address, udp_port, tcp_port) values (?, ?, ?, ?)',
|
||||
tuple([(binascii.hexlify(p.node_id), p.address, p.udp_port, p.tcp_port) for p in peers])
|
||||
((binascii.hexlify(p.node_id), p.address, p.udp_port, p.tcp_port) for p in peers)
|
||||
).fetchall()
|
||||
return await self.db.run(_save_kademlia_peers)
|
||||
File renamed without changes.
@@ -0,0 +1,306 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import typing
|
||||
from typing import Optional
|
||||
from aiohttp.web import Request
|
||||
from lbry.error import ResolveError, DownloadSDTimeoutError, InsufficientFundsError
|
||||
from lbry.error import ResolveTimeoutError, DownloadDataTimeoutError, KeyFeeAboveMaxAllowedError
|
||||
from lbry.error import InvalidStreamURLError
|
||||
from lbry.stream.managed_stream import ManagedStream
|
||||
from lbry.torrent.torrent_manager import TorrentSource
|
||||
from lbry.utils import cache_concurrent
|
||||
from scribe.schema.url import URL
|
||||
from lbry.wallet.dewies import dewies_to_lbc
|
||||
from lbry.file.source_manager import SourceManager
|
||||
from lbry.file.source import ManagedDownloadSource
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from lbry.wallet import WalletManager, Output
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FileManager:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', wallet_manager: 'WalletManager',
|
||||
storage: 'SQLiteStorage', analytics_manager: Optional['AnalyticsManager'] = None):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
self.wallet_manager = wallet_manager
|
||||
self.storage = storage
|
||||
self.analytics_manager = analytics_manager
|
||||
self.source_managers: typing.Dict[str, SourceManager] = {}
|
||||
self.started = asyncio.Event()
|
||||
|
||||
@property
|
||||
def streams(self):
|
||||
return self.source_managers['stream']._sources
|
||||
|
||||
async def create_stream(self, file_path: str, key: Optional[bytes] = None, **kwargs) -> ManagedDownloadSource:
|
||||
if 'stream' in self.source_managers:
|
||||
return await self.source_managers['stream'].create(file_path, key, **kwargs)
|
||||
raise NotImplementedError
|
||||
|
||||
async def start(self):
|
||||
await asyncio.gather(*(source_manager.start() for source_manager in self.source_managers.values()))
|
||||
for manager in self.source_managers.values():
|
||||
await manager.started.wait()
|
||||
self.started.set()
|
||||
|
||||
def stop(self):
|
||||
for manager in self.source_managers.values():
|
||||
# fixme: pop or not?
|
||||
manager.stop()
|
||||
self.started.clear()
|
||||
|
||||
@cache_concurrent
|
||||
async def download_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
|
||||
timeout: Optional[float] = None, file_name: Optional[str] = None,
|
||||
download_directory: Optional[str] = None,
|
||||
save_file: Optional[bool] = None, resolve_timeout: float = 3.0,
|
||||
wallet: Optional['Wallet'] = None) -> ManagedDownloadSource:
|
||||
|
||||
wallet = wallet or self.wallet_manager.default_wallet
|
||||
timeout = timeout or self.config.download_timeout
|
||||
start_time = self.loop.time()
|
||||
resolved_time = None
|
||||
stream = None
|
||||
claim = None
|
||||
error = None
|
||||
outpoint = None
|
||||
if save_file is None:
|
||||
save_file = self.config.save_files
|
||||
if file_name and not save_file:
|
||||
save_file = True
|
||||
if save_file:
|
||||
download_directory = download_directory or self.config.download_dir
|
||||
else:
|
||||
download_directory = None
|
||||
|
||||
payment = None
|
||||
try:
|
||||
# resolve the claim
|
||||
try:
|
||||
if not URL.parse(uri).has_stream:
|
||||
raise InvalidStreamURLError(uri)
|
||||
except ValueError:
|
||||
raise InvalidStreamURLError(uri)
|
||||
try:
|
||||
resolved_result = await asyncio.wait_for(
|
||||
self.wallet_manager.ledger.resolve(
|
||||
wallet.accounts, [uri],
|
||||
include_purchase_receipt=True,
|
||||
include_is_my_output=True
|
||||
), resolve_timeout
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
raise ResolveTimeoutError(uri)
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError):
|
||||
raise
|
||||
log.exception("Unexpected error resolving stream:")
|
||||
raise ResolveError(f"Unexpected error resolving stream: {str(err)}")
|
||||
if 'error' in resolved_result:
|
||||
raise ResolveError(f"Unexpected error resolving uri for download: {resolved_result['error']}")
|
||||
if not resolved_result or uri not in resolved_result:
|
||||
raise ResolveError(f"Failed to resolve stream at '{uri}'")
|
||||
txo = resolved_result[uri]
|
||||
if isinstance(txo, dict):
|
||||
raise ResolveError(f"Failed to resolve stream at '{uri}': {txo}")
|
||||
claim = txo.claim
|
||||
outpoint = f"{txo.tx_ref.id}:{txo.position}"
|
||||
resolved_time = self.loop.time() - start_time
|
||||
await self.storage.save_claim_from_output(self.wallet_manager.ledger, txo)
|
||||
|
||||
####################
|
||||
# update or replace
|
||||
####################
|
||||
|
||||
if claim.stream.source.bt_infohash:
|
||||
source_manager = self.source_managers['torrent']
|
||||
existing = source_manager.get_filtered(bt_infohash=claim.stream.source.bt_infohash)
|
||||
elif claim.stream.source.sd_hash:
|
||||
source_manager = self.source_managers['stream']
|
||||
existing = source_manager.get_filtered(sd_hash=claim.stream.source.sd_hash)
|
||||
else:
|
||||
raise ResolveError(f"There is nothing to download at {uri} - Source is unknown or unset")
|
||||
|
||||
# resume or update an existing stream, if the stream changed: download it and delete the old one after
|
||||
to_replace, updated_stream = None, None
|
||||
if existing and existing[0].claim_id != txo.claim_id:
|
||||
raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {txo.claim_id}")
|
||||
if existing:
|
||||
log.info("claim contains a metadata only update to a stream we have")
|
||||
if claim.stream.source.bt_infohash:
|
||||
await self.storage.save_torrent_content_claim(
|
||||
existing[0].identifier, outpoint, existing[0].torrent_length, existing[0].torrent_name
|
||||
)
|
||||
claim_info = await self.storage.get_content_claim_for_torrent(existing[0].identifier)
|
||||
existing[0].set_claim(claim_info, claim)
|
||||
else:
|
||||
await self.storage.save_content_claim(
|
||||
existing[0].stream_hash, outpoint
|
||||
)
|
||||
await source_manager._update_content_claim(existing[0])
|
||||
updated_stream = existing[0]
|
||||
else:
|
||||
existing_for_claim_id = self.get_filtered(claim_id=txo.claim_id)
|
||||
if existing_for_claim_id:
|
||||
log.info("claim contains an update to a stream we have, downloading it")
|
||||
if save_file and existing_for_claim_id[0].output_file_exists:
|
||||
save_file = False
|
||||
if not claim.stream.source.bt_infohash:
|
||||
existing_for_claim_id[0].downloader.node = source_manager.node
|
||||
await existing_for_claim_id[0].start(timeout=timeout, save_now=save_file)
|
||||
if not existing_for_claim_id[0].output_file_exists and (
|
||||
save_file or file_name or download_directory):
|
||||
await existing_for_claim_id[0].save_file(
|
||||
file_name=file_name, download_directory=download_directory
|
||||
)
|
||||
to_replace = existing_for_claim_id[0]
|
||||
|
||||
# resume or update an existing stream, if the stream changed: download it and delete the old one after
|
||||
if updated_stream:
|
||||
log.info("already have stream for %s", uri)
|
||||
if save_file and updated_stream.output_file_exists:
|
||||
save_file = False
|
||||
if not claim.stream.source.bt_infohash:
|
||||
updated_stream.downloader.node = source_manager.node
|
||||
await updated_stream.start(timeout=timeout, save_now=save_file)
|
||||
if not updated_stream.output_file_exists and (save_file or file_name or download_directory):
|
||||
await updated_stream.save_file(
|
||||
file_name=file_name, download_directory=download_directory
|
||||
)
|
||||
return updated_stream
|
||||
|
||||
####################
|
||||
# pay fee
|
||||
####################
|
||||
|
||||
needs_purchasing = (
|
||||
not to_replace and
|
||||
not txo.is_my_output and
|
||||
txo.has_price and
|
||||
not txo.purchase_receipt
|
||||
)
|
||||
|
||||
if needs_purchasing:
|
||||
payment = await self.wallet_manager.create_purchase_transaction(
|
||||
wallet.accounts, txo, exchange_rate_manager
|
||||
)
|
||||
|
||||
####################
|
||||
# make downloader and wait for start
|
||||
####################
|
||||
|
||||
if not claim.stream.source.bt_infohash:
|
||||
# fixme: this shouldnt be here
|
||||
stream = ManagedStream(
|
||||
self.loop, self.config, source_manager.blob_manager, claim.stream.source.sd_hash,
|
||||
download_directory, file_name, ManagedStream.STATUS_RUNNING, content_fee=payment,
|
||||
analytics_manager=self.analytics_manager
|
||||
)
|
||||
stream.downloader.node = source_manager.node
|
||||
else:
|
||||
stream = TorrentSource(
|
||||
self.loop, self.config, self.storage, identifier=claim.stream.source.bt_infohash,
|
||||
file_name=file_name, download_directory=download_directory or self.config.download_dir,
|
||||
status=ManagedStream.STATUS_RUNNING,
|
||||
analytics_manager=self.analytics_manager,
|
||||
torrent_session=source_manager.torrent_session
|
||||
)
|
||||
log.info("starting download for %s", uri)
|
||||
|
||||
before_download = self.loop.time()
|
||||
await stream.start(timeout, save_file)
|
||||
|
||||
####################
|
||||
# success case: delete to_replace if applicable, broadcast fee payment
|
||||
####################
|
||||
|
||||
if to_replace: # delete old stream now that the replacement has started downloading
|
||||
await source_manager.delete(to_replace)
|
||||
|
||||
if payment is not None:
|
||||
await self.wallet_manager.broadcast_or_release(payment)
|
||||
payment = None # to avoid releasing in `finally` later
|
||||
log.info("paid fee of %s for %s", dewies_to_lbc(stream.content_fee.outputs[0].amount), uri)
|
||||
await self.storage.save_content_fee(stream.stream_hash, stream.content_fee)
|
||||
|
||||
source_manager.add(stream)
|
||||
|
||||
if not claim.stream.source.bt_infohash:
|
||||
await self.storage.save_content_claim(stream.stream_hash, outpoint)
|
||||
else:
|
||||
await self.storage.save_torrent_content_claim(
|
||||
stream.identifier, outpoint, stream.torrent_length, stream.torrent_name
|
||||
)
|
||||
claim_info = await self.storage.get_content_claim_for_torrent(stream.identifier)
|
||||
stream.set_claim(claim_info, claim)
|
||||
if save_file:
|
||||
await asyncio.wait_for(stream.save_file(), timeout - (self.loop.time() - before_download),
|
||||
loop=self.loop)
|
||||
return stream
|
||||
except asyncio.TimeoutError:
|
||||
error = DownloadDataTimeoutError(stream.sd_hash)
|
||||
raise error
|
||||
except Exception as err: # forgive data timeout, don't delete stream
|
||||
expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError,
|
||||
KeyFeeAboveMaxAllowedError, ResolveError, InvalidStreamURLError)
|
||||
if isinstance(err, expected):
|
||||
log.warning("Failed to download %s: %s", uri, str(err))
|
||||
elif isinstance(err, asyncio.CancelledError):
|
||||
pass
|
||||
else:
|
||||
log.exception("Unexpected error downloading stream:")
|
||||
error = err
|
||||
raise
|
||||
finally:
|
||||
if payment is not None:
|
||||
# payment is set to None after broadcasting, if we're here an exception probably happened
|
||||
await self.wallet_manager.ledger.release_tx(payment)
|
||||
if self.analytics_manager and claim and claim.stream.source.bt_infohash:
|
||||
# TODO: analytics for torrents
|
||||
pass
|
||||
elif self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or
|
||||
stream.downloader.time_to_first_bytes))):
|
||||
server = self.wallet_manager.ledger.network.client.server
|
||||
self.loop.create_task(
|
||||
self.analytics_manager.send_time_to_first_bytes(
|
||||
resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id,
|
||||
uri, outpoint,
|
||||
None if not stream else len(stream.downloader.blob_downloader.active_connections),
|
||||
None if not stream else len(stream.downloader.blob_downloader.scores),
|
||||
None if not stream else len(stream.downloader.blob_downloader.connection_failures),
|
||||
False if not stream else stream.downloader.added_fixed_peers,
|
||||
self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay,
|
||||
None if not stream else stream.sd_hash,
|
||||
None if not stream else stream.downloader.time_to_descriptor,
|
||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
|
||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
|
||||
None if not stream else stream.downloader.time_to_first_bytes,
|
||||
None if not error else error.__class__.__name__,
|
||||
None if not error else str(error),
|
||||
None if not server else f"{server[0]}:{server[1]}"
|
||||
)
|
||||
)
|
||||
|
||||
async def stream_partial_content(self, request: Request, sd_hash: str):
|
||||
return await self.source_managers['stream'].stream_partial_content(request, sd_hash)
|
||||
|
||||
def get_filtered(self, *args, **kwargs) -> typing.List[ManagedDownloadSource]:
|
||||
"""
|
||||
Get a list of filtered and sorted ManagedStream objects
|
||||
|
||||
:param sort_by: field to sort by
|
||||
:param reverse: reverse sorting
|
||||
:param comparison: comparison operator used for filtering
|
||||
:param search_by: fields and values to filter by
|
||||
"""
|
||||
return sum((manager.get_filtered(*args, **kwargs) for manager in self.source_managers.values()), [])
|
||||
|
||||
async def delete(self, source: ManagedDownloadSource, delete_file=False):
|
||||
for manager in self.source_managers.values():
|
||||
await manager.delete(source, delete_file)
|
||||
@@ -0,0 +1,161 @@
|
||||
import os
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
import binascii
|
||||
from typing import Optional
|
||||
from lbry.utils import generate_id
|
||||
from lbry.extras.daemon.storage import StoredContentClaim
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.wallet.transaction import Transaction
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ManagedDownloadSource:
|
||||
STATUS_RUNNING = "running"
|
||||
STATUS_STOPPED = "stopped"
|
||||
STATUS_FINISHED = "finished"
|
||||
|
||||
SAVING_ID = 1
|
||||
STREAMING_ID = 2
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage', identifier: str,
|
||||
file_name: Optional[str] = None, download_directory: Optional[str] = None,
|
||||
status: Optional[str] = STATUS_STOPPED, claim: Optional[StoredContentClaim] = None,
|
||||
download_id: Optional[str] = None, rowid: Optional[int] = None,
|
||||
content_fee: Optional['Transaction'] = None,
|
||||
analytics_manager: Optional['AnalyticsManager'] = None,
|
||||
added_on: Optional[int] = None):
|
||||
self.loop = loop
|
||||
self.storage = storage
|
||||
self.config = config
|
||||
self.identifier = identifier
|
||||
self.download_directory = download_directory
|
||||
self._file_name = file_name
|
||||
self._status = status
|
||||
self.stream_claim_info = claim
|
||||
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
|
||||
self.rowid = rowid
|
||||
self.content_fee = content_fee
|
||||
self.purchase_receipt = None
|
||||
self._added_on = added_on
|
||||
self.analytics_manager = analytics_manager
|
||||
|
||||
self.saving = asyncio.Event(loop=self.loop)
|
||||
self.finished_writing = asyncio.Event(loop=self.loop)
|
||||
self.started_writing = asyncio.Event(loop=self.loop)
|
||||
self.finished_write_attempt = asyncio.Event(loop=self.loop)
|
||||
|
||||
# @classmethod
|
||||
# async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', file_path: str,
|
||||
# key: Optional[bytes] = None,
|
||||
# iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedDownloadSource':
|
||||
# raise NotImplementedError()
|
||||
|
||||
async def start(self, timeout: Optional[float] = None, save_now: Optional[bool] = False):
|
||||
raise NotImplementedError()
|
||||
|
||||
async def stop(self, finished: bool = False):
|
||||
raise NotImplementedError()
|
||||
|
||||
async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
|
||||
raise NotImplementedError()
|
||||
|
||||
def stop_tasks(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
def set_claim(self, claim_info: typing.Dict, claim: 'Claim'):
|
||||
self.stream_claim_info = StoredContentClaim(
|
||||
f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
|
||||
claim_info['name'], claim_info['amount'], claim_info['height'],
|
||||
binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'],
|
||||
claim_info['claim_sequence'], claim_info.get('channel_name')
|
||||
)
|
||||
|
||||
# async def update_content_claim(self, claim_info: Optional[typing.Dict] = None):
|
||||
# if not claim_info:
|
||||
# claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash)
|
||||
# self.set_claim(claim_info, claim_info['value'])
|
||||
|
||||
@property
|
||||
def file_name(self) -> Optional[str]:
|
||||
return self._file_name
|
||||
|
||||
@property
|
||||
def added_on(self) -> Optional[int]:
|
||||
return self._added_on
|
||||
|
||||
@property
|
||||
def status(self) -> str:
|
||||
return self._status
|
||||
|
||||
@property
|
||||
def completed(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
# @property
|
||||
# def stream_url(self):
|
||||
# return f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash}
|
||||
|
||||
@property
|
||||
def finished(self) -> bool:
|
||||
return self.status == self.STATUS_FINISHED
|
||||
|
||||
@property
|
||||
def running(self) -> bool:
|
||||
return self.status == self.STATUS_RUNNING
|
||||
|
||||
@property
|
||||
def claim_id(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim_id
|
||||
|
||||
@property
|
||||
def txid(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.txid
|
||||
|
||||
@property
|
||||
def nout(self) -> Optional[int]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.nout
|
||||
|
||||
@property
|
||||
def outpoint(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.outpoint
|
||||
|
||||
@property
|
||||
def claim_height(self) -> Optional[int]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.height
|
||||
|
||||
@property
|
||||
def channel_claim_id(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.channel_claim_id
|
||||
|
||||
@property
|
||||
def channel_name(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.channel_name
|
||||
|
||||
@property
|
||||
def claim_name(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim_name
|
||||
|
||||
@property
|
||||
def metadata(self) -> Optional[typing.Dict]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict()
|
||||
|
||||
@property
|
||||
def metadata_protobuf(self) -> bytes:
|
||||
if self.stream_claim_info:
|
||||
return binascii.hexlify(self.stream_claim_info.claim.to_bytes())
|
||||
|
||||
@property
|
||||
def full_path(self) -> Optional[str]:
|
||||
return os.path.join(self.download_directory, os.path.basename(self.file_name)) \
|
||||
if self.file_name and self.download_directory else None
|
||||
|
||||
@property
|
||||
def output_file_exists(self):
|
||||
return os.path.isfile(self.full_path) if self.full_path else False
|
||||
@@ -0,0 +1,138 @@
|
||||
import os
|
||||
import asyncio
|
||||
import logging
|
||||
import typing
|
||||
from typing import Optional
|
||||
from lbry.file.source import ManagedDownloadSource
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
COMPARISON_OPERATORS = {
|
||||
'eq': lambda a, b: a == b,
|
||||
'ne': lambda a, b: a != b,
|
||||
'g': lambda a, b: a > b,
|
||||
'l': lambda a, b: a < b,
|
||||
'ge': lambda a, b: a >= b,
|
||||
'le': lambda a, b: a <= b,
|
||||
}
|
||||
|
||||
|
||||
class SourceManager:
|
||||
filter_fields = {
|
||||
'rowid',
|
||||
'status',
|
||||
'file_name',
|
||||
'added_on',
|
||||
'download_path',
|
||||
'claim_name',
|
||||
'claim_height',
|
||||
'claim_id',
|
||||
'outpoint',
|
||||
'txid',
|
||||
'nout',
|
||||
'channel_claim_id',
|
||||
'channel_name',
|
||||
'completed'
|
||||
}
|
||||
|
||||
set_filter_fields = {
|
||||
"claim_ids": "claim_id",
|
||||
"channel_claim_ids": "channel_claim_id",
|
||||
"outpoints": "outpoint"
|
||||
}
|
||||
|
||||
source_class = ManagedDownloadSource
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage',
|
||||
analytics_manager: Optional['AnalyticsManager'] = None):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
self.storage = storage
|
||||
self.analytics_manager = analytics_manager
|
||||
self._sources: typing.Dict[str, ManagedDownloadSource] = {}
|
||||
self.started = asyncio.Event(loop=self.loop)
|
||||
|
||||
def add(self, source: ManagedDownloadSource):
|
||||
self._sources[source.identifier] = source
|
||||
|
||||
def remove(self, source: ManagedDownloadSource):
|
||||
if source.identifier not in self._sources:
|
||||
return
|
||||
self._sources.pop(source.identifier)
|
||||
source.stop_tasks()
|
||||
|
||||
async def initialize_from_database(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
async def start(self):
|
||||
await self.initialize_from_database()
|
||||
self.started.set()
|
||||
|
||||
def stop(self):
|
||||
while self._sources:
|
||||
_, source = self._sources.popitem()
|
||||
source.stop_tasks()
|
||||
self.started.clear()
|
||||
|
||||
async def create(self, file_path: str, key: Optional[bytes] = None,
|
||||
iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedDownloadSource:
|
||||
raise NotImplementedError()
|
||||
|
||||
async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
|
||||
self.remove(source)
|
||||
if delete_file and source.output_file_exists:
|
||||
os.remove(source.full_path)
|
||||
|
||||
def get_filtered(self, sort_by: Optional[str] = None, reverse: Optional[bool] = False,
|
||||
comparison: Optional[str] = None, **search_by) -> typing.List[ManagedDownloadSource]:
|
||||
"""
|
||||
Get a list of filtered and sorted ManagedStream objects
|
||||
|
||||
:param sort_by: field to sort by
|
||||
:param reverse: reverse sorting
|
||||
:param comparison: comparison operator used for filtering
|
||||
:param search_by: fields and values to filter by
|
||||
"""
|
||||
if sort_by and sort_by not in self.filter_fields:
|
||||
raise ValueError(f"'{sort_by}' is not a valid field to sort by")
|
||||
if comparison and comparison not in COMPARISON_OPERATORS:
|
||||
raise ValueError(f"'{comparison}' is not a valid comparison")
|
||||
if 'full_status' in search_by:
|
||||
del search_by['full_status']
|
||||
|
||||
for search in search_by:
|
||||
if search not in self.filter_fields:
|
||||
raise ValueError(f"'{search}' is not a valid search operation")
|
||||
|
||||
compare_sets = {}
|
||||
if isinstance(search_by.get('claim_id'), list):
|
||||
compare_sets['claim_ids'] = search_by.pop('claim_id')
|
||||
if isinstance(search_by.get('outpoint'), list):
|
||||
compare_sets['outpoints'] = search_by.pop('outpoint')
|
||||
if isinstance(search_by.get('channel_claim_id'), list):
|
||||
compare_sets['channel_claim_ids'] = search_by.pop('channel_claim_id')
|
||||
|
||||
if search_by or compare_sets:
|
||||
comparison = comparison or 'eq'
|
||||
streams = []
|
||||
for stream in self._sources.values():
|
||||
if compare_sets and not all(
|
||||
getattr(stream, self.set_filter_fields[set_search]) in val
|
||||
for set_search, val in compare_sets.items()):
|
||||
continue
|
||||
if search_by and not all(
|
||||
COMPARISON_OPERATORS[comparison](getattr(stream, search), val)
|
||||
for search, val in search_by.items()):
|
||||
continue
|
||||
streams.append(stream)
|
||||
else:
|
||||
streams = list(self._sources.values())
|
||||
if sort_by:
|
||||
streams.sort(key=lambda s: getattr(s, sort_by) or "")
|
||||
if reverse:
|
||||
streams.reverse()
|
||||
return streams
|
||||
+159
-68
@@ -3,10 +3,14 @@ import json
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import platform
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
from math import ceil
|
||||
|
||||
import lbry.utils
|
||||
from lbry.conf import TranscodeConfig
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -14,60 +18,104 @@ log = logging.getLogger(__name__)
|
||||
|
||||
class VideoFileAnalyzer:
|
||||
|
||||
def _replace_or_pop_env(self, variable):
|
||||
if variable + '_ORIG' in self._env_copy:
|
||||
self._env_copy[variable] = self._env_copy[variable + '_ORIG']
|
||||
else:
|
||||
self._env_copy.pop(variable, None)
|
||||
|
||||
def __init__(self, conf: TranscodeConfig):
|
||||
self._conf = conf
|
||||
self._available_encoders = ""
|
||||
self._ffmpeg_installed = False
|
||||
self._which = None
|
||||
self._ffmpeg_installed = None
|
||||
self._which_ffmpeg = None
|
||||
self._which_ffprobe = None
|
||||
self._env_copy = dict(os.environ)
|
||||
self._checked_ffmpeg = False
|
||||
if lbry.utils.is_running_from_bundle():
|
||||
# handle the situation where PyInstaller overrides our runtime environment:
|
||||
self._replace_or_pop_env('LD_LIBRARY_PATH')
|
||||
|
||||
async def _execute(self, command, arguments):
|
||||
args = shlex.split(arguments)
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
os.path.join(self._conf.ffmpeg_folder, command), *args,
|
||||
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
|
||||
)
|
||||
stdout, stderr = await process.communicate() # returns when the streams are closed
|
||||
return stdout.decode(errors='replace') + stderr.decode(errors='replace'), process.returncode
|
||||
|
||||
async def _verify_executable(self, name):
|
||||
@staticmethod
|
||||
def _execute(command, environment):
|
||||
# log.debug("Executing: %s", command)
|
||||
try:
|
||||
version, code = await self._execute(name, "-version")
|
||||
with subprocess.Popen(
|
||||
shlex.split(command) if platform.system() != 'Windows' else command,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=environment
|
||||
) as process:
|
||||
(stdout, stderr) = process.communicate() # blocks until the process exits
|
||||
return stdout.decode(errors='replace') + stderr.decode(errors='replace'), process.returncode
|
||||
except subprocess.SubprocessError as e:
|
||||
return str(e), -1
|
||||
|
||||
# This create_subprocess_exec call is broken in Windows Python 3.7, but it's prettier than what's here.
|
||||
# The recommended fix is switching to ProactorEventLoop, but that breaks UDP in Linux Python 3.7.
|
||||
# We work around that issue here by using run_in_executor. Check it again in Python 3.8.
|
||||
async def _execute_ffmpeg(self, arguments):
|
||||
arguments = self._which_ffmpeg + " " + arguments
|
||||
return await asyncio.get_event_loop().run_in_executor(None, self._execute, arguments, self._env_copy)
|
||||
|
||||
async def _execute_ffprobe(self, arguments):
|
||||
arguments = self._which_ffprobe + " " + arguments
|
||||
return await asyncio.get_event_loop().run_in_executor(None, self._execute, arguments, self._env_copy)
|
||||
|
||||
async def _verify_executables(self):
|
||||
try:
|
||||
await self._execute_ffprobe("-version")
|
||||
version, code = await self._execute_ffmpeg("-version")
|
||||
except Exception as e:
|
||||
log.warning("Unable to run %s, but it was requested. Message: %s", name, str(e))
|
||||
code = -1
|
||||
version = ""
|
||||
if code != 0 or not version.startswith(name):
|
||||
raise FileNotFoundError(f"Unable to locate or run {name}. Please install FFmpeg "
|
||||
f"and ensure that it is callable via PATH or conf.ffmpeg_folder")
|
||||
version = str(e)
|
||||
if code != 0 or not version.startswith("ffmpeg"):
|
||||
log.warning("Unable to run ffmpeg, but it was requested. Code: %d; Message: %s", code, version)
|
||||
raise FileNotFoundError("Unable to locate or run ffmpeg or ffprobe. Please install FFmpeg "
|
||||
"and ensure that it is callable via PATH or conf.ffmpeg_path")
|
||||
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which_ffmpeg)
|
||||
return version
|
||||
|
||||
@staticmethod
|
||||
def _which_ffmpeg_and_ffmprobe(path):
|
||||
return shutil.which("ffmpeg", path=path), shutil.which("ffprobe", path=path)
|
||||
|
||||
async def _verify_ffmpeg_installed(self):
|
||||
if self._ffmpeg_installed:
|
||||
return
|
||||
await self._verify_executable("ffprobe")
|
||||
version = await self._verify_executable("ffmpeg")
|
||||
self._which = shutil.which(os.path.join(self._conf.ffmpeg_folder, "ffmpeg"))
|
||||
self._ffmpeg_installed = False
|
||||
path = self._conf.ffmpeg_path
|
||||
if hasattr(self._conf, "data_dir"):
|
||||
path += os.path.pathsep + os.path.join(getattr(self._conf, "data_dir"), "ffmpeg", "bin")
|
||||
path += os.path.pathsep + self._env_copy.get("PATH", "")
|
||||
self._which_ffmpeg, self._which_ffprobe = await asyncio.get_running_loop().run_in_executor(
|
||||
None, self._which_ffmpeg_and_ffmprobe, path
|
||||
)
|
||||
if not self._which_ffmpeg:
|
||||
log.warning("Unable to locate ffmpeg executable. Path: %s", path)
|
||||
raise FileNotFoundError(f"Unable to locate ffmpeg executable. Path: {path}")
|
||||
if not self._which_ffprobe:
|
||||
log.warning("Unable to locate ffprobe executable. Path: %s", path)
|
||||
raise FileNotFoundError(f"Unable to locate ffprobe executable. Path: {path}")
|
||||
if os.path.dirname(self._which_ffmpeg) != os.path.dirname(self._which_ffprobe):
|
||||
log.warning("ffmpeg and ffprobe are in different folders!")
|
||||
|
||||
await self._verify_executables()
|
||||
self._ffmpeg_installed = True
|
||||
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which)
|
||||
|
||||
async def status(self, reset=False, recheck=False):
|
||||
if reset:
|
||||
self._available_encoders = ""
|
||||
self._ffmpeg_installed = False
|
||||
self._which = None
|
||||
self._ffmpeg_installed = None
|
||||
if self._checked_ffmpeg and not recheck:
|
||||
installed = self._ffmpeg_installed
|
||||
else:
|
||||
installed = True
|
||||
pass
|
||||
elif self._ffmpeg_installed is None:
|
||||
try:
|
||||
await self._verify_ffmpeg_installed()
|
||||
except FileNotFoundError:
|
||||
installed = False
|
||||
pass
|
||||
self._checked_ffmpeg = True
|
||||
return {
|
||||
"available": installed,
|
||||
"which": self._which,
|
||||
"available": self._ffmpeg_installed,
|
||||
"which": self._which_ffmpeg,
|
||||
"analyze_audio_volume": int(self._conf.volume_analysis_time) > 0
|
||||
}
|
||||
|
||||
@@ -75,9 +123,24 @@ class VideoFileAnalyzer:
|
||||
def _verify_container(scan_data: json):
|
||||
container = scan_data["format"]["format_name"]
|
||||
log.debug(" Detected container is %s", container)
|
||||
if not {"webm", "mp4", "3gp", "ogg"}.intersection(container.split(",")):
|
||||
splits = container.split(",")
|
||||
if not {"webm", "mp4", "3gp", "ogg"}.intersection(splits):
|
||||
return "Container format is not in the approved list of WebM, MP4. " \
|
||||
f"Actual: {container} [{scan_data['format']['format_long_name']}]"
|
||||
|
||||
if "matroska" in splits:
|
||||
for stream in scan_data["streams"]:
|
||||
if stream["codec_type"] == "video":
|
||||
codec = stream["codec_name"]
|
||||
if not {"vp8", "vp9", "av1"}.intersection(codec.split(",")):
|
||||
return "WebM format requires VP8/9 or AV1 video. " \
|
||||
f"Actual: {codec} [{stream['codec_long_name']}]"
|
||||
elif stream["codec_type"] == "audio":
|
||||
codec = stream["codec_name"]
|
||||
if not {"vorbis", "opus"}.intersection(codec.split(",")):
|
||||
return "WebM format requires Vorbis or Opus audio. " \
|
||||
f"Actual: {codec} [{stream['codec_long_name']}]"
|
||||
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
@@ -106,12 +169,12 @@ class VideoFileAnalyzer:
|
||||
bit_rate = float(scan_data["format"]["bit_rate"])
|
||||
else:
|
||||
bit_rate = os.stat(file_path).st_size / float(scan_data["format"]["duration"])
|
||||
log.debug(" Detected bitrate is %s Mbps. Allowed is %s Mbps",
|
||||
log.debug(" Detected bitrate is %s Mbps. Allowed max: %s Mbps",
|
||||
str(bit_rate / 1000000.0), str(bit_rate_max / 1000000.0))
|
||||
|
||||
if bit_rate > bit_rate_max:
|
||||
return "The bit rate is above the configured maximum. Actual: " \
|
||||
f"{bit_rate / 1000000.0} Mbps; Allowed: {bit_rate_max / 1000000.0} Mbps"
|
||||
f"{bit_rate / 1000000.0} Mbps; Allowed max: {bit_rate_max / 1000000.0} Mbps"
|
||||
|
||||
return ""
|
||||
|
||||
@@ -120,7 +183,7 @@ class VideoFileAnalyzer:
|
||||
if {"webm", "ogg"}.intersection(container.split(",")):
|
||||
return ""
|
||||
|
||||
result, _ = await self._execute("ffprobe", f'-v debug "{video_file}"')
|
||||
result, _ = await self._execute_ffprobe(f'-v debug "{video_file}"')
|
||||
match = re.search(r"Before avformat_find_stream_info.+?\s+seeks:(\d+)\s+", result)
|
||||
if match and int(match.group(1)) != 0:
|
||||
return "Video stream descriptors are not at the start of the file (the faststart flag was not used)."
|
||||
@@ -136,6 +199,8 @@ class VideoFileAnalyzer:
|
||||
if not {"aac", "mp3", "flac", "vorbis", "opus"}.intersection(codec.split(",")):
|
||||
return "Audio codec is not in the approved list of AAC, FLAC, MP3, Vorbis, and Opus. " \
|
||||
f"Actual: {codec} [{stream['codec_long_name']}]"
|
||||
if int(stream['sample_rate']) > 48000:
|
||||
return "Sample rate out of range"
|
||||
|
||||
return ""
|
||||
|
||||
@@ -148,8 +213,8 @@ class VideoFileAnalyzer:
|
||||
if not validate_volume:
|
||||
return ""
|
||||
|
||||
result, _ = await self._execute("ffmpeg", f'-i "{video_file}" -t {seconds} '
|
||||
f'-af volumedetect -vn -sn -dn -f null "{os.devnull}"')
|
||||
result, _ = await self._execute_ffmpeg(f'-i "{video_file}" -t {seconds} '
|
||||
f'-af volumedetect -vn -sn -dn -f null "{os.devnull}"')
|
||||
try:
|
||||
mean_volume = float(re.search(r"mean_volume:\s+([-+]?\d*\.\d+|\d+)", result).group(1))
|
||||
max_volume = float(re.search(r"max_volume:\s+([-+]?\d*\.\d+|\d+)", result).group(1))
|
||||
@@ -184,7 +249,7 @@ class VideoFileAnalyzer:
|
||||
# if we don't have h264 use vp9; it's fairly compatible even though it's slow
|
||||
|
||||
if not self._available_encoders:
|
||||
self._available_encoders, _ = await self._execute("ffmpeg", "-encoders -v quiet")
|
||||
self._available_encoders, _ = await self._execute_ffmpeg("-encoders -v quiet")
|
||||
|
||||
encoder = self._conf.video_encoder.split(" ", 1)[0]
|
||||
if re.search(fr"^\s*V..... {encoder} ", self._available_encoders, re.MULTILINE):
|
||||
@@ -218,7 +283,7 @@ class VideoFileAnalyzer:
|
||||
|
||||
wants_opus = extension != "mp4"
|
||||
if not self._available_encoders:
|
||||
self._available_encoders, _ = await self._execute("ffmpeg", "-encoders -v quiet")
|
||||
self._available_encoders, _ = await self._execute_ffmpeg("-encoders -v quiet")
|
||||
|
||||
encoder = self._conf.audio_encoder.split(" ", 1)[0]
|
||||
if wants_opus and 'opus' in encoder:
|
||||
@@ -241,9 +306,6 @@ class VideoFileAnalyzer:
|
||||
|
||||
raise Exception(f"The audio encoder is not available. Requested: {encoder or 'aac'}")
|
||||
|
||||
async def _get_volume_filter(self):
|
||||
return self._conf.volume_filter if self._conf.volume_filter else "-af loudnorm"
|
||||
|
||||
@staticmethod
|
||||
def _get_best_container_extension(scan_data, video_encoder):
|
||||
# the container is chosen by the video format
|
||||
@@ -251,25 +313,27 @@ class VideoFileAnalyzer:
|
||||
# if we are vp8/vp9/av1 we want webm
|
||||
# use mp4 for anything else
|
||||
|
||||
if not video_encoder: # not re-encoding video
|
||||
for stream in scan_data["streams"]:
|
||||
if stream["codec_type"] != "video":
|
||||
continue
|
||||
codec = stream["codec_name"].split(",")
|
||||
if "theora" in codec:
|
||||
return "ogv"
|
||||
if {"vp8", "vp9", "av1"}.intersection(codec):
|
||||
return "webm"
|
||||
if video_encoder: # not re-encoding video
|
||||
if "theora" in video_encoder:
|
||||
return "ogv"
|
||||
if re.search(r"vp[89x]|av1", video_encoder.split(" ", 1)[0]):
|
||||
return "webm"
|
||||
return "mp4"
|
||||
|
||||
for stream in scan_data["streams"]:
|
||||
if stream["codec_type"] != "video":
|
||||
continue
|
||||
codec = stream["codec_name"].split(",")
|
||||
if "theora" in codec:
|
||||
return "ogv"
|
||||
if {"vp8", "vp9", "av1"}.intersection(codec):
|
||||
return "webm"
|
||||
|
||||
if "theora" in video_encoder:
|
||||
return "ogv"
|
||||
elif re.search(r"vp[89x]|av1", video_encoder.split(" ", 1)[0]):
|
||||
return "webm"
|
||||
return "mp4"
|
||||
|
||||
async def _get_scan_data(self, validate, file_path):
|
||||
result, _ = await self._execute("ffprobe",
|
||||
f'-v quiet -print_format json -show_format -show_streams "{file_path}"')
|
||||
arguments = f'-v quiet -print_format json -show_format -show_streams "{file_path}"'
|
||||
result, _ = await self._execute_ffprobe(arguments)
|
||||
try:
|
||||
scan_data = json.loads(result)
|
||||
except Exception as e:
|
||||
@@ -278,7 +342,7 @@ class VideoFileAnalyzer:
|
||||
|
||||
if "format" not in scan_data or "duration" not in scan_data["format"]:
|
||||
log.debug("Format data is missing from ffprobe results for: %s", file_path)
|
||||
raise ValueError(f'Media file does not appear to contain video content at: {file_path}')
|
||||
raise ValueError(f'Media file does not appear to contain video content: {file_path}')
|
||||
|
||||
if float(scan_data["format"]["duration"]) < 0.1:
|
||||
log.debug("Media file appears to be an image: %s", file_path)
|
||||
@@ -286,20 +350,46 @@ class VideoFileAnalyzer:
|
||||
|
||||
return scan_data
|
||||
|
||||
@staticmethod
|
||||
def _build_spec(scan_data):
|
||||
assert scan_data
|
||||
|
||||
duration = ceil(float(scan_data["format"]["duration"])) # existence verified when scan_data made
|
||||
width = -1
|
||||
height = -1
|
||||
for stream in scan_data["streams"]:
|
||||
if stream["codec_type"] != "video":
|
||||
continue
|
||||
width = max(width, int(stream["width"]))
|
||||
height = max(height, int(stream["height"]))
|
||||
|
||||
log.debug(" Detected duration: %d sec. with resolution: %d x %d", duration, width, height)
|
||||
|
||||
spec = {"duration": duration}
|
||||
if height >= 0:
|
||||
spec["height"] = height
|
||||
if width >= 0:
|
||||
spec["width"] = width
|
||||
return spec
|
||||
|
||||
async def verify_or_repair(self, validate, repair, file_path, ignore_non_video=False):
|
||||
if not validate and not repair:
|
||||
return file_path
|
||||
return file_path, {}
|
||||
|
||||
if ignore_non_video and not file_path:
|
||||
return file_path, {}
|
||||
|
||||
await self._verify_ffmpeg_installed()
|
||||
try:
|
||||
scan_data = await self._get_scan_data(validate, file_path)
|
||||
except ValueError:
|
||||
if ignore_non_video:
|
||||
return file_path
|
||||
return file_path, {}
|
||||
raise
|
||||
|
||||
fast_start_msg = await self._verify_fast_start(scan_data, file_path)
|
||||
log.debug("Analyzing %s:", file_path)
|
||||
spec = self._build_spec(scan_data)
|
||||
log.debug(" Detected faststart is %s", "false" if fast_start_msg else "true")
|
||||
container_msg = self._verify_container(scan_data)
|
||||
bitrate_msg = self._verify_bitrate(scan_data, file_path)
|
||||
@@ -309,7 +399,7 @@ class VideoFileAnalyzer:
|
||||
messages = [container_msg, bitrate_msg, fast_start_msg, video_msg, audio_msg, volume_msg]
|
||||
|
||||
if not any(messages):
|
||||
return file_path
|
||||
return file_path, spec
|
||||
|
||||
if not repair:
|
||||
errors = ["Streamability verification failed:"]
|
||||
@@ -319,7 +409,7 @@ class VideoFileAnalyzer:
|
||||
# the plan for transcoding:
|
||||
# we have to re-encode the video if it is in a nonstandard format
|
||||
# we also re-encode if we are h264 but not yuv420p (both errors caught in video_msg)
|
||||
# we also re-encode if our bitrate is too high
|
||||
# we also re-encode if our bitrate or sample rate is too high
|
||||
|
||||
try:
|
||||
transcode_command = [f'-i "{file_path}" -y -c:s copy -c:d copy -c:v']
|
||||
@@ -334,25 +424,26 @@ class VideoFileAnalyzer:
|
||||
transcode_command.append("copy")
|
||||
|
||||
transcode_command.append("-movflags +faststart -c:a")
|
||||
path = pathlib.Path(file_path)
|
||||
extension = self._get_best_container_extension(scan_data, video_encoder)
|
||||
|
||||
if audio_msg or volume_msg:
|
||||
audio_encoder = await self._get_audio_encoder(extension)
|
||||
transcode_command.append(audio_encoder)
|
||||
if volume_msg:
|
||||
volume_filter = await self._get_volume_filter()
|
||||
transcode_command.append(volume_filter)
|
||||
if volume_msg and self._conf.volume_filter:
|
||||
transcode_command.append(self._conf.volume_filter)
|
||||
if audio_msg == "Sample rate out of range":
|
||||
transcode_command.append(" -ar 48000 ")
|
||||
else:
|
||||
transcode_command.append("copy")
|
||||
|
||||
# TODO: put it in a temp folder and delete it after we upload?
|
||||
path = pathlib.Path(file_path)
|
||||
output = path.parent / f"{path.stem}_fixed.{extension}"
|
||||
transcode_command.append(f'"{output}"')
|
||||
|
||||
ffmpeg_command = " ".join(transcode_command)
|
||||
log.info("Proceeding on transcode via: ffmpeg %s", ffmpeg_command)
|
||||
result, code = await self._execute("ffmpeg", ffmpeg_command)
|
||||
result, code = await self._execute_ffmpeg(ffmpeg_command)
|
||||
if code != 0:
|
||||
raise Exception(f"Failure to complete the transcode command. Output: {result}")
|
||||
except Exception as e:
|
||||
@@ -360,6 +451,6 @@ class VideoFileAnalyzer:
|
||||
raise
|
||||
log.info("Unable to transcode %s . Message: %s", file_path, str(e))
|
||||
# TODO: delete partial output file here if it exists?
|
||||
return file_path
|
||||
return file_path, spec
|
||||
|
||||
return str(output)
|
||||
return str(output), spec
|
||||
@@ -0,0 +1,68 @@
|
||||
import time
|
||||
import logging
|
||||
import asyncio
|
||||
import asyncio.tasks
|
||||
from aiohttp import web
|
||||
from prometheus_client import generate_latest as prom_generate_latest
|
||||
from prometheus_client import Counter, Histogram, Gauge
|
||||
|
||||
|
||||
PROBES_IN_FLIGHT = Counter("probes_in_flight", "Number of loop probes in flight", namespace='asyncio')
|
||||
PROBES_FINISHED = Counter("probes_finished", "Number of finished loop probes", namespace='asyncio')
|
||||
PROBE_TIMES = Histogram("probe_times", "Loop probe times", namespace='asyncio')
|
||||
TASK_COUNT = Gauge("running_tasks", "Number of running tasks", namespace='asyncio')
|
||||
|
||||
|
||||
def get_loop_metrics(delay=1):
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def callback(started):
|
||||
PROBE_TIMES.observe(time.perf_counter() - started - delay)
|
||||
PROBES_FINISHED.inc()
|
||||
|
||||
async def monitor_loop_responsiveness():
|
||||
while True:
|
||||
now = time.perf_counter()
|
||||
loop.call_later(delay, callback, now)
|
||||
PROBES_IN_FLIGHT.inc()
|
||||
TASK_COUNT.set(len(asyncio.tasks._all_tasks))
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
return loop.create_task(monitor_loop_responsiveness())
|
||||
|
||||
|
||||
class PrometheusServer:
|
||||
def __init__(self, logger=None):
|
||||
self.runner = None
|
||||
self.logger = logger or logging.getLogger(__name__)
|
||||
self._monitor_loop_task = None
|
||||
|
||||
async def start(self, interface: str, port: int):
|
||||
self.logger.info("start prometheus metrics")
|
||||
prom_app = web.Application()
|
||||
prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
|
||||
self.runner = web.AppRunner(prom_app)
|
||||
await self.runner.setup()
|
||||
|
||||
metrics_site = web.TCPSite(self.runner, interface, port, shutdown_timeout=.5)
|
||||
await metrics_site.start()
|
||||
self.logger.info(
|
||||
'prometheus metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2]
|
||||
)
|
||||
self._monitor_loop_task = get_loop_metrics()
|
||||
|
||||
async def handle_metrics_get_request(self, request: web.Request):
|
||||
try:
|
||||
return web.Response(
|
||||
text=prom_generate_latest().decode(),
|
||||
content_type='text/plain; version=0.0.4'
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception('could not generate prometheus data')
|
||||
raise
|
||||
|
||||
async def stop(self):
|
||||
if self._monitor_loop_task and not self._monitor_loop_task.done():
|
||||
self._monitor_loop_task.cancel()
|
||||
self._monitor_loop_task = None
|
||||
await self.runner.cleanup()
|
||||
@@ -1,5 +0,0 @@
|
||||
build:
|
||||
rm types/v2/* -rf
|
||||
touch types/v2/__init__.py
|
||||
cd types/v2/ && protoc --python_out=. -I ../../../../../types/v2/proto/ ../../../../../types/v2/proto/*.proto
|
||||
sed -e 's/^import\ \(.*\)_pb2\ /from . import\ \1_pb2\ /g' -i types/v2/*.py
|
||||
@@ -1 +0,0 @@
|
||||
from .claim import Claim
|
||||
@@ -1,562 +0,0 @@
|
||||
import json
|
||||
import logging
|
||||
import os.path
|
||||
import hashlib
|
||||
from typing import Tuple, List
|
||||
from string import ascii_letters
|
||||
from decimal import Decimal, ROUND_UP
|
||||
from binascii import hexlify, unhexlify
|
||||
from google.protobuf.json_format import MessageToDict
|
||||
|
||||
from lbry.crypto.base58 import Base58
|
||||
from lbry.constants import COIN
|
||||
|
||||
from lbry.schema.mime_types import guess_media_type
|
||||
from lbry.schema.base import Metadata, BaseMessageList
|
||||
from lbry.schema.tags import clean_tags, normalize_tag
|
||||
from lbry.schema.types.v2.claim_pb2 import (
|
||||
Fee as FeeMessage,
|
||||
Location as LocationMessage,
|
||||
Language as LanguageMessage
|
||||
)
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def calculate_sha384_file_hash(file_path):
|
||||
sha384 = hashlib.sha384()
|
||||
with open(file_path, 'rb') as f:
|
||||
for chunk in iter(lambda: f.read(128 * sha384.block_size), b''):
|
||||
sha384.update(chunk)
|
||||
return sha384.digest()
|
||||
|
||||
|
||||
class Dimmensional(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def width(self) -> int:
|
||||
return self.message.width
|
||||
|
||||
@width.setter
|
||||
def width(self, width: int):
|
||||
self.message.width = width
|
||||
|
||||
@property
|
||||
def height(self) -> int:
|
||||
return self.message.height
|
||||
|
||||
@height.setter
|
||||
def height(self, height: int):
|
||||
self.message.height = height
|
||||
|
||||
@property
|
||||
def dimensions(self) -> Tuple[int, int]:
|
||||
return self.width, self.height
|
||||
|
||||
@dimensions.setter
|
||||
def dimensions(self, dimensions: Tuple[int, int]):
|
||||
self.message.width, self.message.height = dimensions
|
||||
|
||||
def _extract(self, file_metadata, field):
|
||||
try:
|
||||
setattr(self, field, file_metadata.getValues(field)[0])
|
||||
except:
|
||||
log.exception(f'Could not extract {field} from file metadata.')
|
||||
|
||||
def update(self, file_metadata=None, height=None, width=None):
|
||||
if height is not None:
|
||||
self.height = height
|
||||
elif file_metadata:
|
||||
self._extract(file_metadata, 'height')
|
||||
|
||||
if width is not None:
|
||||
self.width = width
|
||||
elif file_metadata:
|
||||
self._extract(file_metadata, 'width')
|
||||
|
||||
|
||||
class Playable(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def duration(self) -> int:
|
||||
return self.message.duration
|
||||
|
||||
@duration.setter
|
||||
def duration(self, duration: int):
|
||||
self.message.duration = duration
|
||||
|
||||
def update(self, file_metadata=None, duration=None):
|
||||
if duration is not None:
|
||||
self.duration = duration
|
||||
elif file_metadata:
|
||||
try:
|
||||
self.duration = file_metadata.getValues('duration')[0].seconds
|
||||
except:
|
||||
log.exception('Could not extract duration from file metadata.')
|
||||
|
||||
|
||||
class Image(Dimmensional):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
|
||||
class Audio(Playable):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
|
||||
class Video(Dimmensional, Playable):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def update(self, file_metadata=None, height=None, width=None, duration=None):
|
||||
Dimmensional.update(self, file_metadata, height, width)
|
||||
Playable.update(self, file_metadata, duration)
|
||||
|
||||
|
||||
class Source(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def update(self, file_path=None):
|
||||
if file_path is not None:
|
||||
self.name = os.path.basename(file_path)
|
||||
self.media_type, stream_type = guess_media_type(file_path)
|
||||
if not os.path.isfile(file_path):
|
||||
raise Exception(f"File does not exist: {file_path}")
|
||||
self.size = os.path.getsize(file_path)
|
||||
if self.size == 0:
|
||||
raise Exception(f"Cannot publish empty file: {file_path}")
|
||||
self.file_hash_bytes = calculate_sha384_file_hash(file_path)
|
||||
return stream_type
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self.message.name
|
||||
|
||||
@name.setter
|
||||
def name(self, name: str):
|
||||
self.message.name = name
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
return self.message.size
|
||||
|
||||
@size.setter
|
||||
def size(self, size: int):
|
||||
self.message.size = size
|
||||
|
||||
@property
|
||||
def media_type(self) -> str:
|
||||
return self.message.media_type
|
||||
|
||||
@media_type.setter
|
||||
def media_type(self, media_type: str):
|
||||
self.message.media_type = media_type
|
||||
|
||||
@property
|
||||
def file_hash(self) -> str:
|
||||
return hexlify(self.message.hash).decode()
|
||||
|
||||
@file_hash.setter
|
||||
def file_hash(self, file_hash: str):
|
||||
self.message.hash = unhexlify(file_hash.encode())
|
||||
|
||||
@property
|
||||
def file_hash_bytes(self) -> bytes:
|
||||
return self.message.hash
|
||||
|
||||
@file_hash_bytes.setter
|
||||
def file_hash_bytes(self, file_hash_bytes: bytes):
|
||||
self.message.hash = file_hash_bytes
|
||||
|
||||
@property
|
||||
def sd_hash(self) -> str:
|
||||
return hexlify(self.message.sd_hash).decode()
|
||||
|
||||
@sd_hash.setter
|
||||
def sd_hash(self, sd_hash: str):
|
||||
self.message.sd_hash = unhexlify(sd_hash.encode())
|
||||
|
||||
@property
|
||||
def sd_hash_bytes(self) -> bytes:
|
||||
return self.message.sd_hash
|
||||
|
||||
@sd_hash_bytes.setter
|
||||
def sd_hash_bytes(self, sd_hash: bytes):
|
||||
self.message.sd_hash = sd_hash
|
||||
|
||||
@property
|
||||
def bt_infohash(self) -> str:
|
||||
return hexlify(self.message.bt_infohash).decode()
|
||||
|
||||
@bt_infohash.setter
|
||||
def bt_infohash(self, bt_infohash: str):
|
||||
self.message.bt_infohash = unhexlify(bt_infohash.encode())
|
||||
|
||||
@property
|
||||
def bt_infohash_bytes(self) -> bytes:
|
||||
return self.message.bt_infohash.decode()
|
||||
|
||||
@bt_infohash_bytes.setter
|
||||
def bt_infohash_bytes(self, bt_infohash: bytes):
|
||||
self.message.bt_infohash = bt_infohash
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
return self.message.url
|
||||
|
||||
@url.setter
|
||||
def url(self, url: str):
|
||||
self.message.url = url
|
||||
|
||||
|
||||
class Fee(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def update(self, address: str = None, currency: str = None, amount=None):
|
||||
if amount:
|
||||
currency = (currency or self.currency or '').lower()
|
||||
if not currency:
|
||||
raise Exception('In order to set a fee amount, please specify a fee currency.')
|
||||
if currency not in ('lbc', 'btc', 'usd'):
|
||||
raise Exception(f'Missing or unknown currency provided: {currency}')
|
||||
setattr(self, currency, Decimal(amount))
|
||||
elif currency:
|
||||
raise Exception('In order to set a fee currency, please specify a fee amount.')
|
||||
if address:
|
||||
if not self.currency:
|
||||
raise Exception('In order to set a fee address, please specify a fee amount and currency.')
|
||||
self.address = address
|
||||
|
||||
@property
|
||||
def currency(self) -> str:
|
||||
if self.message.currency:
|
||||
return FeeMessage.Currency.Name(self.message.currency)
|
||||
|
||||
@property
|
||||
def address(self) -> str:
|
||||
if self.address_bytes:
|
||||
return Base58.encode(self.address_bytes)
|
||||
|
||||
@address.setter
|
||||
def address(self, address: str):
|
||||
self.address_bytes = Base58.decode(address)
|
||||
|
||||
@property
|
||||
def address_bytes(self) -> bytes:
|
||||
return self.message.address
|
||||
|
||||
@address_bytes.setter
|
||||
def address_bytes(self, address: bytes):
|
||||
self.message.address = address
|
||||
|
||||
@property
|
||||
def amount(self) -> Decimal:
|
||||
if self.currency == 'LBC':
|
||||
return self.lbc
|
||||
if self.currency == 'BTC':
|
||||
return self.btc
|
||||
if self.currency == 'USD':
|
||||
return self.usd
|
||||
|
||||
DEWIES = Decimal(COIN)
|
||||
|
||||
@property
|
||||
def lbc(self) -> Decimal:
|
||||
if self.message.currency != FeeMessage.LBC:
|
||||
raise ValueError('LBC can only be returned for LBC fees.')
|
||||
return Decimal(self.message.amount / self.DEWIES)
|
||||
|
||||
@lbc.setter
|
||||
def lbc(self, amount: Decimal):
|
||||
self.dewies = int(amount * self.DEWIES)
|
||||
|
||||
@property
|
||||
def dewies(self) -> int:
|
||||
if self.message.currency != FeeMessage.LBC:
|
||||
raise ValueError('Dewies can only be returned for LBC fees.')
|
||||
return self.message.amount
|
||||
|
||||
@dewies.setter
|
||||
def dewies(self, amount: int):
|
||||
self.message.amount = amount
|
||||
self.message.currency = FeeMessage.LBC
|
||||
|
||||
SATOSHIES = Decimal(COIN)
|
||||
|
||||
@property
|
||||
def btc(self) -> Decimal:
|
||||
if self.message.currency != FeeMessage.BTC:
|
||||
raise ValueError('BTC can only be returned for BTC fees.')
|
||||
return Decimal(self.message.amount / self.SATOSHIES)
|
||||
|
||||
@btc.setter
|
||||
def btc(self, amount: Decimal):
|
||||
self.satoshis = int(amount * self.SATOSHIES)
|
||||
|
||||
@property
|
||||
def satoshis(self) -> int:
|
||||
if self.message.currency != FeeMessage.BTC:
|
||||
raise ValueError('Satoshies can only be returned for BTC fees.')
|
||||
return self.message.amount
|
||||
|
||||
@satoshis.setter
|
||||
def satoshis(self, amount: int):
|
||||
self.message.amount = amount
|
||||
self.message.currency = FeeMessage.BTC
|
||||
|
||||
PENNIES = Decimal('100.0')
|
||||
PENNY = Decimal('0.01')
|
||||
|
||||
@property
|
||||
def usd(self) -> Decimal:
|
||||
if self.message.currency != FeeMessage.USD:
|
||||
raise ValueError('USD can only be returned for USD fees.')
|
||||
return Decimal(self.message.amount / self.PENNIES)
|
||||
|
||||
@usd.setter
|
||||
def usd(self, amount: Decimal):
|
||||
self.pennies = int(amount.quantize(self.PENNY, ROUND_UP) * self.PENNIES)
|
||||
|
||||
@property
|
||||
def pennies(self) -> int:
|
||||
if self.message.currency != FeeMessage.USD:
|
||||
raise ValueError('Pennies can only be returned for USD fees.')
|
||||
return self.message.amount
|
||||
|
||||
@pennies.setter
|
||||
def pennies(self, amount: int):
|
||||
self.message.amount = amount
|
||||
self.message.currency = FeeMessage.USD
|
||||
|
||||
|
||||
class ClaimReference(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def claim_id(self) -> str:
|
||||
return hexlify(self.claim_hash[::-1]).decode()
|
||||
|
||||
@claim_id.setter
|
||||
def claim_id(self, claim_id: str):
|
||||
self.claim_hash = unhexlify(claim_id)[::-1]
|
||||
|
||||
@property
|
||||
def claim_hash(self) -> bytes:
|
||||
return self.message.claim_hash
|
||||
|
||||
@claim_hash.setter
|
||||
def claim_hash(self, claim_hash: bytes):
|
||||
self.message.claim_hash = claim_hash
|
||||
|
||||
|
||||
class ClaimList(BaseMessageList[ClaimReference]):
|
||||
|
||||
__slots__ = ()
|
||||
item_class = ClaimReference
|
||||
|
||||
@property
|
||||
def _message(self):
|
||||
return self.message.claim_references
|
||||
|
||||
def append(self, value):
|
||||
self.add().claim_id = value
|
||||
|
||||
@property
|
||||
def ids(self) -> List[str]:
|
||||
return [c.claim_id for c in self]
|
||||
|
||||
|
||||
class Language(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def langtag(self) -> str:
|
||||
langtag = []
|
||||
if self.language:
|
||||
langtag.append(self.language)
|
||||
if self.script:
|
||||
langtag.append(self.script)
|
||||
if self.region:
|
||||
langtag.append(self.region)
|
||||
return '-'.join(langtag)
|
||||
|
||||
@langtag.setter
|
||||
def langtag(self, langtag: str):
|
||||
parts = langtag.split('-')
|
||||
self.language = parts.pop(0)
|
||||
if parts and len(parts[0]) == 4:
|
||||
self.script = parts.pop(0)
|
||||
if parts and len(parts[0]) == 2 and parts[0].isalpha():
|
||||
self.region = parts.pop(0)
|
||||
if parts and len(parts[0]) == 3 and parts[0].isdigit():
|
||||
self.region = parts.pop(0)
|
||||
assert not parts, f"Failed to parse language tag: {langtag}"
|
||||
|
||||
@property
|
||||
def language(self) -> str:
|
||||
if self.message.language:
|
||||
return LanguageMessage.Language.Name(self.message.language)
|
||||
|
||||
@language.setter
|
||||
def language(self, language: str):
|
||||
self.message.language = LanguageMessage.Language.Value(language)
|
||||
|
||||
@property
|
||||
def script(self) -> str:
|
||||
if self.message.script:
|
||||
return LanguageMessage.Script.Name(self.message.script)
|
||||
|
||||
@script.setter
|
||||
def script(self, script: str):
|
||||
self.message.script = LanguageMessage.Script.Value(script)
|
||||
|
||||
@property
|
||||
def region(self) -> str:
|
||||
if self.message.region:
|
||||
r = LocationMessage.Country.Name(self.message.region)
|
||||
return r[1:] if r.startswith('R') else r
|
||||
|
||||
@region.setter
|
||||
def region(self, region: str):
|
||||
if len(region) == 3:
|
||||
region = 'R'+region
|
||||
self.message.region = LocationMessage.Country.Value(region)
|
||||
|
||||
|
||||
class LanguageList(BaseMessageList[Language]):
|
||||
__slots__ = ()
|
||||
item_class = Language
|
||||
|
||||
def append(self, value: str):
|
||||
self.add().langtag = value
|
||||
|
||||
|
||||
class Location(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def from_value(self, value):
|
||||
if isinstance(value, str) and value.startswith('{'):
|
||||
value = json.loads(value)
|
||||
|
||||
if isinstance(value, dict):
|
||||
for key, val in value.items():
|
||||
setattr(self, key, val)
|
||||
|
||||
elif isinstance(value, str):
|
||||
parts = value.split(':')
|
||||
if len(parts) > 2 or (parts[0] and parts[0][0] in ascii_letters):
|
||||
country = parts and parts.pop(0)
|
||||
if country:
|
||||
self.country = country
|
||||
state = parts and parts.pop(0)
|
||||
if state:
|
||||
self.state = state
|
||||
city = parts and parts.pop(0)
|
||||
if city:
|
||||
self.city = city
|
||||
code = parts and parts.pop(0)
|
||||
if code:
|
||||
self.code = code
|
||||
latitude = parts and parts.pop(0)
|
||||
if latitude:
|
||||
self.latitude = latitude
|
||||
longitude = parts and parts.pop(0)
|
||||
if longitude:
|
||||
self.longitude = longitude
|
||||
|
||||
else:
|
||||
raise ValueError(f'Could not parse country value: {value}')
|
||||
|
||||
def to_dict(self):
|
||||
d = MessageToDict(self.message)
|
||||
if self.message.longitude:
|
||||
d['longitude'] = self.longitude
|
||||
if self.message.latitude:
|
||||
d['latitude'] = self.latitude
|
||||
return d
|
||||
|
||||
@property
|
||||
def country(self) -> str:
|
||||
if self.message.country:
|
||||
return LocationMessage.Country.Name(self.message.country)
|
||||
|
||||
@country.setter
|
||||
def country(self, country: str):
|
||||
self.message.country = LocationMessage.Country.Value(country)
|
||||
|
||||
@property
|
||||
def state(self) -> str:
|
||||
return self.message.state
|
||||
|
||||
@state.setter
|
||||
def state(self, state: str):
|
||||
self.message.state = state
|
||||
|
||||
@property
|
||||
def city(self) -> str:
|
||||
return self.message.city
|
||||
|
||||
@city.setter
|
||||
def city(self, city: str):
|
||||
self.message.city = city
|
||||
|
||||
@property
|
||||
def code(self) -> str:
|
||||
return self.message.code
|
||||
|
||||
@code.setter
|
||||
def code(self, code: str):
|
||||
self.message.code = code
|
||||
|
||||
GPS_PRECISION = Decimal('10000000')
|
||||
|
||||
@property
|
||||
def latitude(self) -> str:
|
||||
if self.message.latitude:
|
||||
return str(Decimal(self.message.latitude) / self.GPS_PRECISION)
|
||||
|
||||
@latitude.setter
|
||||
def latitude(self, latitude: str):
|
||||
latitude = Decimal(latitude)
|
||||
assert -90 <= latitude <= 90, "Latitude must be between -90 and 90 degrees."
|
||||
self.message.latitude = int(latitude * self.GPS_PRECISION)
|
||||
|
||||
@property
|
||||
def longitude(self) -> str:
|
||||
if self.message.longitude:
|
||||
return str(Decimal(self.message.longitude) / self.GPS_PRECISION)
|
||||
|
||||
@longitude.setter
|
||||
def longitude(self, longitude: str):
|
||||
longitude = Decimal(longitude)
|
||||
assert -180 <= longitude <= 180, "Longitude must be between -180 and 180 degrees."
|
||||
self.message.longitude = int(longitude * self.GPS_PRECISION)
|
||||
|
||||
|
||||
class LocationList(BaseMessageList[Location]):
|
||||
__slots__ = ()
|
||||
item_class = Location
|
||||
|
||||
def append(self, value):
|
||||
self.add().from_value(value)
|
||||
|
||||
|
||||
class TagList(BaseMessageList[str]):
|
||||
__slots__ = ()
|
||||
item_class = str
|
||||
|
||||
def append(self, tag: str):
|
||||
tag = normalize_tag(tag)
|
||||
if tag and tag not in self.message:
|
||||
self.message.append(tag)
|
||||
@@ -1,124 +0,0 @@
|
||||
from binascii import hexlify, unhexlify
|
||||
from typing import List, Iterator, TypeVar, Generic
|
||||
|
||||
from google.protobuf.message import DecodeError
|
||||
from google.protobuf.json_format import MessageToDict
|
||||
|
||||
|
||||
class Signable:
|
||||
|
||||
__slots__ = (
|
||||
'message', 'version', 'signature',
|
||||
'signature_type', 'unsigned_payload', 'signing_channel_hash'
|
||||
)
|
||||
|
||||
message_class = None
|
||||
|
||||
def __init__(self, message=None):
|
||||
self.message = message or self.message_class()
|
||||
self.version = 2
|
||||
self.signature = None
|
||||
self.signature_type = 'SECP256k1'
|
||||
self.unsigned_payload = None
|
||||
self.signing_channel_hash = None
|
||||
|
||||
def clear_signature(self):
|
||||
self.signature = None
|
||||
self.unsigned_payload = None
|
||||
self.signing_channel_hash = None
|
||||
|
||||
@property
|
||||
def signing_channel_id(self):
|
||||
return hexlify(self.signing_channel_hash[::-1]).decode() if self.signing_channel_hash else None
|
||||
|
||||
@signing_channel_id.setter
|
||||
def signing_channel_id(self, channel_id: str):
|
||||
self.signing_channel_hash = unhexlify(channel_id)[::-1]
|
||||
|
||||
@property
|
||||
def is_signed(self):
|
||||
return self.signature is not None
|
||||
|
||||
def to_dict(self):
|
||||
return MessageToDict(self.message)
|
||||
|
||||
def to_message_bytes(self) -> bytes:
|
||||
return self.message.SerializeToString()
|
||||
|
||||
def to_bytes(self) -> bytes:
|
||||
pieces = bytearray()
|
||||
if self.is_signed:
|
||||
pieces.append(1)
|
||||
pieces.extend(self.signing_channel_hash)
|
||||
pieces.extend(self.signature)
|
||||
else:
|
||||
pieces.append(0)
|
||||
pieces.extend(self.to_message_bytes())
|
||||
return bytes(pieces)
|
||||
|
||||
@classmethod
|
||||
def from_bytes(cls, data: bytes):
|
||||
signable = cls()
|
||||
if data[0] == 0:
|
||||
signable.message.ParseFromString(data[1:])
|
||||
elif data[0] == 1:
|
||||
signable.signing_channel_hash = data[1:21]
|
||||
signable.signature = data[21:85]
|
||||
signable.message.ParseFromString(data[85:])
|
||||
else:
|
||||
raise DecodeError('Could not determine message format version.')
|
||||
return signable
|
||||
|
||||
def __len__(self):
|
||||
return len(self.to_bytes())
|
||||
|
||||
def __bytes__(self):
|
||||
return self.to_bytes()
|
||||
|
||||
|
||||
class Metadata:
|
||||
|
||||
__slots__ = 'message',
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
|
||||
|
||||
I = TypeVar('I')
|
||||
|
||||
|
||||
class BaseMessageList(Metadata, Generic[I]):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
item_class = None
|
||||
|
||||
@property
|
||||
def _message(self):
|
||||
return self.message
|
||||
|
||||
def add(self) -> I:
|
||||
return self.item_class(self._message.add())
|
||||
|
||||
def extend(self, values: List[str]):
|
||||
for value in values:
|
||||
self.append(value)
|
||||
|
||||
def append(self, value: str):
|
||||
raise NotImplemented
|
||||
|
||||
def __len__(self):
|
||||
return len(self._message)
|
||||
|
||||
def __iter__(self) -> Iterator[I]:
|
||||
for item in self._message:
|
||||
yield self.item_class(item)
|
||||
|
||||
def __getitem__(self, item) -> I:
|
||||
return self.item_class(self._message[item])
|
||||
|
||||
def __delitem__(self, key):
|
||||
del self._message[key]
|
||||
|
||||
def __eq__(self, other) -> bool:
|
||||
return self._message == other
|
||||
@@ -1,415 +0,0 @@
|
||||
import logging
|
||||
from typing import List
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
from google.protobuf.json_format import MessageToDict
|
||||
from google.protobuf.message import DecodeError
|
||||
from hachoir.core.log import log as hachoir_log
|
||||
from hachoir.parser import createParser as binary_file_parser
|
||||
from hachoir.metadata import extractMetadata as binary_file_metadata
|
||||
|
||||
from lbry.schema import compat
|
||||
from lbry.schema.base import Signable
|
||||
from lbry.schema.mime_types import guess_media_type, guess_stream_type
|
||||
from lbry.schema.attrs import (
|
||||
Source, Playable, Dimmensional, Fee, Image, Video, Audio,
|
||||
LanguageList, LocationList, ClaimList, ClaimReference, TagList
|
||||
)
|
||||
from lbry.schema.types.v2.claim_pb2 import Claim as ClaimMessage
|
||||
from lbry.error import InputValueIsNoneError
|
||||
|
||||
|
||||
hachoir_log.use_print = False
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Claim(Signable):
|
||||
|
||||
STREAM = 'stream'
|
||||
CHANNEL = 'channel'
|
||||
COLLECTION = 'collection'
|
||||
REPOST = 'repost'
|
||||
|
||||
__slots__ = 'version',
|
||||
|
||||
message_class = ClaimMessage
|
||||
|
||||
def __init__(self, message=None):
|
||||
super().__init__(message)
|
||||
self.version = 2
|
||||
|
||||
@property
|
||||
def claim_type(self) -> str:
|
||||
return self.message.WhichOneof('type')
|
||||
|
||||
def get_message(self, type_name):
|
||||
message = getattr(self.message, type_name)
|
||||
if self.claim_type is None:
|
||||
message.SetInParent()
|
||||
if self.claim_type != type_name:
|
||||
raise ValueError(f'Claim is not a {type_name}.')
|
||||
return message
|
||||
|
||||
@property
|
||||
def is_stream(self):
|
||||
return self.claim_type == self.STREAM
|
||||
|
||||
@property
|
||||
def stream(self) -> 'Stream':
|
||||
return Stream(self)
|
||||
|
||||
@property
|
||||
def is_channel(self):
|
||||
return self.claim_type == self.CHANNEL
|
||||
|
||||
@property
|
||||
def channel(self) -> 'Channel':
|
||||
return Channel(self)
|
||||
|
||||
@property
|
||||
def is_repost(self):
|
||||
return self.claim_type == self.REPOST
|
||||
|
||||
@property
|
||||
def repost(self) -> 'Repost':
|
||||
return Repost(self)
|
||||
|
||||
@property
|
||||
def is_collection(self):
|
||||
return self.claim_type == self.COLLECTION
|
||||
|
||||
@property
|
||||
def collection(self) -> 'Collection':
|
||||
return Collection(self)
|
||||
|
||||
@classmethod
|
||||
def from_bytes(cls, data: bytes) -> 'Claim':
|
||||
try:
|
||||
return super().from_bytes(data)
|
||||
except DecodeError:
|
||||
claim = cls()
|
||||
if data[0] == ord('{'):
|
||||
claim.version = 0
|
||||
compat.from_old_json_schema(claim, data)
|
||||
elif data[0] not in (0, 1):
|
||||
claim.version = 1
|
||||
compat.from_types_v1(claim, data)
|
||||
else:
|
||||
raise
|
||||
return claim
|
||||
|
||||
|
||||
class BaseClaim:
|
||||
|
||||
__slots__ = 'claim', 'message'
|
||||
|
||||
claim_type = None
|
||||
object_fields = 'thumbnail',
|
||||
repeat_fields = 'tags', 'languages', 'locations'
|
||||
|
||||
def __init__(self, claim: Claim = None):
|
||||
self.claim = claim or Claim()
|
||||
self.message = self.claim.get_message(self.claim_type)
|
||||
|
||||
def to_dict(self):
|
||||
claim = MessageToDict(self.claim.message, preserving_proto_field_name=True)
|
||||
claim.update(claim.pop(self.claim_type))
|
||||
if 'languages' in claim:
|
||||
claim['languages'] = self.langtags
|
||||
if 'locations' in claim:
|
||||
claim['locations'] = [l.to_dict() for l in self.locations]
|
||||
return claim
|
||||
|
||||
def none_check(self, kwargs):
|
||||
for key, value in kwargs.items():
|
||||
if value is None:
|
||||
raise InputValueIsNoneError(key)
|
||||
|
||||
def update(self, **kwargs):
|
||||
self.none_check(kwargs)
|
||||
|
||||
for key in list(kwargs):
|
||||
for field in self.object_fields:
|
||||
if key.startswith(f'{field}_'):
|
||||
attr = getattr(self, field)
|
||||
setattr(attr, key[len(f'{field}_'):], kwargs.pop(key))
|
||||
continue
|
||||
|
||||
for l in self.repeat_fields:
|
||||
field = getattr(self, l)
|
||||
if kwargs.pop(f'clear_{l}', False):
|
||||
del field[:]
|
||||
items = kwargs.pop(l, None)
|
||||
if items is not None:
|
||||
if isinstance(items, str):
|
||||
field.append(items)
|
||||
elif isinstance(items, list):
|
||||
field.extend(items)
|
||||
else:
|
||||
raise ValueError(f"Unknown {l} value: {items}")
|
||||
|
||||
for key, value in kwargs.items():
|
||||
setattr(self, key, value)
|
||||
|
||||
@property
|
||||
def title(self) -> str:
|
||||
return self.claim.message.title
|
||||
|
||||
@title.setter
|
||||
def title(self, title: str):
|
||||
self.claim.message.title = title
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
return self.claim.message.description
|
||||
|
||||
@description.setter
|
||||
def description(self, description: str):
|
||||
self.claim.message.description = description
|
||||
|
||||
@property
|
||||
def thumbnail(self) -> Source:
|
||||
return Source(self.claim.message.thumbnail)
|
||||
|
||||
@property
|
||||
def tags(self) -> List[str]:
|
||||
return TagList(self.claim.message.tags)
|
||||
|
||||
@property
|
||||
def languages(self) -> LanguageList:
|
||||
return LanguageList(self.claim.message.languages)
|
||||
|
||||
@property
|
||||
def langtags(self) -> List[str]:
|
||||
return [l.langtag for l in self.languages]
|
||||
|
||||
@property
|
||||
def locations(self) -> LocationList:
|
||||
return LocationList(self.claim.message.locations)
|
||||
|
||||
|
||||
class Stream(BaseClaim):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
claim_type = Claim.STREAM
|
||||
|
||||
object_fields = BaseClaim.object_fields + ('source',)
|
||||
|
||||
def to_dict(self):
|
||||
claim = super().to_dict()
|
||||
if 'source' in claim:
|
||||
if 'hash' in claim['source']:
|
||||
claim['source']['hash'] = self.source.file_hash
|
||||
if 'sd_hash' in claim['source']:
|
||||
claim['source']['sd_hash'] = self.source.sd_hash
|
||||
elif 'bt_infohash' in claim['source']:
|
||||
claim['source']['bt_infohash'] = self.source.bt_infohash
|
||||
if 'media_type' in claim['source']:
|
||||
claim['stream_type'] = guess_stream_type(claim['source']['media_type'])
|
||||
fee = claim.get('fee', {})
|
||||
if 'address' in fee:
|
||||
fee['address'] = self.fee.address
|
||||
if 'amount' in fee:
|
||||
fee['amount'] = str(self.fee.amount)
|
||||
return claim
|
||||
|
||||
def update(self, file_path=None, height=None, width=None, duration=None, **kwargs):
|
||||
|
||||
if kwargs.pop('clear_fee', False):
|
||||
self.message.ClearField('fee')
|
||||
else:
|
||||
self.fee.update(
|
||||
kwargs.pop('fee_address', None),
|
||||
kwargs.pop('fee_currency', None),
|
||||
kwargs.pop('fee_amount', None)
|
||||
)
|
||||
|
||||
self.none_check(kwargs)
|
||||
|
||||
if 'sd_hash' in kwargs:
|
||||
self.source.sd_hash = kwargs.pop('sd_hash')
|
||||
elif 'bt_infohash' in kwargs:
|
||||
self.source.bt_infohash = kwargs.pop('bt_infohash')
|
||||
if 'file_name' in kwargs:
|
||||
self.source.name = kwargs.pop('file_name')
|
||||
if 'file_hash' in kwargs:
|
||||
self.source.file_hash = kwargs.pop('file_hash')
|
||||
|
||||
stream_type = None
|
||||
if file_path is not None:
|
||||
stream_type = self.source.update(file_path=file_path)
|
||||
elif self.source.name:
|
||||
self.source.media_type, stream_type = guess_media_type(self.source.name)
|
||||
elif self.source.media_type:
|
||||
stream_type = guess_stream_type(self.source.media_type)
|
||||
|
||||
if 'file_size' in kwargs:
|
||||
self.source.size = kwargs.pop('file_size')
|
||||
|
||||
if self.stream_type is not None and self.stream_type != stream_type:
|
||||
self.message.ClearField(self.stream_type)
|
||||
|
||||
if stream_type in ('image', 'video', 'audio'):
|
||||
media = getattr(self, stream_type)
|
||||
media_args = {'file_metadata': None}
|
||||
if file_path is not None:
|
||||
try:
|
||||
media_args['file_metadata'] = binary_file_metadata(binary_file_parser(file_path))
|
||||
except:
|
||||
log.exception('Could not read file metadata.')
|
||||
if isinstance(media, Playable):
|
||||
media_args['duration'] = duration
|
||||
if isinstance(media, Dimmensional):
|
||||
media_args['height'] = height
|
||||
media_args['width'] = width
|
||||
media.update(**media_args)
|
||||
|
||||
super().update(**kwargs)
|
||||
|
||||
@property
|
||||
def author(self) -> str:
|
||||
return self.message.author
|
||||
|
||||
@author.setter
|
||||
def author(self, author: str):
|
||||
self.message.author = author
|
||||
|
||||
@property
|
||||
def license(self) -> str:
|
||||
return self.message.license
|
||||
|
||||
@license.setter
|
||||
def license(self, license: str):
|
||||
self.message.license = license
|
||||
|
||||
@property
|
||||
def license_url(self) -> str:
|
||||
return self.message.license_url
|
||||
|
||||
@license_url.setter
|
||||
def license_url(self, license_url: str):
|
||||
self.message.license_url = license_url
|
||||
|
||||
@property
|
||||
def release_time(self) -> int:
|
||||
return self.message.release_time
|
||||
|
||||
@release_time.setter
|
||||
def release_time(self, release_time: int):
|
||||
self.message.release_time = release_time
|
||||
|
||||
@property
|
||||
def fee(self) -> Fee:
|
||||
return Fee(self.message.fee)
|
||||
|
||||
@property
|
||||
def has_fee(self) -> bool:
|
||||
return self.message.HasField('fee')
|
||||
|
||||
@property
|
||||
def source(self) -> Source:
|
||||
return Source(self.message.source)
|
||||
|
||||
@property
|
||||
def stream_type(self) -> str:
|
||||
return self.message.WhichOneof('type')
|
||||
|
||||
@property
|
||||
def image(self) -> Image:
|
||||
return Image(self.message.image)
|
||||
|
||||
@property
|
||||
def video(self) -> Video:
|
||||
return Video(self.message.video)
|
||||
|
||||
@property
|
||||
def audio(self) -> Audio:
|
||||
return Audio(self.message.audio)
|
||||
|
||||
|
||||
class Channel(BaseClaim):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
claim_type = Claim.CHANNEL
|
||||
|
||||
object_fields = BaseClaim.object_fields + ('cover',)
|
||||
repeat_fields = BaseClaim.repeat_fields + ('featured',)
|
||||
|
||||
def to_dict(self):
|
||||
claim = super().to_dict()
|
||||
claim['public_key'] = self.public_key
|
||||
if 'featured' in claim:
|
||||
claim['featured'] = self.featured.ids
|
||||
return claim
|
||||
|
||||
@property
|
||||
def public_key(self) -> str:
|
||||
return hexlify(self.message.public_key).decode()
|
||||
|
||||
@public_key.setter
|
||||
def public_key(self, sd_public_key: str):
|
||||
self.message.public_key = unhexlify(sd_public_key.encode())
|
||||
|
||||
@property
|
||||
def public_key_bytes(self) -> bytes:
|
||||
return self.message.public_key
|
||||
|
||||
@public_key_bytes.setter
|
||||
def public_key_bytes(self, public_key: bytes):
|
||||
self.message.public_key = public_key
|
||||
|
||||
@property
|
||||
def email(self) -> str:
|
||||
return self.message.email
|
||||
|
||||
@email.setter
|
||||
def email(self, email: str):
|
||||
self.message.email = email
|
||||
|
||||
@property
|
||||
def website_url(self) -> str:
|
||||
return self.message.website_url
|
||||
|
||||
@website_url.setter
|
||||
def website_url(self, website_url: str):
|
||||
self.message.website_url = website_url
|
||||
|
||||
@property
|
||||
def cover(self) -> Source:
|
||||
return Source(self.message.cover)
|
||||
|
||||
@property
|
||||
def featured(self) -> ClaimList:
|
||||
return ClaimList(self.message.featured)
|
||||
|
||||
|
||||
class Repost(BaseClaim):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
claim_type = Claim.REPOST
|
||||
|
||||
@property
|
||||
def reference(self) -> ClaimReference:
|
||||
return ClaimReference(self.message)
|
||||
|
||||
|
||||
class Collection(BaseClaim):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
claim_type = Claim.COLLECTION
|
||||
|
||||
repeat_fields = BaseClaim.repeat_fields + ('claims',)
|
||||
|
||||
def to_dict(self):
|
||||
claim = super().to_dict()
|
||||
if claim.pop('claim_references', None):
|
||||
claim['claims'] = self.claims.ids
|
||||
return claim
|
||||
|
||||
@property
|
||||
def claims(self) -> ClaimList:
|
||||
return ClaimList(self.message)
|
||||
@@ -1,93 +0,0 @@
|
||||
import json
|
||||
from decimal import Decimal
|
||||
|
||||
from google.protobuf.message import DecodeError
|
||||
|
||||
from lbry.schema.types.v1.legacy_claim_pb2 import Claim as OldClaimMessage
|
||||
from lbry.schema.types.v1.certificate_pb2 import KeyType
|
||||
from lbry.schema.types.v1.fee_pb2 import Fee as FeeMessage
|
||||
|
||||
|
||||
def from_old_json_schema(claim, payload: bytes):
|
||||
try:
|
||||
value = json.loads(payload)
|
||||
except:
|
||||
raise DecodeError('Could not parse JSON.')
|
||||
stream = claim.stream
|
||||
stream.source.sd_hash = value['sources']['lbry_sd_hash']
|
||||
stream.source.media_type = (
|
||||
value.get('content_type', value.get('content-type')) or
|
||||
'application/octet-stream'
|
||||
)
|
||||
stream.title = value.get('title', '')
|
||||
stream.description = value.get('description', '')
|
||||
if value.get('thumbnail', ''):
|
||||
stream.thumbnail.url = value.get('thumbnail', '')
|
||||
stream.author = value.get('author', '')
|
||||
stream.license = value.get('license', '')
|
||||
stream.license_url = value.get('license_url', '')
|
||||
language = value.get('language', '')
|
||||
if language:
|
||||
if language.lower() == 'english':
|
||||
language = 'en'
|
||||
try:
|
||||
stream.languages.append(language)
|
||||
except:
|
||||
pass
|
||||
if value.get('nsfw', False):
|
||||
stream.tags.append('mature')
|
||||
if "fee" in value and isinstance(value['fee'], dict):
|
||||
fee = value["fee"]
|
||||
currency = list(fee.keys())[0]
|
||||
if currency == 'LBC':
|
||||
stream.fee.lbc = Decimal(fee[currency]['amount'])
|
||||
elif currency == 'USD':
|
||||
stream.fee.usd = Decimal(fee[currency]['amount'])
|
||||
elif currency == 'BTC':
|
||||
stream.fee.btc = Decimal(fee[currency]['amount'])
|
||||
else:
|
||||
raise DecodeError(f'Unknown currency: {currency}')
|
||||
stream.fee.address = fee[currency]['address']
|
||||
return claim
|
||||
|
||||
|
||||
def from_types_v1(claim, payload: bytes):
|
||||
old = OldClaimMessage()
|
||||
old.ParseFromString(payload)
|
||||
if old.claimType == 2:
|
||||
channel = claim.channel
|
||||
channel.public_key_bytes = old.certificate.publicKey
|
||||
else:
|
||||
stream = claim.stream
|
||||
stream.title = old.stream.metadata.title
|
||||
stream.description = old.stream.metadata.description
|
||||
stream.author = old.stream.metadata.author
|
||||
stream.license = old.stream.metadata.license
|
||||
stream.license_url = old.stream.metadata.licenseUrl
|
||||
stream.thumbnail.url = old.stream.metadata.thumbnail
|
||||
if old.stream.metadata.HasField('language'):
|
||||
stream.languages.add().message.language = old.stream.metadata.language
|
||||
stream.source.media_type = old.stream.source.contentType
|
||||
stream.source.sd_hash_bytes = old.stream.source.source
|
||||
if old.stream.metadata.nsfw:
|
||||
stream.tags.append('mature')
|
||||
if old.stream.metadata.HasField('fee'):
|
||||
fee = old.stream.metadata.fee
|
||||
stream.fee.address_bytes = fee.address
|
||||
currency = FeeMessage.Currency.Name(fee.currency)
|
||||
if currency == 'LBC':
|
||||
stream.fee.lbc = Decimal(fee.amount)
|
||||
elif currency == 'USD':
|
||||
stream.fee.usd = Decimal(fee.amount)
|
||||
elif currency == 'BTC':
|
||||
stream.fee.btc = Decimal(fee.amount)
|
||||
else:
|
||||
raise DecodeError(f'Unsupported currency: {currency}')
|
||||
if old.HasField('publisherSignature'):
|
||||
sig = old.publisherSignature
|
||||
claim.signature = sig.signature
|
||||
claim.signature_type = KeyType.Name(sig.signatureType)
|
||||
claim.signing_channel_hash = sig.certificateId[::-1]
|
||||
old.ClearField("publisherSignature")
|
||||
claim.unsigned_payload = old.SerializeToString()
|
||||
return claim
|
||||
@@ -1,179 +0,0 @@
|
||||
import os
|
||||
|
||||
types_map = {
|
||||
# http://www.iana.org/assignments/media-types
|
||||
# Type mapping for automated metadata extraction (video, audio, image, document, binary, model)
|
||||
'.a': ('application/octet-stream', 'binary'),
|
||||
'.ai': ('application/postscript', 'image'),
|
||||
'.aif': ('audio/x-aiff', 'audio'),
|
||||
'.aifc': ('audio/x-aiff', 'audio'),
|
||||
'.aiff': ('audio/x-aiff', 'audio'),
|
||||
'.au': ('audio/basic', 'audio'),
|
||||
'.avi': ('video/x-msvideo', 'video'),
|
||||
'.bat': ('text/plain', 'document'),
|
||||
'.bcpio': ('application/x-bcpio', 'binary'),
|
||||
'.bin': ('application/octet-stream', 'binary'),
|
||||
'.bmp': ('image/bmp', 'image'),
|
||||
'.c': ('text/plain', 'document'),
|
||||
'.cdf': ('application/x-netcdf', 'binary'),
|
||||
'.cpio': ('application/x-cpio', 'binary'),
|
||||
'.csh': ('application/x-csh', 'binary'),
|
||||
'.css': ('text/css', 'document'),
|
||||
'.csv': ('text/csv', 'document'),
|
||||
'.dll': ('application/octet-stream', 'binary'),
|
||||
'.doc': ('application/msword', 'document'),
|
||||
'.dot': ('application/msword', 'document'),
|
||||
'.dvi': ('application/x-dvi', 'binary'),
|
||||
'.eml': ('message/rfc822', 'document'),
|
||||
'.eps': ('application/postscript', 'document'),
|
||||
'.epub': ('application/epub+zip', 'document'),
|
||||
'.etx': ('text/x-setext', 'document'),
|
||||
'.exe': ('application/octet-stream', 'binary'),
|
||||
'.gif': ('image/gif', 'image'),
|
||||
'.gtar': ('application/x-gtar', 'binary'),
|
||||
'.h': ('text/plain', 'document'),
|
||||
'.hdf': ('application/x-hdf', 'binary'),
|
||||
'.htm': ('text/html', 'document'),
|
||||
'.html': ('text/html', 'document'),
|
||||
'.ico': ('image/vnd.microsoft.icon', 'image'),
|
||||
'.ief': ('image/ief', 'image'),
|
||||
'.iges': ('model/iges', 'model'),
|
||||
'.jpe': ('image/jpeg', 'image'),
|
||||
'.jpeg': ('image/jpeg', 'image'),
|
||||
'.jpg': ('image/jpeg', 'image'),
|
||||
'.js': ('application/javascript', 'document'),
|
||||
'.json': ('application/json', 'document'),
|
||||
'.ksh': ('text/plain', 'document'),
|
||||
'.latex': ('application/x-latex', 'binary'),
|
||||
'.m1v': ('video/mpeg', 'video'),
|
||||
'.m3u': ('application/vnd.apple.mpegurl', 'audio'),
|
||||
'.m3u8': ('application/vnd.apple.mpegurl', 'audio'),
|
||||
'.man': ('application/x-troff-man', 'document'),
|
||||
'.markdown': ('text/markdown', 'document'),
|
||||
'.md': ('text/markdown', 'document'),
|
||||
'.me': ('application/x-troff-me', 'binary'),
|
||||
'.mht': ('message/rfc822', 'document'),
|
||||
'.mhtml': ('message/rfc822', 'document'),
|
||||
'.mif': ('application/x-mif', 'binary'),
|
||||
'.mov': ('video/quicktime', 'video'),
|
||||
'.movie': ('video/x-sgi-movie', 'video'),
|
||||
'.mp2': ('audio/mpeg', 'audio'),
|
||||
'.mp3': ('audio/mpeg', 'audio'),
|
||||
'.mp4': ('video/mp4', 'video'),
|
||||
'.mpa': ('video/mpeg', 'video'),
|
||||
'.mpe': ('video/mpeg', 'video'),
|
||||
'.mpeg': ('video/mpeg', 'video'),
|
||||
'.mpg': ('video/mpeg', 'video'),
|
||||
'.ms': ('application/x-troff-ms', 'binary'),
|
||||
'.nc': ('application/x-netcdf', 'binary'),
|
||||
'.nws': ('message/rfc822', 'document'),
|
||||
'.o': ('application/octet-stream', 'binary'),
|
||||
'.obj': ('application/octet-stream', 'model'),
|
||||
'.oda': ('application/oda', 'binary'),
|
||||
'.p12': ('application/x-pkcs12', 'binary'),
|
||||
'.p7c': ('application/pkcs7-mime', 'binary'),
|
||||
'.pbm': ('image/x-portable-bitmap', 'image'),
|
||||
'.pdf': ('application/pdf', 'document'),
|
||||
'.pfx': ('application/x-pkcs12', 'binary'),
|
||||
'.pgm': ('image/x-portable-graymap', 'image'),
|
||||
'.pl': ('text/plain', 'document'),
|
||||
'.png': ('image/png', 'image'),
|
||||
'.pnm': ('image/x-portable-anymap', 'image'),
|
||||
'.pot': ('application/vnd.ms-powerpoint', 'document'),
|
||||
'.ppa': ('application/vnd.ms-powerpoint', 'document'),
|
||||
'.ppm': ('image/x-portable-pixmap', 'image'),
|
||||
'.pps': ('application/vnd.ms-powerpoint', 'document'),
|
||||
'.ppt': ('application/vnd.ms-powerpoint', 'document'),
|
||||
'.ps': ('application/postscript', 'document'),
|
||||
'.pwz': ('application/vnd.ms-powerpoint', 'document'),
|
||||
'.py': ('text/x-python', 'document'),
|
||||
'.pyc': ('application/x-python-code', 'binary'),
|
||||
'.pyo': ('application/x-python-code', 'binary'),
|
||||
'.qt': ('video/quicktime', 'video'),
|
||||
'.ra': ('audio/x-pn-realaudio', 'audio'),
|
||||
'.ram': ('application/x-pn-realaudio', 'audio'),
|
||||
'.ras': ('image/x-cmu-raster', 'image'),
|
||||
'.rdf': ('application/xml', 'binary'),
|
||||
'.rgb': ('image/x-rgb', 'image'),
|
||||
'.roff': ('application/x-troff', 'binary'),
|
||||
'.rtx': ('text/richtext', 'document'),
|
||||
'.sgm': ('text/x-sgml', 'document'),
|
||||
'.sgml': ('text/x-sgml', 'document'),
|
||||
'.sh': ('application/x-sh', 'document'),
|
||||
'.shar': ('application/x-shar', 'binary'),
|
||||
'.snd': ('audio/basic', 'audio'),
|
||||
'.so': ('application/octet-stream', 'binary'),
|
||||
'.src': ('application/x-wais-source', 'binary'),
|
||||
'.stl': ('model/stl', 'model'),
|
||||
'.sv4cpio': ('application/x-sv4cpio', 'binary'),
|
||||
'.sv4crc': ('application/x-sv4crc', 'binary'),
|
||||
'.svg': ('image/svg+xml', 'image'),
|
||||
'.swf': ('application/x-shockwave-flash', 'binary'),
|
||||
'.t': ('application/x-troff', 'binary'),
|
||||
'.tar': ('application/x-tar', 'binary'),
|
||||
'.tcl': ('application/x-tcl', 'binary'),
|
||||
'.tex': ('application/x-tex', 'binary'),
|
||||
'.texi': ('application/x-texinfo', 'binary'),
|
||||
'.texinfo': ('application/x-texinfo', 'binary'),
|
||||
'.tif': ('image/tiff', 'image'),
|
||||
'.tiff': ('image/tiff', 'image'),
|
||||
'.tr': ('application/x-troff', 'binary'),
|
||||
'.tsv': ('text/tab-separated-values', 'document'),
|
||||
'.txt': ('text/plain', 'document'),
|
||||
'.ustar': ('application/x-ustar', 'binary'),
|
||||
'.vcf': ('text/x-vcard', 'document'),
|
||||
'.wav': ('audio/x-wav', 'audio'),
|
||||
'.webm': ('video/webm', 'video'),
|
||||
'.wiz': ('application/msword', 'document'),
|
||||
'.wsdl': ('application/xml', 'document'),
|
||||
'.xbm': ('image/x-xbitmap', 'image'),
|
||||
'.xlb': ('application/vnd.ms-excel', 'document'),
|
||||
'.xls': ('application/vnd.ms-excel', 'document'),
|
||||
'.xml': ('text/xml', 'document'),
|
||||
'.xpdl': ('application/xml', 'document'),
|
||||
'.xpm': ('image/x-xpixmap', 'image'),
|
||||
'.xsl': ('application/xml', 'document'),
|
||||
'.xwd': ('image/x-xwindowdump', 'image'),
|
||||
'.zip': ('application/zip', 'binary'),
|
||||
|
||||
# These are non-standard types, commonly found in the wild.
|
||||
'.cbr': ('application/vnd.comicbook-rar', 'document'),
|
||||
'.cbz': ('application/vnd.comicbook+zip', 'document'),
|
||||
'.flac': ('audio/flac', 'audio'),
|
||||
'.lbry': ('application/x-ext-lbry', 'document'),
|
||||
'.m4v': ('video/m4v', 'video'),
|
||||
'.mid': ('audio/midi', 'audio'),
|
||||
'.midi': ('audio/midi', 'audio'),
|
||||
'.mkv': ('video/x-matroska', 'video'),
|
||||
'.mobi': ('application/x-mobipocket-ebook', 'document'),
|
||||
'.oga': ('audio/ogg', 'audio'),
|
||||
'.ogv': ('video/ogg', 'video'),
|
||||
'.ogg': ('video/ogg', 'video'),
|
||||
'.pct': ('image/pict', 'image'),
|
||||
'.pic': ('image/pict', 'image'),
|
||||
'.pict': ('image/pict', 'image'),
|
||||
'.prc': ('application/x-mobipocket-ebook', 'document'),
|
||||
'.rtf': ('application/rtf', 'document'),
|
||||
'.xul': ('text/xul', 'document'),
|
||||
|
||||
# microsoft is special and has its own 'standard'
|
||||
# https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
|
||||
'.wmv': ('video/x-ms-wmv', 'video')
|
||||
}
|
||||
|
||||
|
||||
def guess_media_type(path):
|
||||
_, ext = os.path.splitext(path)
|
||||
extension = ext.strip().lower()
|
||||
if extension[1:]:
|
||||
if extension in types_map:
|
||||
return types_map[extension]
|
||||
return f'application/x-ext-{extension[1:]}', 'binary'
|
||||
return 'application/octet-stream', 'binary'
|
||||
|
||||
|
||||
def guess_stream_type(media_type):
|
||||
for media, stream in types_map.values():
|
||||
if media == media_type:
|
||||
return stream
|
||||
return 'binary'
|
||||
@@ -1,47 +0,0 @@
|
||||
from google.protobuf.message import DecodeError
|
||||
from google.protobuf.json_format import MessageToDict
|
||||
from lbry.schema.types.v2.purchase_pb2 import Purchase as PurchaseMessage
|
||||
from .attrs import ClaimReference
|
||||
|
||||
|
||||
class Purchase(ClaimReference):
|
||||
|
||||
START_BYTE = ord('P')
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def __init__(self, claim_id=None):
|
||||
super().__init__(PurchaseMessage())
|
||||
if claim_id is not None:
|
||||
self.claim_id = claim_id
|
||||
|
||||
def to_dict(self):
|
||||
return MessageToDict(self.message)
|
||||
|
||||
def to_message_bytes(self) -> bytes:
|
||||
return self.message.SerializeToString()
|
||||
|
||||
def to_bytes(self) -> bytes:
|
||||
pieces = bytearray()
|
||||
pieces.append(self.START_BYTE)
|
||||
pieces.extend(self.to_message_bytes())
|
||||
return bytes(pieces)
|
||||
|
||||
@classmethod
|
||||
def has_start_byte(cls, data: bytes):
|
||||
return data and data[0] == cls.START_BYTE
|
||||
|
||||
@classmethod
|
||||
def from_bytes(cls, data: bytes):
|
||||
purchase = cls()
|
||||
if purchase.has_start_byte(data):
|
||||
purchase.message.ParseFromString(data[1:])
|
||||
else:
|
||||
raise DecodeError('Message does not start with correct byte.')
|
||||
return purchase
|
||||
|
||||
def __len__(self):
|
||||
return len(self.to_bytes())
|
||||
|
||||
def __bytes__(self):
|
||||
return self.to_bytes()
|
||||
@@ -1,210 +0,0 @@
|
||||
import base64
|
||||
import struct
|
||||
from typing import List
|
||||
from binascii import hexlify
|
||||
from itertools import chain
|
||||
|
||||
from lbry.error import ResolveCensoredError
|
||||
from lbry.schema.types.v2.result_pb2 import Outputs as OutputsMessage
|
||||
from lbry.schema.types.v2.result_pb2 import Error as ErrorMessage
|
||||
|
||||
INVALID = ErrorMessage.Code.Name(ErrorMessage.INVALID)
|
||||
NOT_FOUND = ErrorMessage.Code.Name(ErrorMessage.NOT_FOUND)
|
||||
BLOCKED = ErrorMessage.Code.Name(ErrorMessage.BLOCKED)
|
||||
|
||||
|
||||
def set_reference(reference, claim_hash, rows):
|
||||
if claim_hash:
|
||||
for txo in rows:
|
||||
if claim_hash == txo['claim_hash']:
|
||||
reference.tx_hash = txo['txo_hash'][:32]
|
||||
reference.nout = struct.unpack('<I', txo['txo_hash'][32:])[0]
|
||||
reference.height = txo['height']
|
||||
return
|
||||
|
||||
|
||||
class Censor:
|
||||
|
||||
__slots__ = 'streams', 'channels', 'censored', 'total'
|
||||
|
||||
def __init__(self, streams: dict = None, channels: dict = None):
|
||||
self.streams = streams or {}
|
||||
self.channels = channels or {}
|
||||
self.censored = {}
|
||||
self.total = 0
|
||||
|
||||
def censor(self, row) -> bool:
|
||||
was_censored = False
|
||||
for claim_hash, lookup in (
|
||||
(row['claim_hash'], self.streams),
|
||||
(row['claim_hash'], self.channels),
|
||||
(row['channel_hash'], self.channels),
|
||||
(row['reposted_claim_hash'], self.streams),
|
||||
(row['reposted_claim_hash'], self.channels)):
|
||||
censoring_channel_hash = lookup.get(claim_hash)
|
||||
if censoring_channel_hash:
|
||||
was_censored = True
|
||||
self.censored.setdefault(censoring_channel_hash, 0)
|
||||
self.censored[censoring_channel_hash] += 1
|
||||
break
|
||||
if was_censored:
|
||||
self.total += 1
|
||||
return was_censored
|
||||
|
||||
def to_message(self, outputs: OutputsMessage, extra_txo_rows):
|
||||
outputs.blocked_total = self.total
|
||||
for censoring_channel_hash, count in self.censored.items():
|
||||
blocked = outputs.blocked.add()
|
||||
blocked.count = count
|
||||
set_reference(blocked.channel, censoring_channel_hash, extra_txo_rows)
|
||||
|
||||
|
||||
class Outputs:
|
||||
|
||||
__slots__ = 'txos', 'extra_txos', 'txs', 'offset', 'total', 'blocked', 'blocked_total'
|
||||
|
||||
def __init__(self, txos: List, extra_txos: List, txs: set,
|
||||
offset: int, total: int, blocked: List, blocked_total: int):
|
||||
self.txos = txos
|
||||
self.txs = txs
|
||||
self.extra_txos = extra_txos
|
||||
self.offset = offset
|
||||
self.total = total
|
||||
self.blocked = blocked
|
||||
self.blocked_total = blocked_total
|
||||
|
||||
def inflate(self, txs):
|
||||
tx_map = {tx.hash: tx for tx in txs}
|
||||
for txo_message in self.extra_txos:
|
||||
self.message_to_txo(txo_message, tx_map)
|
||||
txos = [self.message_to_txo(txo_message, tx_map) for txo_message in self.txos]
|
||||
return txos, self.inflate_blocked(tx_map)
|
||||
|
||||
def inflate_blocked(self, tx_map):
|
||||
return {
|
||||
"total": self.blocked_total,
|
||||
"channels": [{
|
||||
'channel': self.message_to_txo(blocked.channel, tx_map),
|
||||
'blocked': blocked.count
|
||||
} for blocked in self.blocked]
|
||||
}
|
||||
|
||||
def message_to_txo(self, txo_message, tx_map):
|
||||
if txo_message.WhichOneof('meta') == 'error':
|
||||
error = {
|
||||
'error': {
|
||||
'name': txo_message.error.Code.Name(txo_message.error.code),
|
||||
'text': txo_message.error.text,
|
||||
}
|
||||
}
|
||||
if error['error']['name'] == BLOCKED:
|
||||
error['error']['censor'] = self.message_to_txo(
|
||||
txo_message.error.blocked.channel, tx_map
|
||||
)
|
||||
return error
|
||||
|
||||
tx = tx_map.get(txo_message.tx_hash)
|
||||
if not tx:
|
||||
return
|
||||
txo = tx.outputs[txo_message.nout]
|
||||
if txo_message.WhichOneof('meta') == 'claim':
|
||||
claim = txo_message.claim
|
||||
txo.meta = {
|
||||
'short_url': f'lbry://{claim.short_url}',
|
||||
'canonical_url': f'lbry://{claim.canonical_url or claim.short_url}',
|
||||
'reposted': claim.reposted,
|
||||
'is_controlling': claim.is_controlling,
|
||||
'take_over_height': claim.take_over_height,
|
||||
'creation_height': claim.creation_height,
|
||||
'activation_height': claim.activation_height,
|
||||
'expiration_height': claim.expiration_height,
|
||||
'effective_amount': claim.effective_amount,
|
||||
'support_amount': claim.support_amount,
|
||||
'trending_group': claim.trending_group,
|
||||
'trending_mixed': claim.trending_mixed,
|
||||
'trending_local': claim.trending_local,
|
||||
'trending_global': claim.trending_global,
|
||||
}
|
||||
if claim.HasField('channel'):
|
||||
txo.channel = tx_map[claim.channel.tx_hash].outputs[claim.channel.nout]
|
||||
if claim.HasField('repost'):
|
||||
txo.reposted_claim = tx_map[claim.repost.tx_hash].outputs[claim.repost.nout]
|
||||
try:
|
||||
if txo.claim.is_channel:
|
||||
txo.meta['claims_in_channel'] = claim.claims_in_channel
|
||||
except:
|
||||
pass
|
||||
return txo
|
||||
|
||||
@classmethod
|
||||
def from_base64(cls, data: str) -> 'Outputs':
|
||||
return cls.from_bytes(base64.b64decode(data))
|
||||
|
||||
@classmethod
|
||||
def from_bytes(cls, data: bytes) -> 'Outputs':
|
||||
outputs = OutputsMessage()
|
||||
outputs.ParseFromString(data)
|
||||
txs = set()
|
||||
for txo_message in chain(outputs.txos, outputs.extra_txos):
|
||||
if txo_message.WhichOneof('meta') == 'error':
|
||||
continue
|
||||
txs.add((hexlify(txo_message.tx_hash[::-1]).decode(), txo_message.height))
|
||||
return cls(
|
||||
outputs.txos, outputs.extra_txos, txs,
|
||||
outputs.offset, outputs.total,
|
||||
outputs.blocked, outputs.blocked_total
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def to_base64(cls, txo_rows, extra_txo_rows, offset=0, total=None, blocked=None) -> str:
|
||||
return base64.b64encode(cls.to_bytes(txo_rows, extra_txo_rows, offset, total, blocked)).decode()
|
||||
|
||||
@classmethod
|
||||
def to_bytes(cls, txo_rows, extra_txo_rows, offset=0, total=None, blocked: Censor = None) -> bytes:
|
||||
page = OutputsMessage()
|
||||
page.offset = offset
|
||||
if total is not None:
|
||||
page.total = total
|
||||
if blocked is not None:
|
||||
blocked.to_message(page, extra_txo_rows)
|
||||
for row in txo_rows:
|
||||
cls.row_to_message(row, page.txos.add(), extra_txo_rows)
|
||||
for row in extra_txo_rows:
|
||||
cls.row_to_message(row, page.extra_txos.add(), extra_txo_rows)
|
||||
return page.SerializeToString()
|
||||
|
||||
@classmethod
|
||||
def row_to_message(cls, txo, txo_message, extra_txo_rows):
|
||||
if isinstance(txo, Exception):
|
||||
txo_message.error.text = txo.args[0]
|
||||
if isinstance(txo, ValueError):
|
||||
txo_message.error.code = ErrorMessage.INVALID
|
||||
elif isinstance(txo, LookupError):
|
||||
txo_message.error.code = ErrorMessage.NOT_FOUND
|
||||
elif isinstance(txo, ResolveCensoredError):
|
||||
txo_message.error.code = ErrorMessage.BLOCKED
|
||||
set_reference(txo_message.error.blocked.channel, txo.censor_hash, extra_txo_rows)
|
||||
return
|
||||
txo_message.tx_hash = txo['txo_hash'][:32]
|
||||
txo_message.nout, = struct.unpack('<I', txo['txo_hash'][32:])
|
||||
txo_message.height = txo['height']
|
||||
txo_message.claim.short_url = txo['short_url']
|
||||
txo_message.claim.reposted = txo['reposted']
|
||||
if txo['canonical_url'] is not None:
|
||||
txo_message.claim.canonical_url = txo['canonical_url']
|
||||
txo_message.claim.is_controlling = bool(txo['is_controlling'])
|
||||
if txo['last_take_over_height'] is not None:
|
||||
txo_message.claim.take_over_height = txo['last_take_over_height']
|
||||
txo_message.claim.creation_height = txo['creation_height']
|
||||
txo_message.claim.activation_height = txo['activation_height']
|
||||
txo_message.claim.expiration_height = txo['expiration_height']
|
||||
if txo['claims_in_channel'] is not None:
|
||||
txo_message.claim.claims_in_channel = txo['claims_in_channel']
|
||||
txo_message.claim.effective_amount = txo['effective_amount']
|
||||
txo_message.claim.support_amount = txo['support_amount']
|
||||
txo_message.claim.trending_group = txo['trending_group']
|
||||
txo_message.claim.trending_mixed = txo['trending_mixed']
|
||||
txo_message.claim.trending_local = txo['trending_local']
|
||||
txo_message.claim.trending_global = txo['trending_global']
|
||||
set_reference(txo_message.claim.channel, txo['channel_hash'], extra_txo_rows)
|
||||
set_reference(txo_message.claim.repost, txo['reposted_claim_hash'], extra_txo_rows)
|
||||
@@ -1,6 +0,0 @@
|
||||
from lbry.schema.base import Signable
|
||||
|
||||
|
||||
class Support(Signable):
|
||||
__slots__ = ()
|
||||
message_class = None # TODO: add support protobufs
|
||||
@@ -1,13 +0,0 @@
|
||||
from typing import List
|
||||
import re
|
||||
|
||||
MULTI_SPACE_RE = re.compile(r"\s{2,}")
|
||||
WEIRD_CHARS_RE = re.compile(r"[#!~]")
|
||||
|
||||
|
||||
def normalize_tag(tag: str):
|
||||
return MULTI_SPACE_RE.sub(' ', WEIRD_CHARS_RE.sub(' ', tag.lower().replace("'", ""))).strip()
|
||||
|
||||
|
||||
def clean_tags(tags: List[str]):
|
||||
return [tag for tag in {normalize_tag(tag) for tag in tags} if tag]
|
||||
@@ -1,146 +0,0 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: certificate.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
from google.protobuf.internal import enum_type_wrapper
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import message as _message
|
||||
from google.protobuf import reflection as _reflection
|
||||
from google.protobuf import symbol_database as _symbol_database
|
||||
# @@protoc_insertion_point(imports)
|
||||
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='certificate.proto',
|
||||
package='legacy_pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n\x11\x63\x65rtificate.proto\x12\tlegacy_pb\"\xa2\x01\n\x0b\x43\x65rtificate\x12/\n\x07version\x18\x01 \x02(\x0e\x32\x1e.legacy_pb.Certificate.Version\x12#\n\x07keyType\x18\x02 \x02(\x0e\x32\x12.legacy_pb.KeyType\x12\x11\n\tpublicKey\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01*Q\n\x07KeyType\x12\x1b\n\x17UNKNOWN_PUBLIC_KEY_TYPE\x10\x00\x12\x0c\n\x08NIST256p\x10\x01\x12\x0c\n\x08NIST384p\x10\x02\x12\r\n\tSECP256k1\x10\x03')
|
||||
)
|
||||
|
||||
_KEYTYPE = _descriptor.EnumDescriptor(
|
||||
name='KeyType',
|
||||
full_name='legacy_pb.KeyType',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='UNKNOWN_PUBLIC_KEY_TYPE', index=0, number=0,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='NIST256p', index=1, number=1,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='NIST384p', index=2, number=2,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='SECP256k1', index=3, number=3,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=197,
|
||||
serialized_end=278,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_KEYTYPE)
|
||||
|
||||
KeyType = enum_type_wrapper.EnumTypeWrapper(_KEYTYPE)
|
||||
UNKNOWN_PUBLIC_KEY_TYPE = 0
|
||||
NIST256p = 1
|
||||
NIST384p = 2
|
||||
SECP256k1 = 3
|
||||
|
||||
|
||||
_CERTIFICATE_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='legacy_pb.Certificate.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='UNKNOWN_VERSION', index=0, number=0,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='_0_0_1', index=1, number=1,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=153,
|
||||
serialized_end=195,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_CERTIFICATE_VERSION)
|
||||
|
||||
|
||||
_CERTIFICATE = _descriptor.Descriptor(
|
||||
name='Certificate',
|
||||
full_name='legacy_pb.Certificate',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='legacy_pb.Certificate.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='keyType', full_name='legacy_pb.Certificate.keyType', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='publicKey', full_name='legacy_pb.Certificate.publicKey', index=2,
|
||||
number=4, type=12, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
_CERTIFICATE_VERSION,
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto2',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=33,
|
||||
serialized_end=195,
|
||||
)
|
||||
|
||||
_CERTIFICATE.fields_by_name['version'].enum_type = _CERTIFICATE_VERSION
|
||||
_CERTIFICATE.fields_by_name['keyType'].enum_type = _KEYTYPE
|
||||
_CERTIFICATE_VERSION.containing_type = _CERTIFICATE
|
||||
DESCRIPTOR.message_types_by_name['Certificate'] = _CERTIFICATE
|
||||
DESCRIPTOR.enum_types_by_name['KeyType'] = _KEYTYPE
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Certificate = _reflection.GeneratedProtocolMessageType('Certificate', (_message.Message,), dict(
|
||||
DESCRIPTOR = _CERTIFICATE,
|
||||
__module__ = 'certificate_pb2'
|
||||
# @@protoc_insertion_point(class_scope:legacy_pb.Certificate)
|
||||
))
|
||||
_sym_db.RegisterMessage(Certificate)
|
||||
|
||||
|
||||
# @@protoc_insertion_point(module_scope)
|
||||
@@ -1,148 +0,0 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: fee.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import message as _message
|
||||
from google.protobuf import reflection as _reflection
|
||||
from google.protobuf import symbol_database as _symbol_database
|
||||
# @@protoc_insertion_point(imports)
|
||||
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='fee.proto',
|
||||
package='legacy_pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n\tfee.proto\x12\tlegacy_pb\"\xe3\x01\n\x03\x46\x65\x65\x12\'\n\x07version\x18\x01 \x02(\x0e\x32\x16.legacy_pb.Fee.Version\x12)\n\x08\x63urrency\x18\x02 \x02(\x0e\x32\x17.legacy_pb.Fee.Currency\x12\x0f\n\x07\x61\x64\x64ress\x18\x03 \x02(\x0c\x12\x0e\n\x06\x61mount\x18\x04 \x02(\x02\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\";\n\x08\x43urrency\x12\x14\n\x10UNKNOWN_CURRENCY\x10\x00\x12\x07\n\x03LBC\x10\x01\x12\x07\n\x03\x42TC\x10\x02\x12\x07\n\x03USD\x10\x03')
|
||||
)
|
||||
|
||||
|
||||
|
||||
_FEE_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='legacy_pb.Fee.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='UNKNOWN_VERSION', index=0, number=0,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='_0_0_1', index=1, number=1,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=149,
|
||||
serialized_end=191,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_FEE_VERSION)
|
||||
|
||||
_FEE_CURRENCY = _descriptor.EnumDescriptor(
|
||||
name='Currency',
|
||||
full_name='legacy_pb.Fee.Currency',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='UNKNOWN_CURRENCY', index=0, number=0,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='LBC', index=1, number=1,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='BTC', index=2, number=2,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='USD', index=3, number=3,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=193,
|
||||
serialized_end=252,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_FEE_CURRENCY)
|
||||
|
||||
|
||||
_FEE = _descriptor.Descriptor(
|
||||
name='Fee',
|
||||
full_name='legacy_pb.Fee',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='legacy_pb.Fee.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='currency', full_name='legacy_pb.Fee.currency', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='address', full_name='legacy_pb.Fee.address', index=2,
|
||||
number=3, type=12, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='amount', full_name='legacy_pb.Fee.amount', index=3,
|
||||
number=4, type=2, cpp_type=6, label=2,
|
||||
has_default_value=False, default_value=float(0),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
_FEE_VERSION,
|
||||
_FEE_CURRENCY,
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto2',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=25,
|
||||
serialized_end=252,
|
||||
)
|
||||
|
||||
_FEE.fields_by_name['version'].enum_type = _FEE_VERSION
|
||||
_FEE.fields_by_name['currency'].enum_type = _FEE_CURRENCY
|
||||
_FEE_VERSION.containing_type = _FEE
|
||||
_FEE_CURRENCY.containing_type = _FEE
|
||||
DESCRIPTOR.message_types_by_name['Fee'] = _FEE
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Fee = _reflection.GeneratedProtocolMessageType('Fee', (_message.Message,), dict(
|
||||
DESCRIPTOR = _FEE,
|
||||
__module__ = 'fee_pb2'
|
||||
# @@protoc_insertion_point(class_scope:legacy_pb.Fee)
|
||||
))
|
||||
_sym_db.RegisterMessage(Fee)
|
||||
|
||||
|
||||
# @@protoc_insertion_point(module_scope)
|
||||
@@ -1,158 +0,0 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: legacy_claim.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import message as _message
|
||||
from google.protobuf import reflection as _reflection
|
||||
from google.protobuf import symbol_database as _symbol_database
|
||||
# @@protoc_insertion_point(imports)
|
||||
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
from . import stream_pb2 as stream__pb2
|
||||
from . import certificate_pb2 as certificate__pb2
|
||||
from . import signature_pb2 as signature__pb2
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='legacy_claim.proto',
|
||||
package='legacy_pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n\x12legacy_claim.proto\x12\tlegacy_pb\x1a\x0cstream.proto\x1a\x11\x63\x65rtificate.proto\x1a\x0fsignature.proto\"\xd9\x02\n\x05\x43laim\x12)\n\x07version\x18\x01 \x02(\x0e\x32\x18.legacy_pb.Claim.Version\x12-\n\tclaimType\x18\x02 \x02(\x0e\x32\x1a.legacy_pb.Claim.ClaimType\x12!\n\x06stream\x18\x03 \x01(\x0b\x32\x11.legacy_pb.Stream\x12+\n\x0b\x63\x65rtificate\x18\x04 \x01(\x0b\x32\x16.legacy_pb.Certificate\x12\x30\n\x12publisherSignature\x18\x05 \x01(\x0b\x32\x14.legacy_pb.Signature\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"H\n\tClaimType\x12\x16\n\x12UNKNOWN_CLAIM_TYPE\x10\x00\x12\x0e\n\nstreamType\x10\x01\x12\x13\n\x0f\x63\x65rtificateType\x10\x02')
|
||||
,
|
||||
dependencies=[stream__pb2.DESCRIPTOR,certificate__pb2.DESCRIPTOR,signature__pb2.DESCRIPTOR,])
|
||||
|
||||
|
||||
|
||||
_CLAIM_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='legacy_pb.Claim.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='UNKNOWN_VERSION', index=0, number=0,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='_0_0_1', index=1, number=1,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=313,
|
||||
serialized_end=355,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_CLAIM_VERSION)
|
||||
|
||||
_CLAIM_CLAIMTYPE = _descriptor.EnumDescriptor(
|
||||
name='ClaimType',
|
||||
full_name='legacy_pb.Claim.ClaimType',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='UNKNOWN_CLAIM_TYPE', index=0, number=0,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='streamType', index=1, number=1,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='certificateType', index=2, number=2,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=357,
|
||||
serialized_end=429,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_CLAIM_CLAIMTYPE)
|
||||
|
||||
|
||||
_CLAIM = _descriptor.Descriptor(
|
||||
name='Claim',
|
||||
full_name='legacy_pb.Claim',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='legacy_pb.Claim.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='claimType', full_name='legacy_pb.Claim.claimType', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='stream', full_name='legacy_pb.Claim.stream', index=2,
|
||||
number=3, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='certificate', full_name='legacy_pb.Claim.certificate', index=3,
|
||||
number=4, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='publisherSignature', full_name='legacy_pb.Claim.publisherSignature', index=4,
|
||||
number=5, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
_CLAIM_VERSION,
|
||||
_CLAIM_CLAIMTYPE,
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto2',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=84,
|
||||
serialized_end=429,
|
||||
)
|
||||
|
||||
_CLAIM.fields_by_name['version'].enum_type = _CLAIM_VERSION
|
||||
_CLAIM.fields_by_name['claimType'].enum_type = _CLAIM_CLAIMTYPE
|
||||
_CLAIM.fields_by_name['stream'].message_type = stream__pb2._STREAM
|
||||
_CLAIM.fields_by_name['certificate'].message_type = certificate__pb2._CERTIFICATE
|
||||
_CLAIM.fields_by_name['publisherSignature'].message_type = signature__pb2._SIGNATURE
|
||||
_CLAIM_VERSION.containing_type = _CLAIM
|
||||
_CLAIM_CLAIMTYPE.containing_type = _CLAIM
|
||||
DESCRIPTOR.message_types_by_name['Claim'] = _CLAIM
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Claim = _reflection.GeneratedProtocolMessageType('Claim', (_message.Message,), dict(
|
||||
DESCRIPTOR = _CLAIM,
|
||||
__module__ = 'legacy_claim_pb2'
|
||||
# @@protoc_insertion_point(class_scope:legacy_pb.Claim)
|
||||
))
|
||||
_sym_db.RegisterMessage(Claim)
|
||||
|
||||
|
||||
# @@protoc_insertion_point(module_scope)
|
||||
@@ -1,936 +0,0 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: metadata.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import message as _message
|
||||
from google.protobuf import reflection as _reflection
|
||||
from google.protobuf import symbol_database as _symbol_database
|
||||
# @@protoc_insertion_point(imports)
|
||||
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
from . import fee_pb2 as fee__pb2
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='metadata.proto',
|
||||
package='legacy_pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n\x0emetadata.proto\x12\tlegacy_pb\x1a\tfee.proto\"\xfc\x0e\n\x08Metadata\x12,\n\x07version\x18\x01 \x02(\x0e\x32\x1b.legacy_pb.Metadata.Version\x12.\n\x08language\x18\x02 \x02(\x0e\x32\x1c.legacy_pb.Metadata.Language\x12\r\n\x05title\x18\x03 \x02(\t\x12\x13\n\x0b\x64\x65scription\x18\x04 \x02(\t\x12\x0e\n\x06\x61uthor\x18\x05 \x02(\t\x12\x0f\n\x07license\x18\x06 \x02(\t\x12\x0c\n\x04nsfw\x18\x07 \x02(\x08\x12\x1b\n\x03\x66\x65\x65\x18\x08 \x01(\x0b\x32\x0e.legacy_pb.Fee\x12\x11\n\tthumbnail\x18\t \x01(\t\x12\x0f\n\x07preview\x18\n \x01(\t\x12\x12\n\nlicenseUrl\x18\x0b \x01(\t\"N\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\x12\n\n\x06_0_0_2\x10\x02\x12\n\n\x06_0_0_3\x10\x03\x12\n\n\x06_0_1_0\x10\x04\"\x99\x0c\n\x08Language\x12\x14\n\x10UNKNOWN_LANGUAGE\x10\x00\x12\x06\n\x02\x65n\x10\x01\x12\x06\n\x02\x61\x61\x10\x02\x12\x06\n\x02\x61\x62\x10\x03\x12\x06\n\x02\x61\x65\x10\x04\x12\x06\n\x02\x61\x66\x10\x05\x12\x06\n\x02\x61k\x10\x06\x12\x06\n\x02\x61m\x10\x07\x12\x06\n\x02\x61n\x10\x08\x12\x06\n\x02\x61r\x10\t\x12\x06\n\x02\x61s\x10\n\x12\x06\n\x02\x61v\x10\x0b\x12\x06\n\x02\x61y\x10\x0c\x12\x06\n\x02\x61z\x10\r\x12\x06\n\x02\x62\x61\x10\x0e\x12\x06\n\x02\x62\x65\x10\x0f\x12\x06\n\x02\x62g\x10\x10\x12\x06\n\x02\x62h\x10\x11\x12\x06\n\x02\x62i\x10\x12\x12\x06\n\x02\x62m\x10\x13\x12\x06\n\x02\x62n\x10\x14\x12\x06\n\x02\x62o\x10\x15\x12\x06\n\x02\x62r\x10\x16\x12\x06\n\x02\x62s\x10\x17\x12\x06\n\x02\x63\x61\x10\x18\x12\x06\n\x02\x63\x65\x10\x19\x12\x06\n\x02\x63h\x10\x1a\x12\x06\n\x02\x63o\x10\x1b\x12\x06\n\x02\x63r\x10\x1c\x12\x06\n\x02\x63s\x10\x1d\x12\x06\n\x02\x63u\x10\x1e\x12\x06\n\x02\x63v\x10\x1f\x12\x06\n\x02\x63y\x10 \x12\x06\n\x02\x64\x61\x10!\x12\x06\n\x02\x64\x65\x10\"\x12\x06\n\x02\x64v\x10#\x12\x06\n\x02\x64z\x10$\x12\x06\n\x02\x65\x65\x10%\x12\x06\n\x02\x65l\x10&\x12\x06\n\x02\x65o\x10\'\x12\x06\n\x02\x65s\x10(\x12\x06\n\x02\x65t\x10)\x12\x06\n\x02\x65u\x10*\x12\x06\n\x02\x66\x61\x10+\x12\x06\n\x02\x66\x66\x10,\x12\x06\n\x02\x66i\x10-\x12\x06\n\x02\x66j\x10.\x12\x06\n\x02\x66o\x10/\x12\x06\n\x02\x66r\x10\x30\x12\x06\n\x02\x66y\x10\x31\x12\x06\n\x02ga\x10\x32\x12\x06\n\x02gd\x10\x33\x12\x06\n\x02gl\x10\x34\x12\x06\n\x02gn\x10\x35\x12\x06\n\x02gu\x10\x36\x12\x06\n\x02gv\x10\x37\x12\x06\n\x02ha\x10\x38\x12\x06\n\x02he\x10\x39\x12\x06\n\x02hi\x10:\x12\x06\n\x02ho\x10;\x12\x06\n\x02hr\x10<\x12\x06\n\x02ht\x10=\x12\x06\n\x02hu\x10>\x12\x06\n\x02hy\x10?\x12\x06\n\x02hz\x10@\x12\x06\n\x02ia\x10\x41\x12\x06\n\x02id\x10\x42\x12\x06\n\x02ie\x10\x43\x12\x06\n\x02ig\x10\x44\x12\x06\n\x02ii\x10\x45\x12\x06\n\x02ik\x10\x46\x12\x06\n\x02io\x10G\x12\x06\n\x02is\x10H\x12\x06\n\x02it\x10I\x12\x06\n\x02iu\x10J\x12\x06\n\x02ja\x10K\x12\x06\n\x02jv\x10L\x12\x06\n\x02ka\x10M\x12\x06\n\x02kg\x10N\x12\x06\n\x02ki\x10O\x12\x06\n\x02kj\x10P\x12\x06\n\x02kk\x10Q\x12\x06\n\x02kl\x10R\x12\x06\n\x02km\x10S\x12\x06\n\x02kn\x10T\x12\x06\n\x02ko\x10U\x12\x06\n\x02kr\x10V\x12\x06\n\x02ks\x10W\x12\x06\n\x02ku\x10X\x12\x06\n\x02kv\x10Y\x12\x06\n\x02kw\x10Z\x12\x06\n\x02ky\x10[\x12\x06\n\x02la\x10\\\x12\x06\n\x02lb\x10]\x12\x06\n\x02lg\x10^\x12\x06\n\x02li\x10_\x12\x06\n\x02ln\x10`\x12\x06\n\x02lo\x10\x61\x12\x06\n\x02lt\x10\x62\x12\x06\n\x02lu\x10\x63\x12\x06\n\x02lv\x10\x64\x12\x06\n\x02mg\x10\x65\x12\x06\n\x02mh\x10\x66\x12\x06\n\x02mi\x10g\x12\x06\n\x02mk\x10h\x12\x06\n\x02ml\x10i\x12\x06\n\x02mn\x10j\x12\x06\n\x02mr\x10k\x12\x06\n\x02ms\x10l\x12\x06\n\x02mt\x10m\x12\x06\n\x02my\x10n\x12\x06\n\x02na\x10o\x12\x06\n\x02nb\x10p\x12\x06\n\x02nd\x10q\x12\x06\n\x02ne\x10r\x12\x06\n\x02ng\x10s\x12\x06\n\x02nl\x10t\x12\x06\n\x02nn\x10u\x12\x06\n\x02no\x10v\x12\x06\n\x02nr\x10w\x12\x06\n\x02nv\x10x\x12\x06\n\x02ny\x10y\x12\x06\n\x02oc\x10z\x12\x06\n\x02oj\x10{\x12\x06\n\x02om\x10|\x12\x06\n\x02or\x10}\x12\x06\n\x02os\x10~\x12\x06\n\x02pa\x10\x7f\x12\x07\n\x02pi\x10\x80\x01\x12\x07\n\x02pl\x10\x81\x01\x12\x07\n\x02ps\x10\x82\x01\x12\x07\n\x02pt\x10\x83\x01\x12\x07\n\x02qu\x10\x84\x01\x12\x07\n\x02rm\x10\x85\x01\x12\x07\n\x02rn\x10\x86\x01\x12\x07\n\x02ro\x10\x87\x01\x12\x07\n\x02ru\x10\x88\x01\x12\x07\n\x02rw\x10\x89\x01\x12\x07\n\x02sa\x10\x8a\x01\x12\x07\n\x02sc\x10\x8b\x01\x12\x07\n\x02sd\x10\x8c\x01\x12\x07\n\x02se\x10\x8d\x01\x12\x07\n\x02sg\x10\x8e\x01\x12\x07\n\x02si\x10\x8f\x01\x12\x07\n\x02sk\x10\x90\x01\x12\x07\n\x02sl\x10\x91\x01\x12\x07\n\x02sm\x10\x92\x01\x12\x07\n\x02sn\x10\x93\x01\x12\x07\n\x02so\x10\x94\x01\x12\x07\n\x02sq\x10\x95\x01\x12\x07\n\x02sr\x10\x96\x01\x12\x07\n\x02ss\x10\x97\x01\x12\x07\n\x02st\x10\x98\x01\x12\x07\n\x02su\x10\x99\x01\x12\x07\n\x02sv\x10\x9a\x01\x12\x07\n\x02sw\x10\x9b\x01\x12\x07\n\x02ta\x10\x9c\x01\x12\x07\n\x02te\x10\x9d\x01\x12\x07\n\x02tg\x10\x9e\x01\x12\x07\n\x02th\x10\x9f\x01\x12\x07\n\x02ti\x10\xa0\x01\x12\x07\n\x02tk\x10\xa1\x01\x12\x07\n\x02tl\x10\xa2\x01\x12\x07\n\x02tn\x10\xa3\x01\x12\x07\n\x02to\x10\xa4\x01\x12\x07\n\x02tr\x10\xa5\x01\x12\x07\n\x02ts\x10\xa6\x01\x12\x07\n\x02tt\x10\xa7\x01\x12\x07\n\x02tw\x10\xa8\x01\x12\x07\n\x02tyLine truncated
|
||||
,
|
||||
dependencies=[fee__pb2.DESCRIPTOR,])
|
||||
|
||||
|
||||
|
||||
_METADATA_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='legacy_pb.Metadata.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='UNKNOWN_VERSION', index=0, number=0,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='_0_0_1', index=1, number=1,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='_0_0_2', index=2, number=2,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='_0_0_3', index=3, number=3,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='_0_1_0', index=4, number=4,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=315,
|
||||
serialized_end=393,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_METADATA_VERSION)
|
||||
|
||||
_METADATA_LANGUAGE = _descriptor.EnumDescriptor(
|
||||
name='Language',
|
||||
full_name='legacy_pb.Metadata.Language',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='UNKNOWN_LANGUAGE', index=0, number=0,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='en', index=1, number=1,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='aa', index=2, number=2,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ab', index=3, number=3,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ae', index=4, number=4,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='af', index=5, number=5,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ak', index=6, number=6,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='am', index=7, number=7,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='an', index=8, number=8,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ar', index=9, number=9,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='as', index=10, number=10,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='av', index=11, number=11,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ay', index=12, number=12,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='az', index=13, number=13,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ba', index=14, number=14,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='be', index=15, number=15,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='bg', index=16, number=16,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='bh', index=17, number=17,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='bi', index=18, number=18,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='bm', index=19, number=19,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='bn', index=20, number=20,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='bo', index=21, number=21,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='br', index=22, number=22,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='bs', index=23, number=23,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ca', index=24, number=24,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ce', index=25, number=25,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ch', index=26, number=26,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='co', index=27, number=27,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='cr', index=28, number=28,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='cs', index=29, number=29,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='cu', index=30, number=30,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='cv', index=31, number=31,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='cy', index=32, number=32,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='da', index=33, number=33,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='de', index=34, number=34,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='dv', index=35, number=35,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='dz', index=36, number=36,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ee', index=37, number=37,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='el', index=38, number=38,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='eo', index=39, number=39,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='es', index=40, number=40,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='et', index=41, number=41,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='eu', index=42, number=42,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='fa', index=43, number=43,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ff', index=44, number=44,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='fi', index=45, number=45,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='fj', index=46, number=46,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='fo', index=47, number=47,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='fr', index=48, number=48,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='fy', index=49, number=49,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ga', index=50, number=50,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='gd', index=51, number=51,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='gl', index=52, number=52,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='gn', index=53, number=53,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='gu', index=54, number=54,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='gv', index=55, number=55,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ha', index=56, number=56,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='he', index=57, number=57,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='hi', index=58, number=58,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ho', index=59, number=59,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='hr', index=60, number=60,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ht', index=61, number=61,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='hu', index=62, number=62,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='hy', index=63, number=63,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='hz', index=64, number=64,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ia', index=65, number=65,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='id', index=66, number=66,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ie', index=67, number=67,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ig', index=68, number=68,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ii', index=69, number=69,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ik', index=70, number=70,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='io', index=71, number=71,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='is', index=72, number=72,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='it', index=73, number=73,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='iu', index=74, number=74,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ja', index=75, number=75,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='jv', index=76, number=76,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ka', index=77, number=77,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='kg', index=78, number=78,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ki', index=79, number=79,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='kj', index=80, number=80,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='kk', index=81, number=81,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='kl', index=82, number=82,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='km', index=83, number=83,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='kn', index=84, number=84,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ko', index=85, number=85,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='kr', index=86, number=86,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ks', index=87, number=87,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ku', index=88, number=88,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='kv', index=89, number=89,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='kw', index=90, number=90,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ky', index=91, number=91,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='la', index=92, number=92,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='lb', index=93, number=93,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='lg', index=94, number=94,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='li', index=95, number=95,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ln', index=96, number=96,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='lo', index=97, number=97,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='lt', index=98, number=98,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='lu', index=99, number=99,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='lv', index=100, number=100,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='mg', index=101, number=101,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='mh', index=102, number=102,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='mi', index=103, number=103,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='mk', index=104, number=104,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ml', index=105, number=105,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='mn', index=106, number=106,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='mr', index=107, number=107,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ms', index=108, number=108,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='mt', index=109, number=109,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='my', index=110, number=110,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='na', index=111, number=111,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='nb', index=112, number=112,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='nd', index=113, number=113,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ne', index=114, number=114,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ng', index=115, number=115,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='nl', index=116, number=116,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='nn', index=117, number=117,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='no', index=118, number=118,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='nr', index=119, number=119,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='nv', index=120, number=120,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ny', index=121, number=121,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='oc', index=122, number=122,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='oj', index=123, number=123,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='om', index=124, number=124,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='or', index=125, number=125,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='os', index=126, number=126,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='pa', index=127, number=127,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='pi', index=128, number=128,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='pl', index=129, number=129,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ps', index=130, number=130,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='pt', index=131, number=131,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='qu', index=132, number=132,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='rm', index=133, number=133,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='rn', index=134, number=134,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ro', index=135, number=135,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ru', index=136, number=136,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='rw', index=137, number=137,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='sa', index=138, number=138,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='sc', index=139, number=139,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='sd', index=140, number=140,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='se', index=141, number=141,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='sg', index=142, number=142,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='si', index=143, number=143,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='sk', index=144, number=144,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='sl', index=145, number=145,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='sm', index=146, number=146,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='sn', index=147, number=147,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='so', index=148, number=148,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='sq', index=149, number=149,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='sr', index=150, number=150,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ss', index=151, number=151,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='st', index=152, number=152,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='su', index=153, number=153,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='sv', index=154, number=154,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='sw', index=155, number=155,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ta', index=156, number=156,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='te', index=157, number=157,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='tg', index=158, number=158,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='th', index=159, number=159,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ti', index=160, number=160,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='tk', index=161, number=161,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='tl', index=162, number=162,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='tn', index=163, number=163,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='to', index=164, number=164,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='tr', index=165, number=165,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ts', index=166, number=166,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='tt', index=167, number=167,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='tw', index=168, number=168,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ty', index=169, number=169,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ug', index=170, number=170,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='uk', index=171, number=171,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ur', index=172, number=172,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='uz', index=173, number=173,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ve', index=174, number=174,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='vi', index=175, number=175,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='vo', index=176, number=176,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='wa', index=177, number=177,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='wo', index=178, number=178,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='xh', index=179, number=179,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='yi', index=180, number=180,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='yo', index=181, number=181,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='za', index=182, number=182,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='zh', index=183, number=183,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='zu', index=184, number=184,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=396,
|
||||
serialized_end=1957,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_METADATA_LANGUAGE)
|
||||
|
||||
|
||||
_METADATA = _descriptor.Descriptor(
|
||||
name='Metadata',
|
||||
full_name='legacy_pb.Metadata',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='legacy_pb.Metadata.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='language', full_name='legacy_pb.Metadata.language', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='title', full_name='legacy_pb.Metadata.title', index=2,
|
||||
number=3, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='description', full_name='legacy_pb.Metadata.description', index=3,
|
||||
number=4, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='author', full_name='legacy_pb.Metadata.author', index=4,
|
||||
number=5, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='license', full_name='legacy_pb.Metadata.license', index=5,
|
||||
number=6, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='nsfw', full_name='legacy_pb.Metadata.nsfw', index=6,
|
||||
number=7, type=8, cpp_type=7, label=2,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='fee', full_name='legacy_pb.Metadata.fee', index=7,
|
||||
number=8, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='thumbnail', full_name='legacy_pb.Metadata.thumbnail', index=8,
|
||||
number=9, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='preview', full_name='legacy_pb.Metadata.preview', index=9,
|
||||
number=10, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='licenseUrl', full_name='legacy_pb.Metadata.licenseUrl', index=10,
|
||||
number=11, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
_METADATA_VERSION,
|
||||
_METADATA_LANGUAGE,
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto2',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=41,
|
||||
serialized_end=1957,
|
||||
)
|
||||
|
||||
_METADATA.fields_by_name['version'].enum_type = _METADATA_VERSION
|
||||
_METADATA.fields_by_name['language'].enum_type = _METADATA_LANGUAGE
|
||||
_METADATA.fields_by_name['fee'].message_type = fee__pb2._FEE
|
||||
_METADATA_VERSION.containing_type = _METADATA
|
||||
_METADATA_LANGUAGE.containing_type = _METADATA
|
||||
DESCRIPTOR.message_types_by_name['Metadata'] = _METADATA
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Metadata = _reflection.GeneratedProtocolMessageType('Metadata', (_message.Message,), dict(
|
||||
DESCRIPTOR = _METADATA,
|
||||
__module__ = 'metadata_pb2'
|
||||
# @@protoc_insertion_point(class_scope:legacy_pb.Metadata)
|
||||
))
|
||||
_sym_db.RegisterMessage(Metadata)
|
||||
|
||||
|
||||
# @@protoc_insertion_point(module_scope)
|
||||
@@ -1,118 +0,0 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: signature.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import message as _message
|
||||
from google.protobuf import reflection as _reflection
|
||||
from google.protobuf import symbol_database as _symbol_database
|
||||
# @@protoc_insertion_point(imports)
|
||||
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
from . import certificate_pb2 as certificate__pb2
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='signature.proto',
|
||||
package='legacy_pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n\x0fsignature.proto\x12\tlegacy_pb\x1a\x11\x63\x65rtificate.proto\"\xbb\x01\n\tSignature\x12-\n\x07version\x18\x01 \x02(\x0e\x32\x1c.legacy_pb.Signature.Version\x12)\n\rsignatureType\x18\x02 \x02(\x0e\x32\x12.legacy_pb.KeyType\x12\x11\n\tsignature\x18\x03 \x02(\x0c\x12\x15\n\rcertificateId\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
|
||||
,
|
||||
dependencies=[certificate__pb2.DESCRIPTOR,])
|
||||
|
||||
|
||||
|
||||
_SIGNATURE_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='legacy_pb.Signature.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='UNKNOWN_VERSION', index=0, number=0,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='_0_0_1', index=1, number=1,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=195,
|
||||
serialized_end=237,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_SIGNATURE_VERSION)
|
||||
|
||||
|
||||
_SIGNATURE = _descriptor.Descriptor(
|
||||
name='Signature',
|
||||
full_name='legacy_pb.Signature',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='legacy_pb.Signature.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='signatureType', full_name='legacy_pb.Signature.signatureType', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='signature', full_name='legacy_pb.Signature.signature', index=2,
|
||||
number=3, type=12, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='certificateId', full_name='legacy_pb.Signature.certificateId', index=3,
|
||||
number=4, type=12, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
_SIGNATURE_VERSION,
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto2',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=50,
|
||||
serialized_end=237,
|
||||
)
|
||||
|
||||
_SIGNATURE.fields_by_name['version'].enum_type = _SIGNATURE_VERSION
|
||||
_SIGNATURE.fields_by_name['signatureType'].enum_type = certificate__pb2._KEYTYPE
|
||||
_SIGNATURE_VERSION.containing_type = _SIGNATURE
|
||||
DESCRIPTOR.message_types_by_name['Signature'] = _SIGNATURE
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Signature = _reflection.GeneratedProtocolMessageType('Signature', (_message.Message,), dict(
|
||||
DESCRIPTOR = _SIGNATURE,
|
||||
__module__ = 'signature_pb2'
|
||||
# @@protoc_insertion_point(class_scope:legacy_pb.Signature)
|
||||
))
|
||||
_sym_db.RegisterMessage(Signature)
|
||||
|
||||
|
||||
# @@protoc_insertion_point(module_scope)
|
||||
@@ -1,140 +0,0 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: source.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import message as _message
|
||||
from google.protobuf import reflection as _reflection
|
||||
from google.protobuf import symbol_database as _symbol_database
|
||||
# @@protoc_insertion_point(imports)
|
||||
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='source.proto',
|
||||
package='legacy_pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n\x0csource.proto\x12\tlegacy_pb\"\xf2\x01\n\x06Source\x12*\n\x07version\x18\x01 \x02(\x0e\x32\x19.legacy_pb.Source.Version\x12\x31\n\nsourceType\x18\x02 \x02(\x0e\x32\x1d.legacy_pb.Source.SourceTypes\x12\x0e\n\x06source\x18\x03 \x02(\x0c\x12\x13\n\x0b\x63ontentType\x18\x04 \x02(\t\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"8\n\x0bSourceTypes\x12\x17\n\x13UNKNOWN_SOURCE_TYPE\x10\x00\x12\x10\n\x0clbry_sd_hash\x10\x01')
|
||||
)
|
||||
|
||||
|
||||
|
||||
_SOURCE_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='legacy_pb.Source.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='UNKNOWN_VERSION', index=0, number=0,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='_0_0_1', index=1, number=1,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=170,
|
||||
serialized_end=212,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_SOURCE_VERSION)
|
||||
|
||||
_SOURCE_SOURCETYPES = _descriptor.EnumDescriptor(
|
||||
name='SourceTypes',
|
||||
full_name='legacy_pb.Source.SourceTypes',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='UNKNOWN_SOURCE_TYPE', index=0, number=0,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='lbry_sd_hash', index=1, number=1,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=214,
|
||||
serialized_end=270,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_SOURCE_SOURCETYPES)
|
||||
|
||||
|
||||
_SOURCE = _descriptor.Descriptor(
|
||||
name='Source',
|
||||
full_name='legacy_pb.Source',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='legacy_pb.Source.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='sourceType', full_name='legacy_pb.Source.sourceType', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='source', full_name='legacy_pb.Source.source', index=2,
|
||||
number=3, type=12, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='contentType', full_name='legacy_pb.Source.contentType', index=3,
|
||||
number=4, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
_SOURCE_VERSION,
|
||||
_SOURCE_SOURCETYPES,
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto2',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=28,
|
||||
serialized_end=270,
|
||||
)
|
||||
|
||||
_SOURCE.fields_by_name['version'].enum_type = _SOURCE_VERSION
|
||||
_SOURCE.fields_by_name['sourceType'].enum_type = _SOURCE_SOURCETYPES
|
||||
_SOURCE_VERSION.containing_type = _SOURCE
|
||||
_SOURCE_SOURCETYPES.containing_type = _SOURCE
|
||||
DESCRIPTOR.message_types_by_name['Source'] = _SOURCE
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Source = _reflection.GeneratedProtocolMessageType('Source', (_message.Message,), dict(
|
||||
DESCRIPTOR = _SOURCE,
|
||||
__module__ = 'source_pb2'
|
||||
# @@protoc_insertion_point(class_scope:legacy_pb.Source)
|
||||
))
|
||||
_sym_db.RegisterMessage(Source)
|
||||
|
||||
|
||||
# @@protoc_insertion_point(module_scope)
|
||||
@@ -1,113 +0,0 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: stream.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import message as _message
|
||||
from google.protobuf import reflection as _reflection
|
||||
from google.protobuf import symbol_database as _symbol_database
|
||||
# @@protoc_insertion_point(imports)
|
||||
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
from . import metadata_pb2 as metadata__pb2
|
||||
from . import source_pb2 as source__pb2
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='stream.proto',
|
||||
package='legacy_pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n\x0cstream.proto\x12\tlegacy_pb\x1a\x0emetadata.proto\x1a\x0csource.proto\"\xaa\x01\n\x06Stream\x12*\n\x07version\x18\x01 \x02(\x0e\x32\x19.legacy_pb.Stream.Version\x12%\n\x08metadata\x18\x02 \x02(\x0b\x32\x13.legacy_pb.Metadata\x12!\n\x06source\x18\x03 \x02(\x0b\x32\x11.legacy_pb.Source\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
|
||||
,
|
||||
dependencies=[metadata__pb2.DESCRIPTOR,source__pb2.DESCRIPTOR,])
|
||||
|
||||
|
||||
|
||||
_STREAM_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='legacy_pb.Stream.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='UNKNOWN_VERSION', index=0, number=0,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='_0_0_1', index=1, number=1,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=186,
|
||||
serialized_end=228,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_STREAM_VERSION)
|
||||
|
||||
|
||||
_STREAM = _descriptor.Descriptor(
|
||||
name='Stream',
|
||||
full_name='legacy_pb.Stream',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='legacy_pb.Stream.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='metadata', full_name='legacy_pb.Stream.metadata', index=1,
|
||||
number=2, type=11, cpp_type=10, label=2,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='source', full_name='legacy_pb.Stream.source', index=2,
|
||||
number=3, type=11, cpp_type=10, label=2,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
_STREAM_VERSION,
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto2',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=58,
|
||||
serialized_end=228,
|
||||
)
|
||||
|
||||
_STREAM.fields_by_name['version'].enum_type = _STREAM_VERSION
|
||||
_STREAM.fields_by_name['metadata'].message_type = metadata__pb2._METADATA
|
||||
_STREAM.fields_by_name['source'].message_type = source__pb2._SOURCE
|
||||
_STREAM_VERSION.containing_type = _STREAM
|
||||
DESCRIPTOR.message_types_by_name['Stream'] = _STREAM
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Stream = _reflection.GeneratedProtocolMessageType('Stream', (_message.Message,), dict(
|
||||
DESCRIPTOR = _STREAM,
|
||||
__module__ = 'stream_pb2'
|
||||
# @@protoc_insertion_point(class_scope:legacy_pb.Stream)
|
||||
))
|
||||
_sym_db.RegisterMessage(Stream)
|
||||
|
||||
|
||||
# @@protoc_insertion_point(module_scope)
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,69 +0,0 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: purchase.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import message as _message
|
||||
from google.protobuf import reflection as _reflection
|
||||
from google.protobuf import symbol_database as _symbol_database
|
||||
from google.protobuf import descriptor_pb2
|
||||
# @@protoc_insertion_point(imports)
|
||||
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='purchase.proto',
|
||||
package='pb',
|
||||
syntax='proto3',
|
||||
serialized_pb=_b('\n\x0epurchase.proto\x12\x02pb\"\x1e\n\x08Purchase\x12\x12\n\nclaim_hash\x18\x01 \x01(\x0c\x62\x06proto3')
|
||||
)
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
|
||||
|
||||
|
||||
_PURCHASE = _descriptor.Descriptor(
|
||||
name='Purchase',
|
||||
full_name='pb.Purchase',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='claim_hash', full_name='pb.Purchase.claim_hash', index=0,
|
||||
number=1, type=12, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=22,
|
||||
serialized_end=52,
|
||||
)
|
||||
|
||||
DESCRIPTOR.message_types_by_name['Purchase'] = _PURCHASE
|
||||
|
||||
Purchase = _reflection.GeneratedProtocolMessageType('Purchase', (_message.Message,), dict(
|
||||
DESCRIPTOR = _PURCHASE,
|
||||
__module__ = 'purchase_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.Purchase)
|
||||
))
|
||||
_sym_db.RegisterMessage(Purchase)
|
||||
|
||||
|
||||
# @@protoc_insertion_point(module_scope)
|
||||
@@ -1,472 +0,0 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: result.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import message as _message
|
||||
from google.protobuf import reflection as _reflection
|
||||
from google.protobuf import symbol_database as _symbol_database
|
||||
from google.protobuf import descriptor_pb2
|
||||
# @@protoc_insertion_point(imports)
|
||||
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='result.proto',
|
||||
package='pb',
|
||||
syntax='proto3',
|
||||
serialized_pb=_b('\n\x0cresult.proto\x12\x02pb\"\x97\x01\n\x07Outputs\x12\x18\n\x04txos\x18\x01 \x03(\x0b\x32\n.pb.Output\x12\x1e\n\nextra_txos\x18\x02 \x03(\x0b\x32\n.pb.Output\x12\r\n\x05total\x18\x03 \x01(\r\x12\x0e\n\x06offset\x18\x04 \x01(\r\x12\x1c\n\x07\x62locked\x18\x05 \x03(\x0b\x32\x0b.pb.Blocked\x12\x15\n\rblocked_total\x18\x06 \x01(\r\"{\n\x06Output\x12\x0f\n\x07tx_hash\x18\x01 \x01(\x0c\x12\x0c\n\x04nout\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\x1e\n\x05\x63laim\x18\x07 \x01(\x0b\x32\r.pb.ClaimMetaH\x00\x12\x1a\n\x05\x65rror\x18\x0f \x01(\x0b\x32\t.pb.ErrorH\x00\x42\x06\n\x04meta\"\xaf\x03\n\tClaimMeta\x12\x1b\n\x07\x63hannel\x18\x01 \x01(\x0b\x32\n.pb.Output\x12\x1a\n\x06repost\x18\x02 \x01(\x0b\x32\n.pb.Output\x12\x11\n\tshort_url\x18\x03 \x01(\t\x12\x15\n\rcanonical_url\x18\x04 \x01(\t\x12\x16\n\x0eis_controlling\x18\x05 \x01(\x08\x12\x18\n\x10take_over_height\x18\x06 \x01(\r\x12\x17\n\x0f\x63reation_height\x18\x07 \x01(\r\x12\x19\n\x11\x61\x63tivation_height\x18\x08 \x01(\r\x12\x19\n\x11\x65xpiration_height\x18\t \x01(\r\x12\x19\n\x11\x63laims_in_channel\x18\n \x01(\r\x12\x10\n\x08reposted\x18\x0b \x01(\r\x12\x18\n\x10\x65\x66\x66\x65\x63tive_amount\x18\x14 \x01(\x04\x12\x16\n\x0esupport_amount\x18\x15 \x01(\x04\x12\x16\n\x0etrending_group\x18\x16 \x01(\r\x12\x16\n\x0etrending_mixed\x18\x17 \x01(\x02\x12\x16\n\x0etrending_local\x18\x18 \x01(\x02\x12\x17\n\x0ftrending_global\x18\x19 \x01(\x02\"\x94\x01\n\x05\x45rror\x12\x1c\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x0e.pb.Error.Code\x12\x0c\n\x04text\x18\x02 \x01(\t\x12\x1c\n\x07\x62locked\x18\x03 \x01(\x0b\x32\x0b.pb.Blocked\"A\n\x04\x43ode\x12\x10\n\x0cUNKNOWN_CODE\x10\x00\x12\r\n\tNOT_FOUND\x10\x01\x12\x0b\n\x07INVALID\x10\x02\x12\x0b\n\x07\x42LOCKED\x10\x03\"5\n\x07\x42locked\x12\r\n\x05\x63ount\x18\x01 \x01(\r\x12\x1b\n\x07\x63hannel\x18\x02 \x01(\x0b\x32\n.pb.Outputb\x06proto3')
|
||||
)
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
|
||||
|
||||
_ERROR_CODE = _descriptor.EnumDescriptor(
|
||||
name='Code',
|
||||
full_name='pb.Error.Code',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='UNKNOWN_CODE', index=0, number=0,
|
||||
options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='NOT_FOUND', index=1, number=1,
|
||||
options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='INVALID', index=2, number=2,
|
||||
options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='BLOCKED', index=3, number=3,
|
||||
options=None,
|
||||
type=None),
|
||||
],
|
||||
containing_type=None,
|
||||
options=None,
|
||||
serialized_start=817,
|
||||
serialized_end=882,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_ERROR_CODE)
|
||||
|
||||
|
||||
_OUTPUTS = _descriptor.Descriptor(
|
||||
name='Outputs',
|
||||
full_name='pb.Outputs',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='txos', full_name='pb.Outputs.txos', index=0,
|
||||
number=1, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='extra_txos', full_name='pb.Outputs.extra_txos', index=1,
|
||||
number=2, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='total', full_name='pb.Outputs.total', index=2,
|
||||
number=3, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='offset', full_name='pb.Outputs.offset', index=3,
|
||||
number=4, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='blocked', full_name='pb.Outputs.blocked', index=4,
|
||||
number=5, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='blocked_total', full_name='pb.Outputs.blocked_total', index=5,
|
||||
number=6, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=21,
|
||||
serialized_end=172,
|
||||
)
|
||||
|
||||
|
||||
_OUTPUT = _descriptor.Descriptor(
|
||||
name='Output',
|
||||
full_name='pb.Output',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='tx_hash', full_name='pb.Output.tx_hash', index=0,
|
||||
number=1, type=12, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='nout', full_name='pb.Output.nout', index=1,
|
||||
number=2, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='height', full_name='pb.Output.height', index=2,
|
||||
number=3, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='claim', full_name='pb.Output.claim', index=3,
|
||||
number=7, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='error', full_name='pb.Output.error', index=4,
|
||||
number=15, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
_descriptor.OneofDescriptor(
|
||||
name='meta', full_name='pb.Output.meta',
|
||||
index=0, containing_type=None, fields=[]),
|
||||
],
|
||||
serialized_start=174,
|
||||
serialized_end=297,
|
||||
)
|
||||
|
||||
|
||||
_CLAIMMETA = _descriptor.Descriptor(
|
||||
name='ClaimMeta',
|
||||
full_name='pb.ClaimMeta',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='channel', full_name='pb.ClaimMeta.channel', index=0,
|
||||
number=1, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='repost', full_name='pb.ClaimMeta.repost', index=1,
|
||||
number=2, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='short_url', full_name='pb.ClaimMeta.short_url', index=2,
|
||||
number=3, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='canonical_url', full_name='pb.ClaimMeta.canonical_url', index=3,
|
||||
number=4, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='is_controlling', full_name='pb.ClaimMeta.is_controlling', index=4,
|
||||
number=5, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='take_over_height', full_name='pb.ClaimMeta.take_over_height', index=5,
|
||||
number=6, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='creation_height', full_name='pb.ClaimMeta.creation_height', index=6,
|
||||
number=7, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='activation_height', full_name='pb.ClaimMeta.activation_height', index=7,
|
||||
number=8, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='expiration_height', full_name='pb.ClaimMeta.expiration_height', index=8,
|
||||
number=9, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='claims_in_channel', full_name='pb.ClaimMeta.claims_in_channel', index=9,
|
||||
number=10, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='reposted', full_name='pb.ClaimMeta.reposted', index=10,
|
||||
number=11, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='effective_amount', full_name='pb.ClaimMeta.effective_amount', index=11,
|
||||
number=20, type=4, cpp_type=4, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='support_amount', full_name='pb.ClaimMeta.support_amount', index=12,
|
||||
number=21, type=4, cpp_type=4, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='trending_group', full_name='pb.ClaimMeta.trending_group', index=13,
|
||||
number=22, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='trending_mixed', full_name='pb.ClaimMeta.trending_mixed', index=14,
|
||||
number=23, type=2, cpp_type=6, label=1,
|
||||
has_default_value=False, default_value=float(0),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='trending_local', full_name='pb.ClaimMeta.trending_local', index=15,
|
||||
number=24, type=2, cpp_type=6, label=1,
|
||||
has_default_value=False, default_value=float(0),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='trending_global', full_name='pb.ClaimMeta.trending_global', index=16,
|
||||
number=25, type=2, cpp_type=6, label=1,
|
||||
has_default_value=False, default_value=float(0),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=300,
|
||||
serialized_end=731,
|
||||
)
|
||||
|
||||
|
||||
_ERROR = _descriptor.Descriptor(
|
||||
name='Error',
|
||||
full_name='pb.Error',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='code', full_name='pb.Error.code', index=0,
|
||||
number=1, type=14, cpp_type=8, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='text', full_name='pb.Error.text', index=1,
|
||||
number=2, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='blocked', full_name='pb.Error.blocked', index=2,
|
||||
number=3, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
_ERROR_CODE,
|
||||
],
|
||||
options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=734,
|
||||
serialized_end=882,
|
||||
)
|
||||
|
||||
|
||||
_BLOCKED = _descriptor.Descriptor(
|
||||
name='Blocked',
|
||||
full_name='pb.Blocked',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='count', full_name='pb.Blocked.count', index=0,
|
||||
number=1, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='channel', full_name='pb.Blocked.channel', index=1,
|
||||
number=2, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=884,
|
||||
serialized_end=937,
|
||||
)
|
||||
|
||||
_OUTPUTS.fields_by_name['txos'].message_type = _OUTPUT
|
||||
_OUTPUTS.fields_by_name['extra_txos'].message_type = _OUTPUT
|
||||
_OUTPUTS.fields_by_name['blocked'].message_type = _BLOCKED
|
||||
_OUTPUT.fields_by_name['claim'].message_type = _CLAIMMETA
|
||||
_OUTPUT.fields_by_name['error'].message_type = _ERROR
|
||||
_OUTPUT.oneofs_by_name['meta'].fields.append(
|
||||
_OUTPUT.fields_by_name['claim'])
|
||||
_OUTPUT.fields_by_name['claim'].containing_oneof = _OUTPUT.oneofs_by_name['meta']
|
||||
_OUTPUT.oneofs_by_name['meta'].fields.append(
|
||||
_OUTPUT.fields_by_name['error'])
|
||||
_OUTPUT.fields_by_name['error'].containing_oneof = _OUTPUT.oneofs_by_name['meta']
|
||||
_CLAIMMETA.fields_by_name['channel'].message_type = _OUTPUT
|
||||
_CLAIMMETA.fields_by_name['repost'].message_type = _OUTPUT
|
||||
_ERROR.fields_by_name['code'].enum_type = _ERROR_CODE
|
||||
_ERROR.fields_by_name['blocked'].message_type = _BLOCKED
|
||||
_ERROR_CODE.containing_type = _ERROR
|
||||
_BLOCKED.fields_by_name['channel'].message_type = _OUTPUT
|
||||
DESCRIPTOR.message_types_by_name['Outputs'] = _OUTPUTS
|
||||
DESCRIPTOR.message_types_by_name['Output'] = _OUTPUT
|
||||
DESCRIPTOR.message_types_by_name['ClaimMeta'] = _CLAIMMETA
|
||||
DESCRIPTOR.message_types_by_name['Error'] = _ERROR
|
||||
DESCRIPTOR.message_types_by_name['Blocked'] = _BLOCKED
|
||||
|
||||
Outputs = _reflection.GeneratedProtocolMessageType('Outputs', (_message.Message,), dict(
|
||||
DESCRIPTOR = _OUTPUTS,
|
||||
__module__ = 'result_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.Outputs)
|
||||
))
|
||||
_sym_db.RegisterMessage(Outputs)
|
||||
|
||||
Output = _reflection.GeneratedProtocolMessageType('Output', (_message.Message,), dict(
|
||||
DESCRIPTOR = _OUTPUT,
|
||||
__module__ = 'result_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.Output)
|
||||
))
|
||||
_sym_db.RegisterMessage(Output)
|
||||
|
||||
ClaimMeta = _reflection.GeneratedProtocolMessageType('ClaimMeta', (_message.Message,), dict(
|
||||
DESCRIPTOR = _CLAIMMETA,
|
||||
__module__ = 'result_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.ClaimMeta)
|
||||
))
|
||||
_sym_db.RegisterMessage(ClaimMeta)
|
||||
|
||||
Error = _reflection.GeneratedProtocolMessageType('Error', (_message.Message,), dict(
|
||||
DESCRIPTOR = _ERROR,
|
||||
__module__ = 'result_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.Error)
|
||||
))
|
||||
_sym_db.RegisterMessage(Error)
|
||||
|
||||
Blocked = _reflection.GeneratedProtocolMessageType('Blocked', (_message.Message,), dict(
|
||||
DESCRIPTOR = _BLOCKED,
|
||||
__module__ = 'result_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.Blocked)
|
||||
))
|
||||
_sym_db.RegisterMessage(Blocked)
|
||||
|
||||
|
||||
# @@protoc_insertion_point(module_scope)
|
||||
@@ -1,129 +0,0 @@
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import NamedTuple, Tuple
|
||||
|
||||
|
||||
def _create_url_regex():
|
||||
# see https://spec.lbry.com/ and test_url.py
|
||||
invalid_names_regex = \
|
||||
r"[^=&#:$@%?;\"/\\<>%{}|^~`\[\]" \
|
||||
r"\u0000-\u0020\uD800-\uDFFF\uFFFE-\uFFFF]+"
|
||||
|
||||
def _named(name, regex):
|
||||
return "(?P<" + name + ">" + regex + ")"
|
||||
|
||||
def _group(regex):
|
||||
return "(?:" + regex + ")"
|
||||
|
||||
def _oneof(*choices):
|
||||
return _group('|'.join(choices))
|
||||
|
||||
def _claim(name, prefix=""):
|
||||
return _group(
|
||||
_named(name+"_name", prefix + invalid_names_regex) +
|
||||
_oneof(
|
||||
_group('#' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
|
||||
_group(':' + _named(name+"_sequence", '[1-9][0-9]*')),
|
||||
_group(r'\$' + _named(name+"_amount_order", '[1-9][0-9]*'))
|
||||
) + '?'
|
||||
)
|
||||
|
||||
return (
|
||||
'^' +
|
||||
_named("scheme", "lbry://") + '?' +
|
||||
_oneof(
|
||||
_group(_claim("channel_with_stream", "@") + "/" + _claim("stream_in_channel")),
|
||||
_claim("channel", "@"),
|
||||
_claim("stream")
|
||||
) +
|
||||
'$'
|
||||
)
|
||||
|
||||
|
||||
URL_REGEX = _create_url_regex()
|
||||
|
||||
|
||||
def normalize_name(name):
|
||||
return unicodedata.normalize('NFD', name).casefold()
|
||||
|
||||
|
||||
class PathSegment(NamedTuple):
|
||||
name: str
|
||||
claim_id: str = None
|
||||
sequence: int = None
|
||||
amount_order: int = None
|
||||
|
||||
@property
|
||||
def normalized(self):
|
||||
return normalize_name(self.name)
|
||||
|
||||
def to_dict(self):
|
||||
q = {'name': self.name}
|
||||
if self.claim_id is not None:
|
||||
q['claim_id'] = self.claim_id
|
||||
if self.sequence is not None:
|
||||
q['sequence'] = self.sequence
|
||||
if self.amount_order is not None:
|
||||
q['amount_order'] = self.amount_order
|
||||
return q
|
||||
|
||||
def __str__(self):
|
||||
if self.claim_id is not None:
|
||||
return f"{self.name}#{self.claim_id}"
|
||||
elif self.sequence is not None:
|
||||
return f"{self.name}:{self.sequence}"
|
||||
elif self.amount_order is not None:
|
||||
return f"{self.name}${self.amount_order}"
|
||||
return self.name
|
||||
|
||||
|
||||
class URL(NamedTuple):
|
||||
stream: PathSegment
|
||||
channel: PathSegment
|
||||
|
||||
@property
|
||||
def has_channel(self):
|
||||
return self.channel is not None
|
||||
|
||||
@property
|
||||
def has_stream(self):
|
||||
return self.stream is not None
|
||||
|
||||
@property
|
||||
def has_stream_in_channel(self):
|
||||
return self.has_channel and self.has_stream
|
||||
|
||||
@property
|
||||
def parts(self) -> Tuple:
|
||||
if self.has_stream_in_channel:
|
||||
return self.channel, self.stream
|
||||
if self.has_channel:
|
||||
return self.channel,
|
||||
return self.stream,
|
||||
|
||||
def __str__(self):
|
||||
return f"lbry://{'/'.join(str(p) for p in self.parts)}"
|
||||
|
||||
@classmethod
|
||||
def parse(cls, url):
|
||||
match = re.match(URL_REGEX, url)
|
||||
|
||||
if match is None:
|
||||
raise ValueError('Invalid LBRY URL')
|
||||
|
||||
segments = {}
|
||||
parts = match.groupdict()
|
||||
for segment in ('channel', 'stream', 'channel_with_stream', 'stream_in_channel'):
|
||||
if parts[f'{segment}_name'] is not None:
|
||||
segments[segment] = PathSegment(
|
||||
parts[f'{segment}_name'],
|
||||
parts[f'{segment}_claim_id'],
|
||||
parts[f'{segment}_sequence'],
|
||||
parts[f'{segment}_amount_order']
|
||||
)
|
||||
|
||||
if 'channel_with_stream' in segments:
|
||||
segments['channel'] = segments['channel_with_stream']
|
||||
segments['stream'] = segments['stream_in_channel']
|
||||
|
||||
return cls(segments.get('stream', None), segments.get('channel', None))
|
||||
@@ -0,0 +1,30 @@
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from lbry.stream.downloader import StreamDownloader
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BackgroundDownloader:
|
||||
def __init__(self, conf, storage, blob_manager, dht_node=None):
|
||||
self.storage = storage
|
||||
self.blob_manager = blob_manager
|
||||
self.node = dht_node
|
||||
self.conf = conf
|
||||
|
||||
async def download_blobs(self, sd_hash):
|
||||
downloader = StreamDownloader(asyncio.get_running_loop(), self.conf, self.blob_manager, sd_hash)
|
||||
try:
|
||||
await downloader.start(self.node, save_stream=False)
|
||||
for blob_info in downloader.descriptor.blobs[:-1]:
|
||||
await downloader.download_stream_blob(blob_info)
|
||||
except ValueError:
|
||||
return
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
log.error("Unexpected download error on background downloader")
|
||||
finally:
|
||||
downloader.stop()
|
||||
+36
-19
@@ -4,6 +4,7 @@ import binascii
|
||||
import logging
|
||||
import typing
|
||||
import asyncio
|
||||
import time
|
||||
import re
|
||||
from collections import OrderedDict
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
@@ -44,18 +45,25 @@ def random_iv_generator() -> typing.Generator[bytes, None, None]:
|
||||
yield os.urandom(AES.block_size // 8)
|
||||
|
||||
|
||||
def file_reader(file_path: str):
|
||||
def read_bytes(file_path: str, offset: int, to_read: int):
|
||||
with open(file_path, 'rb') as f:
|
||||
f.seek(offset)
|
||||
return f.read(to_read)
|
||||
|
||||
|
||||
async def file_reader(file_path: str):
|
||||
length = int(os.stat(file_path).st_size)
|
||||
offset = 0
|
||||
|
||||
with open(file_path, 'rb') as stream_file:
|
||||
while offset < length:
|
||||
bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1)
|
||||
if not bytes_to_read:
|
||||
break
|
||||
blob_bytes = stream_file.read(bytes_to_read)
|
||||
yield blob_bytes
|
||||
offset += bytes_to_read
|
||||
while offset < length:
|
||||
bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1)
|
||||
if not bytes_to_read:
|
||||
break
|
||||
blob_bytes = await asyncio.get_event_loop().run_in_executor(
|
||||
None, read_bytes, file_path, offset, bytes_to_read
|
||||
)
|
||||
yield blob_bytes
|
||||
offset += bytes_to_read
|
||||
|
||||
|
||||
def sanitize_file_name(dirty_name: str, default_file_name: str = 'lbry_download'):
|
||||
@@ -145,15 +153,19 @@ class StreamDescriptor:
|
||||
h.update(self.old_sort_json())
|
||||
return h.hexdigest()
|
||||
|
||||
async def make_sd_blob(self, blob_file_obj: typing.Optional[AbstractBlob] = None,
|
||||
old_sort: typing.Optional[bool] = False,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None):
|
||||
async def make_sd_blob(
|
||||
self, blob_file_obj: typing.Optional[AbstractBlob] = None, old_sort: typing.Optional[bool] = False,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None,
|
||||
added_on: float = None, is_mine: bool = False
|
||||
):
|
||||
sd_hash = self.calculate_sd_hash() if not old_sort else self.calculate_old_sort_sd_hash()
|
||||
if not old_sort:
|
||||
sd_data = self.as_json()
|
||||
else:
|
||||
sd_data = self.old_sort_json()
|
||||
sd_blob = blob_file_obj or BlobFile(self.loop, sd_hash, len(sd_data), blob_completed_callback, self.blob_dir)
|
||||
sd_blob = blob_file_obj or BlobFile(
|
||||
self.loop, sd_hash, len(sd_data), blob_completed_callback, self.blob_dir, added_on, is_mine
|
||||
)
|
||||
if blob_file_obj:
|
||||
blob_file_obj.set_length(len(sd_data))
|
||||
if not sd_blob.get_is_verified():
|
||||
@@ -176,11 +188,11 @@ class StreamDescriptor:
|
||||
raise InvalidStreamDescriptorError("Does not decode as valid JSON")
|
||||
if decoded['blobs'][-1]['length'] != 0:
|
||||
raise InvalidStreamDescriptorError("Does not end with a zero-length blob.")
|
||||
if any([blob_info['length'] == 0 for blob_info in decoded['blobs'][:-1]]):
|
||||
if any(blob_info['length'] == 0 for blob_info in decoded['blobs'][:-1]):
|
||||
raise InvalidStreamDescriptorError("Contains zero-length data blob")
|
||||
if 'blob_hash' in decoded['blobs'][-1]:
|
||||
raise InvalidStreamDescriptorError("Stream terminator blob should not have a hash")
|
||||
if any([i != blob_info['blob_num'] for i, blob_info in enumerate(decoded['blobs'])]):
|
||||
if any(i != blob_info['blob_num'] for i, blob_info in enumerate(decoded['blobs'])):
|
||||
raise InvalidStreamDescriptorError("Stream contains out of order or skipped blobs")
|
||||
descriptor = cls(
|
||||
loop, blob_dir,
|
||||
@@ -245,20 +257,25 @@ class StreamDescriptor:
|
||||
iv_generator = iv_generator or random_iv_generator()
|
||||
key = key or os.urandom(AES.block_size // 8)
|
||||
blob_num = -1
|
||||
for blob_bytes in file_reader(file_path):
|
||||
added_on = time.time()
|
||||
async for blob_bytes in file_reader(file_path):
|
||||
blob_num += 1
|
||||
blob_info = await BlobFile.create_from_unencrypted(
|
||||
loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num, blob_completed_callback
|
||||
loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num, added_on, True, blob_completed_callback
|
||||
)
|
||||
blobs.append(blob_info)
|
||||
blobs.append(
|
||||
BlobInfo(len(blobs), 0, binascii.hexlify(next(iv_generator)).decode())) # add the stream terminator
|
||||
# add the stream terminator
|
||||
BlobInfo(len(blobs), 0, binascii.hexlify(next(iv_generator)).decode(), None, added_on, True)
|
||||
)
|
||||
file_name = os.path.basename(file_path)
|
||||
suggested_file_name = sanitize_file_name(file_name)
|
||||
descriptor = cls(
|
||||
loop, blob_dir, file_name, binascii.hexlify(key).decode(), suggested_file_name, blobs
|
||||
)
|
||||
sd_blob = await descriptor.make_sd_blob(old_sort=old_sort, blob_completed_callback=blob_completed_callback)
|
||||
sd_blob = await descriptor.make_sd_blob(
|
||||
old_sort=old_sort, blob_completed_callback=blob_completed_callback, added_on=added_on, is_mine=True
|
||||
)
|
||||
descriptor.sd_hash = sd_blob.blob_hash
|
||||
return descriptor
|
||||
|
||||
|
||||
+12
-23
@@ -3,9 +3,9 @@ import typing
|
||||
import logging
|
||||
import binascii
|
||||
|
||||
from lbry.dht.peer import make_kademlia_peer
|
||||
from lbry.dht.node import get_kademlia_peers_from_hosts
|
||||
from lbry.error import DownloadSDTimeoutError
|
||||
from lbry.utils import resolve_host, lru_cache_concurrent
|
||||
from lbry.utils import lru_cache_concurrent
|
||||
from lbry.stream.descriptor import StreamDescriptor
|
||||
from lbry.blob_exchange.downloader import BlobDownloader
|
||||
if typing.TYPE_CHECKING:
|
||||
@@ -40,7 +40,7 @@ class StreamDownloader:
|
||||
async def cached_read_blob(blob_info: 'BlobInfo') -> bytes:
|
||||
return await self.read_blob(blob_info, 2)
|
||||
|
||||
if self.blob_manager.decrypted_blob_lru_cache:
|
||||
if self.blob_manager.decrypted_blob_lru_cache is not None:
|
||||
cached_read_blob = lru_cache_concurrent(override_lru_cache=self.blob_manager.decrypted_blob_lru_cache)(
|
||||
cached_read_blob
|
||||
)
|
||||
@@ -48,26 +48,19 @@ class StreamDownloader:
|
||||
self.cached_read_blob = cached_read_blob
|
||||
|
||||
async def add_fixed_peers(self):
|
||||
def _delayed_add_fixed_peers():
|
||||
def _add_fixed_peers(fixed_peers):
|
||||
self.peer_queue.put_nowait(fixed_peers)
|
||||
self.added_fixed_peers = True
|
||||
self.peer_queue.put_nowait([
|
||||
make_kademlia_peer(None, address, None, tcp_port=port + 1, allow_localhost=True)
|
||||
for address, port in addresses
|
||||
])
|
||||
|
||||
if not self.config.reflector_servers:
|
||||
if not self.config.fixed_peers:
|
||||
return
|
||||
addresses = [
|
||||
(await resolve_host(url, port + 1, proto='tcp'), port)
|
||||
for url, port in self.config.reflector_servers
|
||||
]
|
||||
if 'dht' in self.config.components_to_skip or not self.node or not \
|
||||
len(self.node.protocol.routing_table.get_peers()) > 0:
|
||||
self.fixed_peers_delay = 0.0
|
||||
else:
|
||||
self.fixed_peers_delay = self.config.fixed_peer_delay
|
||||
|
||||
self.fixed_peers_handle = self.loop.call_later(self.fixed_peers_delay, _delayed_add_fixed_peers)
|
||||
fixed_peers = await get_kademlia_peers_from_hosts(self.config.fixed_peers)
|
||||
self.fixed_peers_handle = self.loop.call_later(self.fixed_peers_delay, _add_fixed_peers, fixed_peers)
|
||||
|
||||
async def load_descriptor(self, connection_id: int = 0):
|
||||
# download or get the sd blob
|
||||
@@ -90,10 +83,10 @@ class StreamDownloader:
|
||||
)
|
||||
log.info("loaded stream manifest %s", self.sd_hash)
|
||||
|
||||
async def start(self, node: typing.Optional['Node'] = None, connection_id: int = 0):
|
||||
async def start(self, node: typing.Optional['Node'] = None, connection_id: int = 0, save_stream=True):
|
||||
# set up peer accumulation
|
||||
if node:
|
||||
self.node = node
|
||||
self.node = node or self.node # fixme: this shouldnt be set here!
|
||||
if self.node:
|
||||
if self.accumulate_task and not self.accumulate_task.done():
|
||||
self.accumulate_task.cancel()
|
||||
_, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue)
|
||||
@@ -105,11 +98,7 @@ class StreamDownloader:
|
||||
if not self.descriptor:
|
||||
await self.load_descriptor(connection_id)
|
||||
|
||||
# add the head blob to the peer search
|
||||
self.search_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
|
||||
log.info("added head blob to peer search for stream %s", self.sd_hash)
|
||||
|
||||
if not await self.blob_manager.storage.stream_exists(self.sd_hash):
|
||||
if not await self.blob_manager.storage.stream_exists(self.sd_hash) and save_stream:
|
||||
await self.blob_manager.storage.store_stream(
|
||||
self.blob_manager.get_blob(self.sd_hash, length=self.descriptor.length), self.descriptor
|
||||
)
|
||||
|
||||
+85
-179
@@ -3,20 +3,20 @@ import asyncio
|
||||
import time
|
||||
import typing
|
||||
import logging
|
||||
import binascii
|
||||
from typing import Optional
|
||||
from aiohttp.web import Request, StreamResponse, HTTPRequestRangeNotSatisfiable
|
||||
from lbry.utils import generate_id
|
||||
from lbry.error import DownloadSDTimeoutError
|
||||
from lbry.schema.mime_types import guess_media_type
|
||||
from scribe.schema.mime_types import guess_media_type
|
||||
from lbry.stream.downloader import StreamDownloader
|
||||
from lbry.stream.descriptor import StreamDescriptor, sanitize_file_name
|
||||
from lbry.stream.reflector.client import StreamReflectorClient
|
||||
from lbry.extras.daemon.storage import StoredContentClaim
|
||||
from lbry.blob import MAX_BLOB_SIZE
|
||||
from lbry.file.source import ManagedDownloadSource
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.schema.claim import Claim
|
||||
from scribe.schema.claim import Claim
|
||||
from lbry.blob.blob_manager import BlobManager
|
||||
from lbry.blob.blob_info import BlobInfo
|
||||
from lbry.dht.node import Node
|
||||
@@ -40,76 +40,35 @@ async def get_next_available_file_name(loop: asyncio.AbstractEventLoop, download
|
||||
return await loop.run_in_executor(None, _get_next_available_file_name, download_directory, file_name)
|
||||
|
||||
|
||||
class ManagedStream:
|
||||
STATUS_RUNNING = "running"
|
||||
STATUS_STOPPED = "stopped"
|
||||
STATUS_FINISHED = "finished"
|
||||
|
||||
SAVING_ID = 1
|
||||
STREAMING_ID = 2
|
||||
|
||||
__slots__ = [
|
||||
'loop',
|
||||
'config',
|
||||
'blob_manager',
|
||||
'sd_hash',
|
||||
'download_directory',
|
||||
'_file_name',
|
||||
'_added_on',
|
||||
'_status',
|
||||
'stream_claim_info',
|
||||
'download_id',
|
||||
'rowid',
|
||||
'content_fee',
|
||||
'purchase_receipt',
|
||||
'downloader',
|
||||
'analytics_manager',
|
||||
'fully_reflected',
|
||||
'file_output_task',
|
||||
'delayed_stop_task',
|
||||
'streaming_responses',
|
||||
'streaming',
|
||||
'_running',
|
||||
'saving',
|
||||
'finished_writing',
|
||||
'started_writing',
|
||||
'finished_write_attempt'
|
||||
]
|
||||
|
||||
class ManagedStream(ManagedDownloadSource):
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
sd_hash: str, download_directory: typing.Optional[str] = None, file_name: typing.Optional[str] = None,
|
||||
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional[StoredContentClaim] = None,
|
||||
download_id: typing.Optional[str] = None, rowid: typing.Optional[int] = None,
|
||||
descriptor: typing.Optional[StreamDescriptor] = None,
|
||||
content_fee: typing.Optional['Transaction'] = None,
|
||||
analytics_manager: typing.Optional['AnalyticsManager'] = None,
|
||||
added_on: typing.Optional[int] = None):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
sd_hash: str, download_directory: Optional[str] = None, file_name: Optional[str] = None,
|
||||
status: Optional[str] = ManagedDownloadSource.STATUS_STOPPED,
|
||||
claim: Optional[StoredContentClaim] = None,
|
||||
download_id: Optional[str] = None, rowid: Optional[int] = None,
|
||||
descriptor: Optional[StreamDescriptor] = None,
|
||||
content_fee: Optional['Transaction'] = None,
|
||||
analytics_manager: Optional['AnalyticsManager'] = None,
|
||||
added_on: Optional[int] = None):
|
||||
super().__init__(loop, config, blob_manager.storage, sd_hash, file_name, download_directory, status, claim,
|
||||
download_id, rowid, content_fee, analytics_manager, added_on)
|
||||
self.blob_manager = blob_manager
|
||||
self.sd_hash = sd_hash
|
||||
self.download_directory = download_directory
|
||||
self._file_name = file_name
|
||||
self._status = status
|
||||
self.stream_claim_info = claim
|
||||
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
|
||||
self.rowid = rowid
|
||||
self.content_fee = content_fee
|
||||
self.purchase_receipt = None
|
||||
self._added_on = added_on
|
||||
self.downloader = StreamDownloader(self.loop, self.config, self.blob_manager, sd_hash, descriptor)
|
||||
self.analytics_manager = analytics_manager
|
||||
|
||||
self.fully_reflected = asyncio.Event(loop=self.loop)
|
||||
self.reflector_progress = 0
|
||||
self.uploading_to_reflector = False
|
||||
self.file_output_task: typing.Optional[asyncio.Task] = None
|
||||
self.delayed_stop_task: typing.Optional[asyncio.Task] = None
|
||||
self.streaming_responses: typing.List[typing.Tuple[Request, StreamResponse]] = []
|
||||
self.fully_reflected = asyncio.Event(loop=self.loop)
|
||||
self.streaming = asyncio.Event(loop=self.loop)
|
||||
self._running = asyncio.Event(loop=self.loop)
|
||||
self.saving = asyncio.Event(loop=self.loop)
|
||||
self.finished_writing = asyncio.Event(loop=self.loop)
|
||||
self.started_writing = asyncio.Event(loop=self.loop)
|
||||
self.finished_write_attempt = asyncio.Event(loop=self.loop)
|
||||
|
||||
@property
|
||||
def sd_hash(self) -> str:
|
||||
return self.identifier
|
||||
|
||||
@property
|
||||
def is_fully_reflected(self) -> bool:
|
||||
@@ -124,17 +83,9 @@ class ManagedStream:
|
||||
return self.descriptor.stream_hash
|
||||
|
||||
@property
|
||||
def file_name(self) -> typing.Optional[str]:
|
||||
def file_name(self) -> Optional[str]:
|
||||
return self._file_name or (self.descriptor.suggested_file_name if self.descriptor else None)
|
||||
|
||||
@property
|
||||
def added_on(self) -> typing.Optional[int]:
|
||||
return self._added_on
|
||||
|
||||
@property
|
||||
def status(self) -> str:
|
||||
return self._status
|
||||
|
||||
@property
|
||||
def written_bytes(self) -> int:
|
||||
return 0 if not self.output_file_exists else os.stat(self.full_path).st_size
|
||||
@@ -152,55 +103,6 @@ class ManagedStream:
|
||||
self._status = status
|
||||
await self.blob_manager.storage.change_file_status(self.stream_hash, status)
|
||||
|
||||
@property
|
||||
def finished(self) -> bool:
|
||||
return self.status == self.STATUS_FINISHED
|
||||
|
||||
@property
|
||||
def running(self) -> bool:
|
||||
return self.status == self.STATUS_RUNNING
|
||||
|
||||
@property
|
||||
def claim_id(self) -> typing.Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim_id
|
||||
|
||||
@property
|
||||
def txid(self) -> typing.Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.txid
|
||||
|
||||
@property
|
||||
def nout(self) -> typing.Optional[int]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.nout
|
||||
|
||||
@property
|
||||
def outpoint(self) -> typing.Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.outpoint
|
||||
|
||||
@property
|
||||
def claim_height(self) -> typing.Optional[int]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.height
|
||||
|
||||
@property
|
||||
def channel_claim_id(self) -> typing.Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.channel_claim_id
|
||||
|
||||
@property
|
||||
def channel_name(self) -> typing.Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.channel_name
|
||||
|
||||
@property
|
||||
def claim_name(self) -> typing.Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim_name
|
||||
|
||||
@property
|
||||
def metadata(self) -> typing.Optional[typing.Dict]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict()
|
||||
|
||||
@property
|
||||
def metadata_protobuf(self) -> bytes:
|
||||
if self.stream_claim_info:
|
||||
return binascii.hexlify(self.stream_claim_info.claim.to_bytes())
|
||||
|
||||
@property
|
||||
def blobs_completed(self) -> int:
|
||||
return sum([1 if b.blob_hash in self.blob_manager.completed_blob_hashes else 0
|
||||
@@ -214,39 +116,34 @@ class ManagedStream:
|
||||
def blobs_remaining(self) -> int:
|
||||
return self.blobs_in_stream - self.blobs_completed
|
||||
|
||||
@property
|
||||
def full_path(self) -> typing.Optional[str]:
|
||||
return os.path.join(self.download_directory, os.path.basename(self.file_name)) \
|
||||
if self.file_name and self.download_directory else None
|
||||
|
||||
@property
|
||||
def output_file_exists(self):
|
||||
return os.path.isfile(self.full_path) if self.full_path else False
|
||||
|
||||
@property
|
||||
def mime_type(self):
|
||||
return guess_media_type(os.path.basename(self.descriptor.suggested_file_name))[0]
|
||||
|
||||
@classmethod
|
||||
async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
file_path: str, key: typing.Optional[bytes] = None,
|
||||
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedStream':
|
||||
"""
|
||||
Generate a stream from a file and save it to the db
|
||||
"""
|
||||
descriptor = await StreamDescriptor.create_stream(
|
||||
loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
|
||||
blob_completed_callback=blob_manager.blob_completed
|
||||
)
|
||||
await blob_manager.storage.store_stream(
|
||||
blob_manager.get_blob(descriptor.sd_hash), descriptor
|
||||
)
|
||||
row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path),
|
||||
os.path.dirname(file_path), 0)
|
||||
return cls(loop, config, blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
|
||||
os.path.basename(file_path), status=cls.STATUS_FINISHED, rowid=row_id, descriptor=descriptor)
|
||||
@property
|
||||
def download_path(self):
|
||||
return f"{self.download_directory}/{self._file_name}" if self.download_directory and self._file_name else None
|
||||
|
||||
async def start(self, node: typing.Optional['Node'] = None, timeout: typing.Optional[float] = None,
|
||||
# @classmethod
|
||||
# async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config',
|
||||
# file_path: str, key: Optional[bytes] = None,
|
||||
# iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedDownloadSource':
|
||||
# """
|
||||
# Generate a stream from a file and save it to the db
|
||||
# """
|
||||
# descriptor = await StreamDescriptor.create_stream(
|
||||
# loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
|
||||
# blob_completed_callback=blob_manager.blob_completed
|
||||
# )
|
||||
# await blob_manager.storage.store_stream(
|
||||
# blob_manager.get_blob(descriptor.sd_hash), descriptor
|
||||
# )
|
||||
# row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path),
|
||||
# os.path.dirname(file_path), 0)
|
||||
# return cls(loop, config, blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
|
||||
# os.path.basename(file_path), status=cls.STATUS_FINISHED, rowid=row_id, descriptor=descriptor)
|
||||
|
||||
async def start(self, timeout: Optional[float] = None,
|
||||
save_now: bool = False):
|
||||
timeout = timeout or self.config.download_timeout
|
||||
if self._running.is_set():
|
||||
@@ -254,7 +151,7 @@ class ManagedStream:
|
||||
log.info("start downloader for stream (sd hash: %s)", self.sd_hash)
|
||||
self._running.set()
|
||||
try:
|
||||
await asyncio.wait_for(self.downloader.start(node), timeout, loop=self.loop)
|
||||
await asyncio.wait_for(self.downloader.start(), timeout, loop=self.loop)
|
||||
except asyncio.TimeoutError:
|
||||
self._running.clear()
|
||||
raise DownloadSDTimeoutError(self.sd_hash)
|
||||
@@ -264,6 +161,11 @@ class ManagedStream:
|
||||
self.delayed_stop_task = self.loop.create_task(self._delayed_stop())
|
||||
if not await self.blob_manager.storage.file_exists(self.sd_hash):
|
||||
if save_now:
|
||||
if not self._file_name:
|
||||
self._file_name = await get_next_available_file_name(
|
||||
self.loop, self.download_directory,
|
||||
self._file_name or sanitize_file_name(self.descriptor.suggested_file_name)
|
||||
)
|
||||
file_name, download_dir = self._file_name, self.download_directory
|
||||
else:
|
||||
file_name, download_dir = None, None
|
||||
@@ -283,7 +185,7 @@ class ManagedStream:
|
||||
if (finished and self.status != self.STATUS_FINISHED) or self.status == self.STATUS_RUNNING:
|
||||
await self.update_status(self.STATUS_FINISHED if finished else self.STATUS_STOPPED)
|
||||
|
||||
async def _aiter_read_stream(self, start_blob_num: typing.Optional[int] = 0, connection_id: int = 0)\
|
||||
async def _aiter_read_stream(self, start_blob_num: Optional[int] = 0, connection_id: int = 0)\
|
||||
-> typing.AsyncIterator[typing.Tuple['BlobInfo', bytes]]:
|
||||
if start_blob_num >= len(self.descriptor.blobs[:-1]):
|
||||
raise IndexError(start_blob_num)
|
||||
@@ -295,13 +197,13 @@ class ManagedStream:
|
||||
decrypted = await self.downloader.read_blob(blob_info, connection_id)
|
||||
yield (blob_info, decrypted)
|
||||
|
||||
async def stream_file(self, request: Request, node: typing.Optional['Node'] = None) -> StreamResponse:
|
||||
async def stream_file(self, request: Request) -> StreamResponse:
|
||||
log.info("stream file to browser for lbry://%s#%s (sd hash %s...)", self.claim_name, self.claim_id,
|
||||
self.sd_hash[:6])
|
||||
headers, size, skip_blobs, first_blob_start_offset = self._prepare_range_response_headers(
|
||||
request.headers.get('range', 'bytes=0-')
|
||||
)
|
||||
await self.start(node)
|
||||
await self.start()
|
||||
response = StreamResponse(
|
||||
status=206,
|
||||
headers=headers
|
||||
@@ -339,9 +241,10 @@ class ManagedStream:
|
||||
self.streaming.clear()
|
||||
|
||||
@staticmethod
|
||||
def _write_decrypted_blob(handle: typing.IO, data: bytes):
|
||||
handle.write(data)
|
||||
handle.flush()
|
||||
def _write_decrypted_blob(output_path: str, data: bytes):
|
||||
with open(output_path, 'ab') as handle:
|
||||
handle.write(data)
|
||||
handle.flush()
|
||||
|
||||
async def _save_file(self, output_path: str):
|
||||
log.info("save file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id, self.sd_hash[:6],
|
||||
@@ -351,12 +254,12 @@ class ManagedStream:
|
||||
self.finished_writing.clear()
|
||||
self.started_writing.clear()
|
||||
try:
|
||||
with open(output_path, 'wb') as file_write_handle:
|
||||
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
|
||||
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
|
||||
await self.loop.run_in_executor(None, self._write_decrypted_blob, file_write_handle, decrypted)
|
||||
if not self.started_writing.is_set():
|
||||
self.started_writing.set()
|
||||
open(output_path, 'wb').close() # pylint: disable=consider-using-with
|
||||
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
|
||||
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
|
||||
await self.loop.run_in_executor(None, self._write_decrypted_blob, output_path, decrypted)
|
||||
if not self.started_writing.is_set():
|
||||
self.started_writing.set()
|
||||
await self.update_status(ManagedStream.STATUS_FINISHED)
|
||||
if self.analytics_manager:
|
||||
self.loop.create_task(self.analytics_manager.send_download_finished(
|
||||
@@ -386,9 +289,8 @@ class ManagedStream:
|
||||
self.saving.clear()
|
||||
self.finished_write_attempt.set()
|
||||
|
||||
async def save_file(self, file_name: typing.Optional[str] = None, download_directory: typing.Optional[str] = None,
|
||||
node: typing.Optional['Node'] = None):
|
||||
await self.start(node)
|
||||
async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
|
||||
await self.start()
|
||||
if self.file_output_task and not self.file_output_task.done(): # cancel an already running save task
|
||||
self.file_output_task.cancel()
|
||||
self.download_directory = download_directory or self.download_directory or self.config.download_dir
|
||||
@@ -430,6 +332,7 @@ class ManagedStream:
|
||||
sent = []
|
||||
protocol = StreamReflectorClient(self.blob_manager, self.descriptor)
|
||||
try:
|
||||
self.uploading_to_reflector = True
|
||||
await self.loop.create_connection(lambda: protocol, host, port)
|
||||
await protocol.send_handshake()
|
||||
sent_sd, needed = await protocol.send_descriptor()
|
||||
@@ -445,30 +348,33 @@ class ManagedStream:
|
||||
]
|
||||
log.info("we have %i/%i needed blobs needed by reflector for lbry://%s#%s", len(we_have), len(needed),
|
||||
self.claim_name, self.claim_id)
|
||||
for blob_hash in we_have:
|
||||
for i, blob_hash in enumerate(we_have):
|
||||
await protocol.send_blob(blob_hash)
|
||||
sent.append(blob_hash)
|
||||
self.reflector_progress = int((i + 1) / len(we_have) * 100)
|
||||
except (asyncio.TimeoutError, ValueError):
|
||||
return sent
|
||||
except ConnectionRefusedError:
|
||||
except ConnectionError:
|
||||
return sent
|
||||
except (OSError, Exception) as err:
|
||||
if isinstance(err, asyncio.CancelledError):
|
||||
log.warning("stopped uploading %s#%s to reflector", self.claim_name, self.claim_id)
|
||||
elif isinstance(err, OSError):
|
||||
log.warning(
|
||||
"stopped uploading %s#%s to reflector because blobs were deleted or moved", self.claim_name,
|
||||
self.claim_id
|
||||
)
|
||||
else:
|
||||
log.exception("unexpected error reflecting %s#%s", self.claim_name, self.claim_id)
|
||||
return sent
|
||||
finally:
|
||||
if protocol.transport:
|
||||
protocol.transport.close()
|
||||
if not self.fully_reflected.is_set():
|
||||
self.fully_reflected.set()
|
||||
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
|
||||
self.uploading_to_reflector = False
|
||||
|
||||
return sent
|
||||
|
||||
def set_claim(self, claim_info: typing.Dict, claim: 'Claim'):
|
||||
self.stream_claim_info = StoredContentClaim(
|
||||
f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
|
||||
claim_info['name'], claim_info['amount'], claim_info['height'],
|
||||
binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'],
|
||||
claim_info['claim_sequence'], claim_info.get('channel_name')
|
||||
)
|
||||
|
||||
async def update_content_claim(self, claim_info: typing.Optional[typing.Dict] = None):
|
||||
async def update_content_claim(self, claim_info: Optional[typing.Dict] = None):
|
||||
if not claim_info:
|
||||
claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash)
|
||||
self.set_claim(claim_info, claim_info['value'])
|
||||
|
||||
@@ -35,6 +35,8 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
def connection_lost(self, exc: typing.Optional[Exception]):
|
||||
self.transport = None
|
||||
self.connected.clear()
|
||||
if self.pending_request:
|
||||
self.pending_request.cancel()
|
||||
if self.reflected_blobs:
|
||||
log.info("Finished sending reflector %i blobs", len(self.reflected_blobs))
|
||||
|
||||
@@ -56,12 +58,18 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
self.response_buff = b''
|
||||
return
|
||||
|
||||
async def send_request(self, request_dict: typing.Dict):
|
||||
msg = json.dumps(request_dict)
|
||||
self.transport.write(msg.encode())
|
||||
async def send_request(self, request_dict: typing.Dict, timeout: int = 180):
|
||||
msg = json.dumps(request_dict, sort_keys=True)
|
||||
try:
|
||||
self.pending_request = self.loop.create_task(self.response_queue.get())
|
||||
self.transport.write(msg.encode())
|
||||
self.pending_request = self.loop.create_task(asyncio.wait_for(self.response_queue.get(), timeout))
|
||||
return await self.pending_request
|
||||
except (AttributeError, asyncio.CancelledError) as err:
|
||||
# attribute error happens when we transport.write after disconnect
|
||||
# cancelled error happens when the pending_request task is cancelled by a disconnect
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
raise err if isinstance(err, asyncio.CancelledError) else asyncio.CancelledError()
|
||||
finally:
|
||||
self.pending_request = None
|
||||
|
||||
@@ -86,8 +94,16 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
needed = response.get('needed_blobs', [])
|
||||
sent_sd = False
|
||||
if response['send_sd_blob']:
|
||||
await sd_blob.sendfile(self)
|
||||
received = await self.response_queue.get()
|
||||
try:
|
||||
sent = await sd_blob.sendfile(self)
|
||||
if sent == -1:
|
||||
log.warning("failed to send sd blob")
|
||||
raise asyncio.CancelledError()
|
||||
received = await asyncio.wait_for(self.response_queue.get(), 30)
|
||||
except asyncio.CancelledError as err:
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
raise err
|
||||
if received.get('received_sd_blob'):
|
||||
sent_sd = True
|
||||
if not needed:
|
||||
@@ -110,8 +126,16 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
if 'send_blob' not in response:
|
||||
raise ValueError("I don't know whether to send the blob or not!")
|
||||
if response['send_blob']:
|
||||
await blob.sendfile(self)
|
||||
received = await self.response_queue.get()
|
||||
try:
|
||||
sent = await blob.sendfile(self)
|
||||
if sent == -1:
|
||||
log.warning("failed to send blob")
|
||||
raise asyncio.CancelledError()
|
||||
received = await asyncio.wait_for(self.response_queue.get(), 30)
|
||||
except asyncio.CancelledError as err:
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
raise err
|
||||
if received.get('received_blob'):
|
||||
self.reflected_blobs.append(blob.blob_hash)
|
||||
log.info("Sent reflector blob %s", blob.blob_hash[:8])
|
||||
|
||||
@@ -15,7 +15,9 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ReflectorServerProtocol(asyncio.Protocol):
|
||||
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000):
|
||||
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000,
|
||||
stop_event: asyncio.Event = None, incoming_event: asyncio.Event = None,
|
||||
not_incoming_event: asyncio.Event = None, partial_event: asyncio.Event = None):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
@@ -27,11 +29,26 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
self.descriptor: typing.Optional['StreamDescriptor'] = None
|
||||
self.sd_blob: typing.Optional['BlobFile'] = None
|
||||
self.received = []
|
||||
self.incoming = asyncio.Event(loop=self.loop)
|
||||
self.incoming = incoming_event or asyncio.Event(loop=self.loop)
|
||||
self.not_incoming = not_incoming_event or asyncio.Event(loop=self.loop)
|
||||
self.stop_event = stop_event or asyncio.Event(loop=self.loop)
|
||||
self.chunk_size = response_chunk_size
|
||||
self.wait_for_stop_task: typing.Optional[asyncio.Task] = None
|
||||
self.partial_event = partial_event
|
||||
|
||||
async def wait_for_stop(self):
|
||||
await self.stop_event.wait()
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
|
||||
def connection_made(self, transport):
|
||||
self.transport = transport
|
||||
self.wait_for_stop_task = self.loop.create_task(self.wait_for_stop())
|
||||
|
||||
def connection_lost(self, exc):
|
||||
if self.wait_for_stop_task:
|
||||
self.wait_for_stop_task.cancel()
|
||||
self.wait_for_stop_task = None
|
||||
|
||||
def data_received(self, data: bytes):
|
||||
if self.incoming.is_set():
|
||||
@@ -73,6 +90,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
self.sd_blob = self.blob_manager.get_blob(request['sd_blob_hash'], request['sd_blob_size'])
|
||||
if not self.sd_blob.get_is_verified():
|
||||
self.writer = self.sd_blob.get_blob_writer(self.transport.get_extra_info('peername'))
|
||||
self.not_incoming.clear()
|
||||
self.incoming.set()
|
||||
self.send_response({"send_sd_blob": True})
|
||||
try:
|
||||
@@ -86,6 +104,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
self.transport.close()
|
||||
finally:
|
||||
self.incoming.clear()
|
||||
self.not_incoming.set()
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
else:
|
||||
@@ -93,13 +112,18 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
self.loop, self.blob_manager.blob_dir, self.sd_blob
|
||||
)
|
||||
self.incoming.clear()
|
||||
self.not_incoming.set()
|
||||
if self.writer:
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
self.send_response({"send_sd_blob": False, 'needed': [
|
||||
blob.blob_hash for blob in self.descriptor.blobs[:-1]
|
||||
if not self.blob_manager.get_blob(blob.blob_hash).get_is_verified()
|
||||
]})
|
||||
|
||||
needs = [blob.blob_hash
|
||||
for blob in self.descriptor.blobs[:-1]
|
||||
if not self.blob_manager.get_blob(blob.blob_hash).get_is_verified()]
|
||||
if needs and not self.partial_event.is_set():
|
||||
needs = needs[:3]
|
||||
self.partial_event.set()
|
||||
self.send_response({"send_sd_blob": False, 'needed_blobs': needs})
|
||||
return
|
||||
return
|
||||
elif self.descriptor:
|
||||
@@ -112,6 +136,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
blob = self.blob_manager.get_blob(request['blob_hash'], request['blob_size'])
|
||||
if not blob.get_is_verified():
|
||||
self.writer = blob.get_blob_writer(self.transport.get_extra_info('peername'))
|
||||
self.not_incoming.clear()
|
||||
self.incoming.set()
|
||||
self.send_response({"send_blob": True})
|
||||
try:
|
||||
@@ -120,6 +145,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
except asyncio.TimeoutError:
|
||||
self.send_response({"received_blob": False})
|
||||
self.incoming.clear()
|
||||
self.not_incoming.set()
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
else:
|
||||
@@ -130,26 +156,39 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
|
||||
|
||||
class ReflectorServer:
|
||||
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000):
|
||||
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000,
|
||||
stop_event: asyncio.Event = None, incoming_event: asyncio.Event = None,
|
||||
not_incoming_event: asyncio.Event = None, partial_needs=False):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: typing.Optional[asyncio.Task] = None
|
||||
self.started_listening = asyncio.Event(loop=self.loop)
|
||||
self.stopped_listening = asyncio.Event(loop=self.loop)
|
||||
self.incoming_event = incoming_event or asyncio.Event(loop=self.loop)
|
||||
self.not_incoming_event = not_incoming_event or asyncio.Event(loop=self.loop)
|
||||
self.response_chunk_size = response_chunk_size
|
||||
self.stop_event = stop_event
|
||||
self.partial_needs = partial_needs # for testing cases where it doesn't know what it wants
|
||||
|
||||
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
|
||||
if self.server_task is not None:
|
||||
raise Exception("already running")
|
||||
|
||||
async def _start_server():
|
||||
server = await self.loop.create_server(
|
||||
lambda: ReflectorServerProtocol(self.blob_manager, self.response_chunk_size),
|
||||
interface, port
|
||||
)
|
||||
partial_event = asyncio.Event()
|
||||
if not self.partial_needs:
|
||||
partial_event.set()
|
||||
server = await self.loop.create_server(lambda: ReflectorServerProtocol(
|
||||
self.blob_manager, self.response_chunk_size, self.stop_event, self.incoming_event,
|
||||
self.not_incoming_event, partial_event), interface, port)
|
||||
self.started_listening.set()
|
||||
self.stopped_listening.clear()
|
||||
log.info("Reflector server listening on TCP %s:%i", interface, port)
|
||||
async with server:
|
||||
await server.serve_forever()
|
||||
try:
|
||||
async with server:
|
||||
await server.serve_forever()
|
||||
finally:
|
||||
self.stopped_listening.set()
|
||||
|
||||
self.server_task = self.loop.create_task(_start_server())
|
||||
|
||||
|
||||
+104
-323
@@ -6,83 +6,67 @@ import random
|
||||
import typing
|
||||
from typing import Optional
|
||||
from aiohttp.web import Request
|
||||
from lbry.error import ResolveError, InvalidStreamDescriptorError, DownloadSDTimeoutError, InsufficientFundsError
|
||||
from lbry.error import ResolveTimeoutError, DownloadDataTimeoutError, KeyFeeAboveMaxAllowedError
|
||||
from lbry.utils import cache_concurrent
|
||||
from lbry.error import InvalidStreamDescriptorError
|
||||
from lbry.file.source_manager import SourceManager
|
||||
from lbry.stream.descriptor import StreamDescriptor
|
||||
from lbry.stream.managed_stream import ManagedStream
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.schema.url import URL
|
||||
from lbry.wallet.dewies import dewies_to_lbc
|
||||
from lbry.wallet import Output
|
||||
|
||||
from lbry.file.source import ManagedDownloadSource
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.blob.blob_manager import BlobManager
|
||||
from lbry.dht.node import Node
|
||||
from lbry.wallet.wallet import WalletManager
|
||||
from lbry.wallet.transaction import Transaction
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
FILTER_FIELDS = [
|
||||
'rowid',
|
||||
'status',
|
||||
'file_name',
|
||||
'added_on',
|
||||
'sd_hash',
|
||||
'stream_hash',
|
||||
'claim_name',
|
||||
'claim_height',
|
||||
'claim_id',
|
||||
'outpoint',
|
||||
'txid',
|
||||
'nout',
|
||||
'channel_claim_id',
|
||||
'channel_name',
|
||||
'full_status', # TODO: remove
|
||||
'blobs_remaining',
|
||||
'blobs_in_stream'
|
||||
]
|
||||
|
||||
COMPARISON_OPERATORS = {
|
||||
'eq': lambda a, b: a == b,
|
||||
'ne': lambda a, b: a != b,
|
||||
'g': lambda a, b: a > b,
|
||||
'l': lambda a, b: a < b,
|
||||
'ge': lambda a, b: a >= b,
|
||||
'le': lambda a, b: a <= b,
|
||||
}
|
||||
|
||||
|
||||
def path_or_none(path) -> Optional[str]:
|
||||
if not path:
|
||||
def path_or_none(encoded_path) -> Optional[str]:
|
||||
if not encoded_path:
|
||||
return
|
||||
return binascii.unhexlify(path).decode()
|
||||
return binascii.unhexlify(encoded_path).decode()
|
||||
|
||||
|
||||
class StreamManager:
|
||||
class StreamManager(SourceManager):
|
||||
_sources: typing.Dict[str, ManagedStream]
|
||||
|
||||
filter_fields = SourceManager.filter_fields
|
||||
filter_fields.update({
|
||||
'sd_hash',
|
||||
'stream_hash',
|
||||
'full_status', # TODO: remove
|
||||
'blobs_remaining',
|
||||
'blobs_in_stream',
|
||||
'uploading_to_reflector',
|
||||
'is_fully_reflected'
|
||||
})
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
wallet_manager: 'WalletManager', storage: 'SQLiteStorage', node: Optional['Node'],
|
||||
analytics_manager: Optional['AnalyticsManager'] = None):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
super().__init__(loop, config, storage, analytics_manager)
|
||||
self.blob_manager = blob_manager
|
||||
self.wallet_manager = wallet_manager
|
||||
self.storage = storage
|
||||
self.node = node
|
||||
self.analytics_manager = analytics_manager
|
||||
self.streams: typing.Dict[str, ManagedStream] = {}
|
||||
self.resume_saving_task: Optional[asyncio.Task] = None
|
||||
self.re_reflect_task: Optional[asyncio.Task] = None
|
||||
self.update_stream_finished_futs: typing.List[asyncio.Future] = []
|
||||
self.running_reflector_uploads: typing.Dict[str, asyncio.Task] = {}
|
||||
self.started = asyncio.Event(loop=self.loop)
|
||||
|
||||
@property
|
||||
def streams(self):
|
||||
return self._sources
|
||||
|
||||
def add(self, source: ManagedStream):
|
||||
super().add(source)
|
||||
self.storage.content_claim_callbacks[source.stream_hash] = lambda: self._update_content_claim(source)
|
||||
|
||||
async def _update_content_claim(self, stream: ManagedStream):
|
||||
claim_info = await self.storage.get_content_claim(stream.stream_hash)
|
||||
self.streams.setdefault(stream.sd_hash, stream).set_claim(claim_info, claim_info['value'])
|
||||
self._sources.setdefault(stream.sd_hash, stream).set_claim(claim_info, claim_info['value'])
|
||||
|
||||
async def recover_streams(self, file_infos: typing.List[typing.Dict]):
|
||||
to_restore = []
|
||||
@@ -113,10 +97,10 @@ class StreamManager:
|
||||
# if self.blob_manager._save_blobs:
|
||||
# log.info("Recovered %i/%i attempted streams", len(to_restore), len(file_infos))
|
||||
|
||||
async def add_stream(self, rowid: int, sd_hash: str, file_name: Optional[str],
|
||||
download_directory: Optional[str], status: str,
|
||||
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
|
||||
added_on: Optional[int], fully_reflected: bool):
|
||||
async def _load_stream(self, rowid: int, sd_hash: str, file_name: Optional[str],
|
||||
download_directory: Optional[str], status: str,
|
||||
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
|
||||
added_on: Optional[int], fully_reflected: Optional[bool]):
|
||||
try:
|
||||
descriptor = await self.blob_manager.get_stream_descriptor(sd_hash)
|
||||
except InvalidStreamDescriptorError as err:
|
||||
@@ -129,10 +113,9 @@ class StreamManager:
|
||||
)
|
||||
if fully_reflected:
|
||||
stream.fully_reflected.set()
|
||||
self.streams[sd_hash] = stream
|
||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||
self.add(stream)
|
||||
|
||||
async def load_and_resume_streams_from_database(self):
|
||||
async def initialize_from_database(self):
|
||||
to_recover = []
|
||||
to_start = []
|
||||
|
||||
@@ -146,7 +129,6 @@ class StreamManager:
|
||||
to_recover.append(file_info)
|
||||
to_start.append(file_info)
|
||||
if to_recover:
|
||||
log.info("Recover %i files", len(to_recover))
|
||||
await self.recover_streams(to_recover)
|
||||
|
||||
log.info("Initializing %i files", len(to_start))
|
||||
@@ -157,7 +139,7 @@ class StreamManager:
|
||||
download_directory = path_or_none(file_info['download_directory'])
|
||||
if file_name and download_directory and not file_info['saved_file'] and file_info['status'] == 'running':
|
||||
to_resume_saving.append((file_name, download_directory, file_info['sd_hash']))
|
||||
add_stream_tasks.append(self.loop.create_task(self.add_stream(
|
||||
add_stream_tasks.append(self.loop.create_task(self._load_stream(
|
||||
file_info['rowid'], file_info['sd_hash'], file_name,
|
||||
download_directory, file_info['status'],
|
||||
file_info['claim'], file_info['content_fee'],
|
||||
@@ -165,25 +147,32 @@ class StreamManager:
|
||||
)))
|
||||
if add_stream_tasks:
|
||||
await asyncio.gather(*add_stream_tasks, loop=self.loop)
|
||||
log.info("Started stream manager with %i files", len(self.streams))
|
||||
log.info("Started stream manager with %i files", len(self._sources))
|
||||
if not self.node:
|
||||
log.info("no DHT node given, resuming downloads trusting that we can contact reflector")
|
||||
if to_resume_saving:
|
||||
self.resume_saving_task = self.loop.create_task(self.resume(to_resume_saving))
|
||||
|
||||
async def resume(self, to_resume_saving):
|
||||
log.info("Resuming saving %i files", len(to_resume_saving))
|
||||
await asyncio.gather(
|
||||
*(self.streams[sd_hash].save_file(file_name, download_directory, node=self.node)
|
||||
for (file_name, download_directory, sd_hash) in to_resume_saving),
|
||||
loop=self.loop
|
||||
)
|
||||
log.info("Resuming saving %i files", len(to_resume_saving))
|
||||
self.resume_saving_task = asyncio.ensure_future(asyncio.gather(
|
||||
*(self._sources[sd_hash].save_file(file_name, download_directory)
|
||||
for (file_name, download_directory, sd_hash) in to_resume_saving),
|
||||
loop=self.loop
|
||||
))
|
||||
|
||||
async def reflect_streams(self):
|
||||
try:
|
||||
return await self._reflect_streams()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
log.exception("reflector task encountered an unexpected error!")
|
||||
|
||||
async def _reflect_streams(self):
|
||||
# todo: those debug statements are temporary for #2987 - remove them if its closed
|
||||
while True:
|
||||
if self.config.reflect_streams and self.config.reflector_servers:
|
||||
log.debug("collecting streams to reflect")
|
||||
sd_hashes = await self.storage.get_streams_to_re_reflect()
|
||||
sd_hashes = [sd for sd in sd_hashes if sd in self.streams]
|
||||
sd_hashes = [sd for sd in sd_hashes if sd in self._sources]
|
||||
batch = []
|
||||
while sd_hashes:
|
||||
stream = self.streams[sd_hashes.pop()]
|
||||
@@ -192,25 +181,26 @@ class StreamManager:
|
||||
stream.fully_reflected.is_set():
|
||||
batch.append(self.reflect_stream(stream))
|
||||
if len(batch) >= self.config.concurrent_reflector_uploads:
|
||||
log.debug("waiting for batch of %s reflecting streams", len(batch))
|
||||
await asyncio.gather(*batch, loop=self.loop)
|
||||
log.debug("done processing %s streams", len(batch))
|
||||
batch = []
|
||||
if batch:
|
||||
log.debug("waiting for batch of %s reflecting streams", len(batch))
|
||||
await asyncio.gather(*batch, loop=self.loop)
|
||||
log.debug("done processing %s streams", len(batch))
|
||||
await asyncio.sleep(300, loop=self.loop)
|
||||
|
||||
async def start(self):
|
||||
await self.load_and_resume_streams_from_database()
|
||||
await super().start()
|
||||
self.re_reflect_task = self.loop.create_task(self.reflect_streams())
|
||||
self.started.set()
|
||||
|
||||
def stop(self):
|
||||
super().stop()
|
||||
if self.resume_saving_task and not self.resume_saving_task.done():
|
||||
self.resume_saving_task.cancel()
|
||||
if self.re_reflect_task and not self.re_reflect_task.done():
|
||||
self.re_reflect_task.cancel()
|
||||
while self.streams:
|
||||
_, stream = self.streams.popitem()
|
||||
stream.stop_tasks()
|
||||
while self.update_stream_finished_futs:
|
||||
self.update_stream_finished_futs.pop().cancel()
|
||||
while self.running_reflector_uploads:
|
||||
@@ -225,7 +215,7 @@ class StreamManager:
|
||||
server, port = random.choice(self.config.reflector_servers)
|
||||
if stream.sd_hash in self.running_reflector_uploads:
|
||||
return self.running_reflector_uploads[stream.sd_hash]
|
||||
task = self.loop.create_task(stream.upload_to_reflector(server, port))
|
||||
task = self.loop.create_task(self._retriable_reflect_stream(stream, server, port))
|
||||
self.running_reflector_uploads[stream.sd_hash] = task
|
||||
task.add_done_callback(
|
||||
lambda _: None if stream.sd_hash not in self.running_reflector_uploads else
|
||||
@@ -233,261 +223,52 @@ class StreamManager:
|
||||
)
|
||||
return task
|
||||
|
||||
async def create_stream(self, file_path: str, key: Optional[bytes] = None,
|
||||
iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
|
||||
stream = await ManagedStream.create(self.loop, self.config, self.blob_manager, file_path, key, iv_generator)
|
||||
async def _retriable_reflect_stream(self, stream, host, port):
|
||||
sent = await stream.upload_to_reflector(host, port)
|
||||
while not stream.is_fully_reflected and stream.reflector_progress > 0 and len(sent) > 0:
|
||||
stream.reflector_progress = 0
|
||||
sent = await stream.upload_to_reflector(host, port)
|
||||
return sent
|
||||
|
||||
async def create(self, file_path: str, key: Optional[bytes] = None,
|
||||
iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
|
||||
descriptor = await StreamDescriptor.create_stream(
|
||||
self.loop, self.blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
|
||||
blob_completed_callback=self.blob_manager.blob_completed
|
||||
)
|
||||
await self.storage.store_stream(
|
||||
self.blob_manager.get_blob(descriptor.sd_hash, is_mine=True), descriptor
|
||||
)
|
||||
row_id = await self.storage.save_published_file(
|
||||
descriptor.stream_hash, os.path.basename(file_path), os.path.dirname(file_path), 0
|
||||
)
|
||||
stream = ManagedStream(
|
||||
self.loop, self.config, self.blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
|
||||
os.path.basename(file_path), status=ManagedDownloadSource.STATUS_FINISHED,
|
||||
rowid=row_id, descriptor=descriptor
|
||||
)
|
||||
self.streams[stream.sd_hash] = stream
|
||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||
if self.config.reflect_streams and self.config.reflector_servers:
|
||||
self.reflect_stream(stream)
|
||||
return stream
|
||||
|
||||
async def delete_stream(self, stream: ManagedStream, delete_file: Optional[bool] = False):
|
||||
if stream.sd_hash in self.running_reflector_uploads:
|
||||
self.running_reflector_uploads[stream.sd_hash].cancel()
|
||||
stream.stop_tasks()
|
||||
if stream.sd_hash in self.streams:
|
||||
del self.streams[stream.sd_hash]
|
||||
blob_hashes = [stream.sd_hash] + [b.blob_hash for b in stream.descriptor.blobs[:-1]]
|
||||
async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
|
||||
if not isinstance(source, ManagedStream):
|
||||
return
|
||||
if source.identifier in self.running_reflector_uploads:
|
||||
self.running_reflector_uploads[source.identifier].cancel()
|
||||
source.stop_tasks()
|
||||
if source.identifier in self.streams:
|
||||
del self.streams[source.identifier]
|
||||
blob_hashes = [source.identifier] + [b.blob_hash for b in source.descriptor.blobs[:-1]]
|
||||
await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False)
|
||||
await self.storage.delete_stream(stream.descriptor)
|
||||
if delete_file and stream.output_file_exists:
|
||||
os.remove(stream.full_path)
|
||||
|
||||
def get_stream_by_stream_hash(self, stream_hash: str) -> Optional[ManagedStream]:
|
||||
streams = tuple(filter(lambda stream: stream.stream_hash == stream_hash, self.streams.values()))
|
||||
if streams:
|
||||
return streams[0]
|
||||
|
||||
def get_filtered_streams(self, sort_by: Optional[str] = None, reverse: Optional[bool] = False,
|
||||
comparison: Optional[str] = None,
|
||||
**search_by) -> typing.List[ManagedStream]:
|
||||
"""
|
||||
Get a list of filtered and sorted ManagedStream objects
|
||||
|
||||
:param sort_by: field to sort by
|
||||
:param reverse: reverse sorting
|
||||
:param comparison: comparison operator used for filtering
|
||||
:param search_by: fields and values to filter by
|
||||
"""
|
||||
if sort_by and sort_by not in FILTER_FIELDS:
|
||||
raise ValueError(f"'{sort_by}' is not a valid field to sort by")
|
||||
if comparison and comparison not in COMPARISON_OPERATORS:
|
||||
raise ValueError(f"'{comparison}' is not a valid comparison")
|
||||
if 'full_status' in search_by:
|
||||
del search_by['full_status']
|
||||
for search in search_by:
|
||||
if search not in FILTER_FIELDS:
|
||||
raise ValueError(f"'{search}' is not a valid search operation")
|
||||
if search_by:
|
||||
comparison = comparison or 'eq'
|
||||
streams = []
|
||||
for stream in self.streams.values():
|
||||
for search, val in search_by.items():
|
||||
if COMPARISON_OPERATORS[comparison](getattr(stream, search), val):
|
||||
streams.append(stream)
|
||||
break
|
||||
else:
|
||||
streams = list(self.streams.values())
|
||||
if sort_by:
|
||||
streams.sort(key=lambda s: getattr(s, sort_by))
|
||||
if reverse:
|
||||
streams.reverse()
|
||||
return streams
|
||||
|
||||
async def _check_update_or_replace(self, outpoint: str, claim_id: str, claim: Claim
|
||||
) -> typing.Tuple[Optional[ManagedStream], Optional[ManagedStream]]:
|
||||
existing = self.get_filtered_streams(outpoint=outpoint)
|
||||
if existing:
|
||||
return existing[0], None
|
||||
existing = self.get_filtered_streams(sd_hash=claim.stream.source.sd_hash)
|
||||
if existing and existing[0].claim_id != claim_id:
|
||||
raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {claim_id}")
|
||||
if existing:
|
||||
log.info("claim contains a metadata only update to a stream we have")
|
||||
await self.storage.save_content_claim(
|
||||
existing[0].stream_hash, outpoint
|
||||
)
|
||||
await self._update_content_claim(existing[0])
|
||||
return existing[0], None
|
||||
else:
|
||||
existing_for_claim_id = self.get_filtered_streams(claim_id=claim_id)
|
||||
if existing_for_claim_id:
|
||||
log.info("claim contains an update to a stream we have, downloading it")
|
||||
return None, existing_for_claim_id[0]
|
||||
return None, None
|
||||
|
||||
@staticmethod
|
||||
def _convert_to_old_resolve_output(wallet_manager, resolves):
|
||||
result = {}
|
||||
for url, txo in resolves.items():
|
||||
if isinstance(txo, Output):
|
||||
tx_height = txo.tx_ref.height
|
||||
best_height = wallet_manager.ledger.headers.height
|
||||
result[url] = {
|
||||
'name': txo.claim_name,
|
||||
'value': txo.claim,
|
||||
'protobuf': binascii.hexlify(txo.claim.to_bytes()),
|
||||
'claim_id': txo.claim_id,
|
||||
'txid': txo.tx_ref.id,
|
||||
'nout': txo.position,
|
||||
'amount': dewies_to_lbc(txo.amount),
|
||||
'effective_amount': txo.meta.get('effective_amount', 0),
|
||||
'height': tx_height,
|
||||
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
|
||||
'claim_sequence': -1,
|
||||
'address': txo.get_address(wallet_manager.ledger),
|
||||
'valid_at_height': txo.meta.get('activation_height', None),
|
||||
'timestamp': wallet_manager.ledger.headers[tx_height]['timestamp'],
|
||||
'supports': []
|
||||
}
|
||||
else:
|
||||
result[url] = txo
|
||||
return result
|
||||
|
||||
@cache_concurrent
|
||||
async def download_stream_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
|
||||
timeout: Optional[float] = None,
|
||||
file_name: Optional[str] = None,
|
||||
download_directory: Optional[str] = None,
|
||||
save_file: Optional[bool] = None,
|
||||
resolve_timeout: float = 3.0,
|
||||
wallet: Optional['Wallet'] = None) -> ManagedStream:
|
||||
manager = self.wallet_manager
|
||||
wallet = wallet or manager.default_wallet
|
||||
timeout = timeout or self.config.download_timeout
|
||||
start_time = self.loop.time()
|
||||
resolved_time = None
|
||||
stream = None
|
||||
txo: Optional[Output] = None
|
||||
error = None
|
||||
outpoint = None
|
||||
if save_file is None:
|
||||
save_file = self.config.save_files
|
||||
if file_name and not save_file:
|
||||
save_file = True
|
||||
if save_file:
|
||||
download_directory = download_directory or self.config.download_dir
|
||||
else:
|
||||
download_directory = None
|
||||
|
||||
payment = None
|
||||
try:
|
||||
# resolve the claim
|
||||
if not URL.parse(uri).has_stream:
|
||||
raise ResolveError("cannot download a channel claim, specify a /path")
|
||||
try:
|
||||
response = await asyncio.wait_for(
|
||||
manager.ledger.resolve(wallet.accounts, [uri]),
|
||||
resolve_timeout
|
||||
)
|
||||
resolved_result = self._convert_to_old_resolve_output(manager, response)
|
||||
except asyncio.TimeoutError:
|
||||
raise ResolveTimeoutError(uri)
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
|
||||
raise
|
||||
log.exception("Unexpected error resolving stream:")
|
||||
raise ResolveError(f"Unexpected error resolving stream: {str(err)}")
|
||||
await self.storage.save_claims_for_resolve([
|
||||
value for value in resolved_result.values() if 'error' not in value
|
||||
])
|
||||
resolved = resolved_result.get(uri, {})
|
||||
resolved = resolved if 'value' in resolved else resolved.get('claim')
|
||||
if not resolved:
|
||||
raise ResolveError(f"Failed to resolve stream at '{uri}'")
|
||||
if 'error' in resolved:
|
||||
raise ResolveError(f"error resolving stream: {resolved['error']}")
|
||||
txo = response[uri]
|
||||
|
||||
claim = Claim.from_bytes(binascii.unhexlify(resolved['protobuf']))
|
||||
outpoint = f"{resolved['txid']}:{resolved['nout']}"
|
||||
resolved_time = self.loop.time() - start_time
|
||||
|
||||
# resume or update an existing stream, if the stream changed: download it and delete the old one after
|
||||
updated_stream, to_replace = await self._check_update_or_replace(outpoint, resolved['claim_id'], claim)
|
||||
if updated_stream:
|
||||
log.info("already have stream for %s", uri)
|
||||
if save_file and updated_stream.output_file_exists:
|
||||
save_file = False
|
||||
await updated_stream.start(node=self.node, timeout=timeout, save_now=save_file)
|
||||
if not updated_stream.output_file_exists and (save_file or file_name or download_directory):
|
||||
await updated_stream.save_file(
|
||||
file_name=file_name, download_directory=download_directory, node=self.node
|
||||
)
|
||||
return updated_stream
|
||||
|
||||
if not to_replace and txo.has_price and not txo.purchase_receipt:
|
||||
payment = await manager.create_purchase_transaction(
|
||||
wallet.accounts, txo, exchange_rate_manager
|
||||
)
|
||||
|
||||
stream = ManagedStream(
|
||||
self.loop, self.config, self.blob_manager, claim.stream.source.sd_hash, download_directory,
|
||||
file_name, ManagedStream.STATUS_RUNNING, content_fee=payment,
|
||||
analytics_manager=self.analytics_manager
|
||||
)
|
||||
log.info("starting download for %s", uri)
|
||||
|
||||
before_download = self.loop.time()
|
||||
await stream.start(self.node, timeout)
|
||||
stream.set_claim(resolved, claim)
|
||||
if to_replace: # delete old stream now that the replacement has started downloading
|
||||
await self.delete_stream(to_replace)
|
||||
|
||||
if payment is not None:
|
||||
await manager.broadcast_or_release(payment)
|
||||
payment = None # to avoid releasing in `finally` later
|
||||
log.info("paid fee of %s for %s", dewies_to_lbc(stream.content_fee.outputs[0].amount), uri)
|
||||
await self.storage.save_content_fee(stream.stream_hash, stream.content_fee)
|
||||
|
||||
self.streams[stream.sd_hash] = stream
|
||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||
await self.storage.save_content_claim(stream.stream_hash, outpoint)
|
||||
if save_file:
|
||||
await asyncio.wait_for(stream.save_file(node=self.node), timeout - (self.loop.time() - before_download),
|
||||
loop=self.loop)
|
||||
return stream
|
||||
except asyncio.TimeoutError:
|
||||
error = DownloadDataTimeoutError(stream.sd_hash)
|
||||
raise error
|
||||
except Exception as err: # forgive data timeout, don't delete stream
|
||||
expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError,
|
||||
KeyFeeAboveMaxAllowedError)
|
||||
if isinstance(err, expected):
|
||||
log.warning("Failed to download %s: %s", uri, str(err))
|
||||
elif isinstance(err, asyncio.CancelledError):
|
||||
pass
|
||||
else:
|
||||
log.exception("Unexpected error downloading stream:")
|
||||
error = err
|
||||
raise
|
||||
finally:
|
||||
if payment is not None:
|
||||
# payment is set to None after broadcasting, if we're here an exception probably happened
|
||||
await manager.ledger.release_tx(payment)
|
||||
if self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or
|
||||
stream.downloader.time_to_first_bytes))):
|
||||
server = self.wallet_manager.ledger.network.client.server
|
||||
self.loop.create_task(
|
||||
self.analytics_manager.send_time_to_first_bytes(
|
||||
resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id,
|
||||
uri, outpoint,
|
||||
None if not stream else len(stream.downloader.blob_downloader.active_connections),
|
||||
None if not stream else len(stream.downloader.blob_downloader.scores),
|
||||
None if not stream else len(stream.downloader.blob_downloader.connection_failures),
|
||||
False if not stream else stream.downloader.added_fixed_peers,
|
||||
self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay,
|
||||
None if not stream else stream.sd_hash,
|
||||
None if not stream else stream.downloader.time_to_descriptor,
|
||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
|
||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
|
||||
None if not stream else stream.downloader.time_to_first_bytes,
|
||||
None if not error else error.__class__.__name__,
|
||||
None if not error else str(error),
|
||||
None if not server else f"{server[0]}:{server[1]}"
|
||||
)
|
||||
)
|
||||
await self.storage.delete_stream(source.descriptor)
|
||||
if delete_file and source.output_file_exists:
|
||||
os.remove(source.full_path)
|
||||
|
||||
async def stream_partial_content(self, request: Request, sd_hash: str):
|
||||
return await self.streams[sd_hash].stream_file(request, self.node)
|
||||
stream = self._sources[sd_hash]
|
||||
if not stream.downloader.node:
|
||||
stream.downloader.node = self.node
|
||||
return await stream.stream_file(request)
|
||||
+202
-45
@@ -17,18 +17,21 @@ from functools import partial
|
||||
from lbry.wallet import WalletManager, Wallet, Ledger, Account, Transaction
|
||||
from lbry.conf import Config
|
||||
from lbry.wallet.util import satoshis_to_coins
|
||||
from lbry.wallet.dewies import lbc_to_dewies
|
||||
from lbry.wallet.orchstr8 import Conductor
|
||||
from lbry.wallet.orchstr8.node import BlockchainNode, WalletNode
|
||||
from lbry.wallet.orchstr8.node import LBCWalletNode, WalletNode, HubNode
|
||||
from scribe.schema.claim import Claim
|
||||
|
||||
from lbry.extras.daemon.daemon import Daemon, jsonrpc_dumps_pretty
|
||||
from lbry.extras.daemon.components import Component, WalletComponent
|
||||
from lbry.extras.daemon.components import (
|
||||
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
DHT_COMPONENT,
|
||||
HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, LIBTORRENT_COMPONENT
|
||||
)
|
||||
from lbry.extras.daemon.componentmanager import ComponentManager
|
||||
from lbry.extras.daemon.exchange_rate_manager import (
|
||||
ExchangeRateManager, ExchangeRate, LBRYFeed, LBRYBTCFeed
|
||||
ExchangeRateManager, ExchangeRate, BittrexBTCFeed, BittrexUSDFeed
|
||||
)
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from lbry.blob.blob_manager import BlobManager
|
||||
@@ -83,6 +86,7 @@ class AsyncioTestCase(unittest.TestCase):
|
||||
# https://bugs.python.org/issue32972
|
||||
|
||||
LOOP_SLOW_CALLBACK_DURATION = 0.2
|
||||
TIMEOUT = 120.0
|
||||
|
||||
maxDiff = None
|
||||
|
||||
@@ -130,15 +134,18 @@ class AsyncioTestCase(unittest.TestCase):
|
||||
|
||||
with outcome.testPartExecutor(self):
|
||||
self.setUp()
|
||||
self.add_timeout()
|
||||
self.loop.run_until_complete(self.asyncSetUp())
|
||||
if outcome.success:
|
||||
outcome.expecting_failure = expecting_failure
|
||||
with outcome.testPartExecutor(self, isTest=True):
|
||||
maybe_coroutine = testMethod()
|
||||
if asyncio.iscoroutine(maybe_coroutine):
|
||||
self.add_timeout()
|
||||
self.loop.run_until_complete(maybe_coroutine)
|
||||
outcome.expecting_failure = False
|
||||
with outcome.testPartExecutor(self):
|
||||
self.add_timeout()
|
||||
self.loop.run_until_complete(self.asyncTearDown())
|
||||
self.tearDown()
|
||||
|
||||
@@ -186,8 +193,19 @@ class AsyncioTestCase(unittest.TestCase):
|
||||
with outcome.testPartExecutor(self):
|
||||
maybe_coroutine = function(*args, **kwargs)
|
||||
if asyncio.iscoroutine(maybe_coroutine):
|
||||
self.add_timeout()
|
||||
self.loop.run_until_complete(maybe_coroutine)
|
||||
|
||||
def cancel(self):
|
||||
for task in asyncio.all_tasks(self.loop):
|
||||
if not task.done():
|
||||
task.print_stack()
|
||||
task.cancel()
|
||||
|
||||
def add_timeout(self):
|
||||
if self.TIMEOUT:
|
||||
self.loop.call_later(self.TIMEOUT, self.cancel)
|
||||
|
||||
|
||||
class AdvanceTimeTestCase(AsyncioTestCase):
|
||||
|
||||
@@ -212,7 +230,8 @@ class IntegrationTestCase(AsyncioTestCase):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.conductor: Optional[Conductor] = None
|
||||
self.blockchain: Optional[BlockchainNode] = None
|
||||
self.blockchain: Optional[LBCWalletNode] = None
|
||||
self.hub: Optional[HubNode] = None
|
||||
self.wallet_node: Optional[WalletNode] = None
|
||||
self.manager: Optional[WalletManager] = None
|
||||
self.ledger: Optional[Ledger] = None
|
||||
@@ -221,13 +240,18 @@ class IntegrationTestCase(AsyncioTestCase):
|
||||
|
||||
async def asyncSetUp(self):
|
||||
self.conductor = Conductor(seed=self.SEED)
|
||||
await self.conductor.start_blockchain()
|
||||
self.addCleanup(self.conductor.stop_blockchain)
|
||||
await self.conductor.start_lbcd()
|
||||
self.addCleanup(self.conductor.stop_lbcd)
|
||||
await self.conductor.start_lbcwallet()
|
||||
self.addCleanup(self.conductor.stop_lbcwallet)
|
||||
await self.conductor.start_spv()
|
||||
self.addCleanup(self.conductor.stop_spv)
|
||||
await self.conductor.start_wallet()
|
||||
self.addCleanup(self.conductor.stop_wallet)
|
||||
self.blockchain = self.conductor.blockchain_node
|
||||
await self.conductor.start_hub()
|
||||
self.addCleanup(self.conductor.stop_hub)
|
||||
self.blockchain = self.conductor.lbcwallet_node
|
||||
self.hub = self.conductor.hub_node
|
||||
self.wallet_node = self.conductor.wallet_node
|
||||
self.manager = self.wallet_node.manager
|
||||
self.ledger = self.wallet_node.ledger
|
||||
@@ -241,6 +265,13 @@ class IntegrationTestCase(AsyncioTestCase):
|
||||
def broadcast(self, tx):
|
||||
return self.ledger.broadcast(tx)
|
||||
|
||||
async def broadcast_and_confirm(self, tx, ledger=None):
|
||||
ledger = ledger or self.ledger
|
||||
notifications = asyncio.create_task(ledger.wait(tx))
|
||||
await ledger.broadcast(tx)
|
||||
await notifications
|
||||
await self.generate_and_wait(1, [tx.id], ledger)
|
||||
|
||||
async def on_header(self, height):
|
||||
if self.ledger.headers.height < height:
|
||||
await self.ledger.on_header.where(
|
||||
@@ -248,11 +279,36 @@ class IntegrationTestCase(AsyncioTestCase):
|
||||
)
|
||||
return True
|
||||
|
||||
def on_transaction_id(self, txid, ledger=None):
|
||||
return (ledger or self.ledger).on_transaction.where(
|
||||
lambda e: e.tx.id == txid
|
||||
async def send_to_address_and_wait(self, address, amount, blocks_to_generate=0, ledger=None):
|
||||
tx_watch = []
|
||||
txid = None
|
||||
done = False
|
||||
watcher = (ledger or self.ledger).on_transaction.where(
|
||||
lambda e: e.tx.id == txid or done or tx_watch.append(e.tx.id)
|
||||
)
|
||||
|
||||
txid = await self.blockchain.send_to_address(address, amount)
|
||||
done = txid in tx_watch
|
||||
await watcher
|
||||
|
||||
await self.generate_and_wait(blocks_to_generate, [txid], ledger)
|
||||
return txid
|
||||
|
||||
async def generate_and_wait(self, blocks_to_generate, txids, ledger=None):
|
||||
if blocks_to_generate > 0:
|
||||
watcher = (ledger or self.ledger).on_transaction.where(
|
||||
lambda e: ((e.tx.id in txids and txids.remove(e.tx.id)), len(txids) <= 0)[-1] # multi-statement lambda
|
||||
)
|
||||
self.conductor.spv_node.server.synchronized.clear()
|
||||
await self.blockchain.generate(blocks_to_generate)
|
||||
height = self.blockchain.block_expected
|
||||
await watcher
|
||||
while True:
|
||||
await self.conductor.spv_node.server.synchronized.wait()
|
||||
self.conductor.spv_node.server.synchronized.clear()
|
||||
if self.conductor.spv_node.server.db.db_height >= height:
|
||||
break
|
||||
|
||||
def on_address_update(self, address):
|
||||
return self.ledger.on_transaction.where(
|
||||
lambda e: e.address == address
|
||||
@@ -263,6 +319,19 @@ class IntegrationTestCase(AsyncioTestCase):
|
||||
lambda e: e.tx.id == tx.id and e.address == address
|
||||
)
|
||||
|
||||
async def generate(self, blocks):
|
||||
""" Ask lbrycrd to generate some blocks and wait until ledger has them. """
|
||||
prepare = self.ledger.on_header.where(self.blockchain.is_expected_block)
|
||||
height = self.blockchain.block_expected
|
||||
self.conductor.spv_node.server.synchronized.clear()
|
||||
await self.blockchain.generate(blocks)
|
||||
await prepare # no guarantee that it didn't happen already, so start waiting from before calling generate
|
||||
while True:
|
||||
await self.conductor.spv_node.server.synchronized.wait()
|
||||
self.conductor.spv_node.server.synchronized.clear()
|
||||
if self.conductor.spv_node.server.db.db_height >= height:
|
||||
break
|
||||
|
||||
|
||||
class FakeExchangeRateManager(ExchangeRateManager):
|
||||
|
||||
@@ -281,8 +350,8 @@ class FakeExchangeRateManager(ExchangeRateManager):
|
||||
|
||||
def get_fake_exchange_rate_manager(rates=None):
|
||||
return FakeExchangeRateManager(
|
||||
[LBRYFeed(), LBRYBTCFeed()],
|
||||
rates or {'BTCLBC': 3.0, 'USDBTC': 2.0}
|
||||
[BittrexBTCFeed(), BittrexUSDFeed()],
|
||||
rates or {'BTCLBC': 3.0, 'USDLBC': 2.0}
|
||||
)
|
||||
|
||||
|
||||
@@ -320,26 +389,32 @@ class CommandTestCase(IntegrationTestCase):
|
||||
self.server_blob_manager = None
|
||||
self.server = None
|
||||
self.reflector = None
|
||||
self.skip_libtorrent = True
|
||||
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
|
||||
logging.getLogger('lbry.blob_exchange').setLevel(self.VERBOSITY)
|
||||
logging.getLogger('lbry.daemon').setLevel(self.VERBOSITY)
|
||||
logging.getLogger('lbry.stream').setLevel(self.VERBOSITY)
|
||||
logging.getLogger('lbry.wallet').setLevel(self.VERBOSITY)
|
||||
|
||||
await super().asyncSetUp()
|
||||
|
||||
self.daemon = await self.add_daemon(self.wallet_node)
|
||||
|
||||
await self.account.ensure_address_gap()
|
||||
address = (await self.account.receiving.get_addresses(limit=1, only_usable=True))[0]
|
||||
sendtxid = await self.blockchain.send_to_address(address, 10)
|
||||
await self.confirm_tx(sendtxid)
|
||||
await self.generate(5)
|
||||
await self.send_to_address_and_wait(address, 10, 6)
|
||||
|
||||
server_tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, server_tmp_dir)
|
||||
self.server_config = Config()
|
||||
self.server_config = Config(
|
||||
data_dir=server_tmp_dir,
|
||||
wallet_dir=server_tmp_dir,
|
||||
save_files=True,
|
||||
download_dir=server_tmp_dir
|
||||
)
|
||||
self.server_config.transaction_cache_size = 10000
|
||||
self.server_storage = SQLiteStorage(self.server_config, ':memory:')
|
||||
await self.server_storage.open()
|
||||
|
||||
@@ -362,6 +437,7 @@ class CommandTestCase(IntegrationTestCase):
|
||||
await daemon.stop()
|
||||
|
||||
async def add_daemon(self, wallet_node=None, seed=None):
|
||||
start_wallet_node = False
|
||||
if wallet_node is None:
|
||||
wallet_node = WalletNode(
|
||||
self.wallet_node.manager_class,
|
||||
@@ -369,30 +445,42 @@ class CommandTestCase(IntegrationTestCase):
|
||||
port=self.extra_wallet_node_port
|
||||
)
|
||||
self.extra_wallet_node_port += 1
|
||||
await wallet_node.start(self.conductor.spv_node, seed=seed)
|
||||
self.extra_wallet_nodes.append(wallet_node)
|
||||
start_wallet_node = True
|
||||
|
||||
upload_dir = os.path.join(wallet_node.data_path, 'uploads')
|
||||
os.mkdir(upload_dir)
|
||||
|
||||
conf = Config()
|
||||
conf.data_dir = wallet_node.data_path
|
||||
conf.wallet_dir = wallet_node.data_path
|
||||
conf.download_dir = wallet_node.data_path
|
||||
conf = Config(
|
||||
# needed during instantiation to access known_hubs path
|
||||
data_dir=wallet_node.data_path,
|
||||
wallet_dir=wallet_node.data_path,
|
||||
save_files=True,
|
||||
download_dir=wallet_node.data_path
|
||||
)
|
||||
conf.upload_dir = upload_dir # not a real conf setting
|
||||
conf.share_usage_data = False
|
||||
conf.use_upnp = False
|
||||
conf.reflect_streams = True
|
||||
conf.blockchain_name = 'lbrycrd_regtest'
|
||||
conf.lbryum_servers = [('127.0.0.1', 50001)]
|
||||
conf.lbryum_servers = [(self.conductor.spv_node.hostname, self.conductor.spv_node.port)]
|
||||
conf.reflector_servers = [('127.0.0.1', 5566)]
|
||||
conf.fixed_peers = [('127.0.0.1', 5567)]
|
||||
conf.known_dht_nodes = []
|
||||
conf.blob_lru_cache_size = self.blob_lru_cache_size
|
||||
conf.transaction_cache_size = 10000
|
||||
conf.components_to_skip = [
|
||||
DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
|
||||
PEER_PROTOCOL_SERVER_COMPONENT
|
||||
]
|
||||
wallet_node.manager.config = conf
|
||||
if self.skip_libtorrent:
|
||||
conf.components_to_skip.append(LIBTORRENT_COMPONENT)
|
||||
|
||||
if start_wallet_node:
|
||||
await wallet_node.start(self.conductor.spv_node, seed=seed, config=conf)
|
||||
self.extra_wallet_nodes.append(wallet_node)
|
||||
else:
|
||||
wallet_node.manager.config = conf
|
||||
wallet_node.manager.ledger.config['known_hubs'] = conf.known_hubs
|
||||
|
||||
def wallet_maker(component_manager):
|
||||
wallet_component = WalletComponent(component_manager)
|
||||
@@ -403,7 +491,7 @@ class CommandTestCase(IntegrationTestCase):
|
||||
daemon = Daemon(conf, ComponentManager(
|
||||
conf, skip_components=conf.components_to_skip, wallet=wallet_maker,
|
||||
exchange_rate_manager=partial(ExchangeRateManagerComponent, rates={
|
||||
'BTCLBC': 1.0, 'USDBTC': 2.0
|
||||
'BTCLBC': 1.0, 'USDLBC': 2.0
|
||||
})
|
||||
))
|
||||
await daemon.initialize()
|
||||
@@ -413,9 +501,14 @@ class CommandTestCase(IntegrationTestCase):
|
||||
|
||||
async def confirm_tx(self, txid, ledger=None):
|
||||
""" Wait for tx to be in mempool, then generate a block, wait for tx to be in a block. """
|
||||
await self.on_transaction_id(txid, ledger)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_id(txid, ledger)
|
||||
# await (ledger or self.ledger).on_transaction.where(lambda e: e.tx.id == txid)
|
||||
on_tx = (ledger or self.ledger).on_transaction.where(lambda e: e.tx.id == txid)
|
||||
await asyncio.wait([self.generate(1), on_tx], timeout=5)
|
||||
|
||||
# # actually, if it's in the mempool or in the block we're fine
|
||||
# await self.generate_and_wait(1, [txid], ledger=ledger)
|
||||
# return txid
|
||||
|
||||
return txid
|
||||
|
||||
async def on_transaction_dict(self, tx):
|
||||
@@ -430,11 +523,6 @@ class CommandTestCase(IntegrationTestCase):
|
||||
addresses.add(txo['address'])
|
||||
return list(addresses)
|
||||
|
||||
async def generate(self, blocks):
|
||||
""" Ask lbrycrd to generate some blocks and wait until ledger has them. """
|
||||
await self.blockchain.generate(blocks)
|
||||
await self.ledger.on_header.where(self.blockchain.is_expected_block)
|
||||
|
||||
async def blockchain_claim_name(self, name: str, value: str, amount: str, confirm=True):
|
||||
txid = await self.blockchain._cli_cmnd('claimname', name, value, amount)
|
||||
if confirm:
|
||||
@@ -455,12 +543,27 @@ class CommandTestCase(IntegrationTestCase):
|
||||
""" Synchronous version of `out` method. """
|
||||
return json.loads(jsonrpc_dumps_pretty(value, ledger=self.ledger))['result']
|
||||
|
||||
async def confirm_and_render(self, awaitable, confirm) -> Transaction:
|
||||
async def confirm_and_render(self, awaitable, confirm, return_tx=False) -> Transaction:
|
||||
tx = await awaitable
|
||||
if confirm:
|
||||
await self.ledger.wait(tx)
|
||||
await self.generate(1)
|
||||
await self.ledger.wait(tx, self.blockchain.block_expected)
|
||||
if not return_tx:
|
||||
return self.sout(tx)
|
||||
return tx
|
||||
|
||||
async def create_nondeterministic_channel(self, name, price, pubkey_bytes, daemon=None, blocking=False):
|
||||
account = (daemon or self.daemon).wallet_manager.default_account
|
||||
claim_address = await account.receiving.get_or_create_usable_address()
|
||||
claim = Claim()
|
||||
claim.channel.public_key_bytes = pubkey_bytes
|
||||
tx = await Transaction.claim_create(
|
||||
name, claim, lbc_to_dewies(price),
|
||||
claim_address, [self.account], self.account
|
||||
)
|
||||
await tx.sign([self.account])
|
||||
await (daemon or self.daemon).broadcast_or_release(tx, blocking)
|
||||
return self.sout(tx)
|
||||
|
||||
def create_upload_file(self, data, prefix=None, suffix=None):
|
||||
@@ -472,26 +575,26 @@ class CommandTestCase(IntegrationTestCase):
|
||||
|
||||
async def stream_create(
|
||||
self, name='hovercraft', bid='1.0', file_path=None,
|
||||
data=b'hi!', confirm=True, prefix=None, suffix=None, **kwargs):
|
||||
if file_path is None:
|
||||
data=b'hi!', confirm=True, prefix=None, suffix=None, return_tx=False, **kwargs):
|
||||
if file_path is None and data is not None:
|
||||
file_path = self.create_upload_file(data=data, prefix=prefix, suffix=suffix)
|
||||
return await self.confirm_and_render(
|
||||
self.daemon.jsonrpc_stream_create(name, bid, file_path=file_path, **kwargs), confirm
|
||||
self.daemon.jsonrpc_stream_create(name, bid, file_path=file_path, **kwargs), confirm, return_tx
|
||||
)
|
||||
|
||||
async def stream_update(
|
||||
self, claim_id, data=None, prefix=None, suffix=None, confirm=True, **kwargs):
|
||||
self, claim_id, data=None, prefix=None, suffix=None, confirm=True, return_tx=False, **kwargs):
|
||||
if data is not None:
|
||||
file_path = self.create_upload_file(data=data, prefix=prefix, suffix=suffix)
|
||||
return await self.confirm_and_render(
|
||||
self.daemon.jsonrpc_stream_update(claim_id, file_path=file_path, **kwargs), confirm
|
||||
self.daemon.jsonrpc_stream_update(claim_id, file_path=file_path, **kwargs), confirm, return_tx
|
||||
)
|
||||
return await self.confirm_and_render(
|
||||
self.daemon.jsonrpc_stream_update(claim_id, **kwargs), confirm
|
||||
)
|
||||
|
||||
def stream_repost(self, claim_id, name='repost', bid='1.0', confirm=True, **kwargs):
|
||||
return self.confirm_and_render(
|
||||
async def stream_repost(self, claim_id, name='repost', bid='1.0', confirm=True, **kwargs):
|
||||
return await self.confirm_and_render(
|
||||
self.daemon.jsonrpc_stream_repost(claim_id=claim_id, name=name, bid=bid, **kwargs), confirm
|
||||
)
|
||||
|
||||
@@ -502,6 +605,11 @@ class CommandTestCase(IntegrationTestCase):
|
||||
self.daemon.jsonrpc_stream_abandon(*args, **kwargs), confirm
|
||||
)
|
||||
|
||||
async def purchase_create(self, *args, confirm=True, **kwargs):
|
||||
return await self.confirm_and_render(
|
||||
self.daemon.jsonrpc_purchase_create(*args, **kwargs), confirm
|
||||
)
|
||||
|
||||
async def publish(self, name, *args, confirm=True, **kwargs):
|
||||
return await self.confirm_and_render(
|
||||
self.daemon.jsonrpc_publish(name, *args, **kwargs), confirm
|
||||
@@ -548,15 +656,58 @@ class CommandTestCase(IntegrationTestCase):
|
||||
self.daemon.jsonrpc_support_create(claim_id, bid, **kwargs), confirm
|
||||
)
|
||||
|
||||
async def resolve(self, uri):
|
||||
return (await self.out(self.daemon.jsonrpc_resolve(uri)))[uri]
|
||||
async def support_abandon(self, *args, confirm=True, **kwargs):
|
||||
if 'blocking' not in kwargs:
|
||||
kwargs['blocking'] = False
|
||||
return await self.confirm_and_render(
|
||||
self.daemon.jsonrpc_support_abandon(*args, **kwargs), confirm
|
||||
)
|
||||
|
||||
async def account_send(self, *args, confirm=True, **kwargs):
|
||||
return await self.confirm_and_render(
|
||||
self.daemon.jsonrpc_account_send(*args, **kwargs), confirm
|
||||
)
|
||||
|
||||
async def wallet_send(self, *args, confirm=True, **kwargs):
|
||||
return await self.confirm_and_render(
|
||||
self.daemon.jsonrpc_wallet_send(*args, **kwargs), confirm
|
||||
)
|
||||
|
||||
async def txo_spend(self, *args, confirm=True, **kwargs):
|
||||
txs = await self.daemon.jsonrpc_txo_spend(*args, **kwargs)
|
||||
if confirm:
|
||||
await asyncio.wait([self.ledger.wait(tx) for tx in txs])
|
||||
await self.generate(1)
|
||||
await asyncio.wait([self.ledger.wait(tx, self.blockchain.block_expected) for tx in txs])
|
||||
return self.sout(txs)
|
||||
|
||||
async def blob_clean(self):
|
||||
return await self.out(self.daemon.jsonrpc_blob_clean())
|
||||
|
||||
async def status(self):
|
||||
return await self.out(self.daemon.jsonrpc_status())
|
||||
|
||||
async def resolve(self, uri, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_resolve(uri, **kwargs)))[uri]
|
||||
|
||||
async def claim_search(self, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_claim_search(**kwargs)))['items']
|
||||
|
||||
async def get_claim_by_claim_id(self, claim_id):
|
||||
return await self.out(self.ledger.get_claim_by_claim_id(claim_id))
|
||||
|
||||
async def file_list(self, *args, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_file_list(*args, **kwargs)))['items']
|
||||
|
||||
async def txo_list(self, *args, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_txo_list(*args, **kwargs)))['items']
|
||||
|
||||
async def txo_sum(self, *args, **kwargs):
|
||||
return await self.out(self.daemon.jsonrpc_txo_sum(*args, **kwargs))
|
||||
|
||||
async def txo_plot(self, *args, **kwargs):
|
||||
return await self.out(self.daemon.jsonrpc_txo_plot(*args, **kwargs))
|
||||
|
||||
async def claim_list(self, *args, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_claim_list(*args, **kwargs)))['items']
|
||||
|
||||
@@ -566,6 +717,12 @@ class CommandTestCase(IntegrationTestCase):
|
||||
async def channel_list(self, *args, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_channel_list(*args, **kwargs)))['items']
|
||||
|
||||
async def transaction_list(self, *args, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_transaction_list(*args, **kwargs)))['items']
|
||||
|
||||
async def blob_list(self, *args, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_blob_list(*args, **kwargs)))['items']
|
||||
|
||||
@staticmethod
|
||||
def get_claim_id(tx):
|
||||
return tx['outputs'][0]['claim_id']
|
||||
|
||||
File renamed without changes.
@@ -0,0 +1,290 @@
|
||||
import asyncio
|
||||
import binascii
|
||||
import os
|
||||
import logging
|
||||
import random
|
||||
from hashlib import sha1
|
||||
from tempfile import mkdtemp
|
||||
from typing import Optional
|
||||
|
||||
import libtorrent
|
||||
|
||||
|
||||
NOTIFICATION_MASKS = [
|
||||
"error",
|
||||
"peer",
|
||||
"port_mapping",
|
||||
"storage",
|
||||
"tracker",
|
||||
"debug",
|
||||
"status",
|
||||
"progress",
|
||||
"ip_block",
|
||||
"dht",
|
||||
"stats",
|
||||
"session_log",
|
||||
"torrent_log",
|
||||
"peer_log",
|
||||
"incoming_request",
|
||||
"dht_log",
|
||||
"dht_operation",
|
||||
"port_mapping_log",
|
||||
"picker_log",
|
||||
"file_progress",
|
||||
"piece_progress",
|
||||
"upload",
|
||||
"block_progress"
|
||||
]
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
DEFAULT_FLAGS = ( # fixme: somehow the logic here is inverted?
|
||||
libtorrent.add_torrent_params_flags_t.flag_auto_managed
|
||||
| libtorrent.add_torrent_params_flags_t.flag_update_subscribe
|
||||
)
|
||||
|
||||
|
||||
def get_notification_type(notification) -> str:
|
||||
for i, notification_type in enumerate(NOTIFICATION_MASKS):
|
||||
if (1 << i) & notification:
|
||||
return notification_type
|
||||
raise ValueError("unrecognized notification type")
|
||||
|
||||
|
||||
class TorrentHandle:
|
||||
def __init__(self, loop, executor, handle):
|
||||
self._loop = loop
|
||||
self._executor = executor
|
||||
self._handle: libtorrent.torrent_handle = handle
|
||||
self.started = asyncio.Event(loop=loop)
|
||||
self.finished = asyncio.Event(loop=loop)
|
||||
self.metadata_completed = asyncio.Event(loop=loop)
|
||||
self.size = 0
|
||||
self.total_wanted_done = 0
|
||||
self.name = ''
|
||||
self.tasks = []
|
||||
self.torrent_file: Optional[libtorrent.file_storage] = None
|
||||
self._base_path = None
|
||||
self._handle.set_sequential_download(1)
|
||||
|
||||
@property
|
||||
def largest_file(self) -> Optional[str]:
|
||||
if not self.torrent_file:
|
||||
return None
|
||||
index = self.largest_file_index
|
||||
return os.path.join(self._base_path, self.torrent_file.at(index).path)
|
||||
|
||||
@property
|
||||
def largest_file_index(self):
|
||||
largest_size, index = 0, 0
|
||||
for file_num in range(self.torrent_file.num_files()):
|
||||
if self.torrent_file.file_size(file_num) > largest_size:
|
||||
largest_size = self.torrent_file.file_size(file_num)
|
||||
index = file_num
|
||||
return index
|
||||
|
||||
def stop_tasks(self):
|
||||
while self.tasks:
|
||||
self.tasks.pop().cancel()
|
||||
|
||||
def _show_status(self):
|
||||
# fixme: cleanup
|
||||
if not self._handle.is_valid():
|
||||
return
|
||||
status = self._handle.status()
|
||||
if status.has_metadata:
|
||||
self.size = status.total_wanted
|
||||
self.total_wanted_done = status.total_wanted_done
|
||||
self.name = status.name
|
||||
if not self.metadata_completed.is_set():
|
||||
self.metadata_completed.set()
|
||||
log.info("Metadata completed for btih:%s - %s", status.info_hash, self.name)
|
||||
self.torrent_file = self._handle.get_torrent_info().files()
|
||||
self._base_path = status.save_path
|
||||
first_piece = self.torrent_file.at(self.largest_file_index).offset
|
||||
if not self.started.is_set():
|
||||
if self._handle.have_piece(first_piece):
|
||||
self.started.set()
|
||||
else:
|
||||
# prioritize it
|
||||
self._handle.set_piece_deadline(first_piece, 100)
|
||||
if not status.is_seeding:
|
||||
log.debug('%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d seeds: %d) %s - %s',
|
||||
status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000,
|
||||
status.num_peers, status.num_seeds, status.state, status.save_path)
|
||||
elif not self.finished.is_set():
|
||||
self.finished.set()
|
||||
log.info("Torrent finished: %s", self.name)
|
||||
|
||||
async def status_loop(self):
|
||||
while True:
|
||||
self._show_status()
|
||||
if self.finished.is_set():
|
||||
break
|
||||
await asyncio.sleep(0.1, loop=self._loop)
|
||||
|
||||
async def pause(self):
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, self._handle.pause
|
||||
)
|
||||
|
||||
async def resume(self):
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, lambda: self._handle.resume() # pylint: disable=unnecessary-lambda
|
||||
)
|
||||
|
||||
|
||||
class TorrentSession:
|
||||
def __init__(self, loop, executor):
|
||||
self._loop = loop
|
||||
self._executor = executor
|
||||
self._session: Optional[libtorrent.session] = None
|
||||
self._handles = {}
|
||||
self.tasks = []
|
||||
self.wait_start = True
|
||||
|
||||
async def add_fake_torrent(self):
|
||||
tmpdir = mkdtemp()
|
||||
info, btih = _create_fake_torrent(tmpdir)
|
||||
flags = libtorrent.add_torrent_params_flags_t.flag_seed_mode
|
||||
handle = self._session.add_torrent({
|
||||
'ti': info, 'save_path': tmpdir, 'flags': flags
|
||||
})
|
||||
self._handles[btih] = TorrentHandle(self._loop, self._executor, handle)
|
||||
return btih
|
||||
|
||||
async def bind(self, interface: str = '0.0.0.0', port: int = 10889):
|
||||
settings = {
|
||||
'listen_interfaces': f"{interface}:{port}",
|
||||
'enable_outgoing_utp': True,
|
||||
'enable_incoming_utp': True,
|
||||
'enable_outgoing_tcp': False,
|
||||
'enable_incoming_tcp': False
|
||||
}
|
||||
self._session = await self._loop.run_in_executor(
|
||||
self._executor, libtorrent.session, settings # pylint: disable=c-extension-no-member
|
||||
)
|
||||
self.tasks.append(self._loop.create_task(self.process_alerts()))
|
||||
|
||||
def stop(self):
|
||||
while self.tasks:
|
||||
self.tasks.pop().cancel()
|
||||
self._session.save_state()
|
||||
self._session.pause()
|
||||
self._session.stop_dht()
|
||||
self._session.stop_lsd()
|
||||
self._session.stop_natpmp()
|
||||
self._session.stop_upnp()
|
||||
self._session = None
|
||||
|
||||
def _pop_alerts(self):
|
||||
for alert in self._session.pop_alerts():
|
||||
log.info("torrent alert: %s", alert)
|
||||
|
||||
async def process_alerts(self):
|
||||
while True:
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, self._pop_alerts
|
||||
)
|
||||
await asyncio.sleep(1, loop=self._loop)
|
||||
|
||||
async def pause(self):
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, lambda: self._session.save_state() # pylint: disable=unnecessary-lambda
|
||||
)
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, lambda: self._session.pause() # pylint: disable=unnecessary-lambda
|
||||
)
|
||||
|
||||
async def resume(self):
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, self._session.resume
|
||||
)
|
||||
|
||||
def _add_torrent(self, btih: str, download_directory: Optional[str]):
|
||||
params = {'info_hash': binascii.unhexlify(btih.encode()), 'flags': DEFAULT_FLAGS}
|
||||
if download_directory:
|
||||
params['save_path'] = download_directory
|
||||
handle = self._session.add_torrent(params)
|
||||
handle.force_dht_announce()
|
||||
self._handles[btih] = TorrentHandle(self._loop, self._executor, handle)
|
||||
|
||||
def full_path(self, btih):
|
||||
return self._handles[btih].largest_file
|
||||
|
||||
async def add_torrent(self, btih, download_path):
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, self._add_torrent, btih, download_path
|
||||
)
|
||||
self._handles[btih].tasks.append(self._loop.create_task(self._handles[btih].status_loop()))
|
||||
await self._handles[btih].metadata_completed.wait()
|
||||
if self.wait_start:
|
||||
# fixme: temporary until we add streaming support, otherwise playback fails!
|
||||
await self._handles[btih].started.wait()
|
||||
|
||||
def remove_torrent(self, btih, remove_files=False):
|
||||
if btih in self._handles:
|
||||
handle = self._handles[btih]
|
||||
handle.stop_tasks()
|
||||
self._session.remove_torrent(handle._handle, 1 if remove_files else 0)
|
||||
self._handles.pop(btih)
|
||||
|
||||
async def save_file(self, btih, download_directory):
|
||||
handle = self._handles[btih]
|
||||
await handle.resume()
|
||||
|
||||
def get_size(self, btih):
|
||||
return self._handles[btih].size
|
||||
|
||||
def get_name(self, btih):
|
||||
return self._handles[btih].name
|
||||
|
||||
def get_downloaded(self, btih):
|
||||
return self._handles[btih].total_wanted_done
|
||||
|
||||
def is_completed(self, btih):
|
||||
return self._handles[btih].finished.is_set()
|
||||
|
||||
|
||||
def get_magnet_uri(btih):
|
||||
return f"magnet:?xt=urn:btih:{btih}"
|
||||
|
||||
|
||||
def _create_fake_torrent(tmpdir):
|
||||
# beware, that's just for testing
|
||||
path = os.path.join(tmpdir, 'tmp')
|
||||
with open(path, 'wb') as myfile:
|
||||
size = myfile.write(bytes([random.randint(0, 255) for _ in range(40)]) * 1024)
|
||||
file_storage = libtorrent.file_storage()
|
||||
file_storage.add_file('tmp', size)
|
||||
t = libtorrent.create_torrent(file_storage, 0, 4 * 1024 * 1024)
|
||||
libtorrent.set_piece_hashes(t, tmpdir)
|
||||
info = libtorrent.torrent_info(t.generate())
|
||||
btih = sha1(info.metadata()).hexdigest()
|
||||
return info, btih
|
||||
|
||||
|
||||
async def main():
|
||||
if os.path.exists("~/Downloads/ubuntu-18.04.3-live-server-amd64.torrent"):
|
||||
os.remove("~/Downloads/ubuntu-18.04.3-live-server-amd64.torrent")
|
||||
if os.path.exists("~/Downloads/ubuntu-18.04.3-live-server-amd64.iso"):
|
||||
os.remove("~/Downloads/ubuntu-18.04.3-live-server-amd64.iso")
|
||||
|
||||
btih = "dd8255ecdc7ca55fb0bbf81323d87062db1f6d1c"
|
||||
|
||||
executor = None
|
||||
session = TorrentSession(asyncio.get_event_loop(), executor)
|
||||
session2 = TorrentSession(asyncio.get_event_loop(), executor)
|
||||
await session.bind('localhost', port=4040)
|
||||
await session2.bind('localhost', port=4041)
|
||||
btih = await session.add_fake_torrent()
|
||||
session2._session.add_dht_node(('localhost', 4040))
|
||||
await session2.add_torrent(btih, "/tmp/down")
|
||||
while True:
|
||||
await asyncio.sleep(100)
|
||||
await session.pause()
|
||||
executor.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,72 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import typing
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TorrentInfo:
|
||||
__slots__ = ('dht_seeds', 'http_seeds', 'trackers', 'total_size')
|
||||
|
||||
def __init__(self, dht_seeds: typing.Tuple[typing.Tuple[str, int]],
|
||||
http_seeds: typing.Tuple[typing.Dict[str, typing.Any]],
|
||||
trackers: typing.Tuple[typing.Tuple[str, int]], total_size: int):
|
||||
self.dht_seeds = dht_seeds
|
||||
self.http_seeds = http_seeds
|
||||
self.trackers = trackers
|
||||
self.total_size = total_size
|
||||
|
||||
@classmethod
|
||||
def from_libtorrent_info(cls, torrent_info):
|
||||
return cls(
|
||||
torrent_info.nodes(), tuple(
|
||||
{
|
||||
'url': web_seed['url'],
|
||||
'type': web_seed['type'],
|
||||
'auth': web_seed['auth']
|
||||
} for web_seed in torrent_info.web_seeds()
|
||||
), tuple(
|
||||
(tracker.url, tracker.tier) for tracker in torrent_info.trackers()
|
||||
), torrent_info.total_size()
|
||||
)
|
||||
|
||||
|
||||
class Torrent:
|
||||
def __init__(self, loop, handle):
|
||||
self._loop = loop
|
||||
self._handle = handle
|
||||
self.finished = asyncio.Event(loop=loop)
|
||||
|
||||
def _threaded_update_status(self):
|
||||
status = self._handle.status()
|
||||
if not status.is_seeding:
|
||||
log.info(
|
||||
'%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d) %s',
|
||||
status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000,
|
||||
status.num_peers, status.state
|
||||
)
|
||||
elif not self.finished.is_set():
|
||||
self.finished.set()
|
||||
|
||||
async def wait_for_finished(self):
|
||||
while True:
|
||||
await self._loop.run_in_executor(
|
||||
None, self._threaded_update_status
|
||||
)
|
||||
if self.finished.is_set():
|
||||
log.info("finished downloading torrent!")
|
||||
await self.pause()
|
||||
break
|
||||
await asyncio.sleep(1, loop=self._loop)
|
||||
|
||||
async def pause(self):
|
||||
log.info("pause torrent")
|
||||
await self._loop.run_in_executor(
|
||||
None, self._handle.pause
|
||||
)
|
||||
|
||||
async def resume(self):
|
||||
await self._loop.run_in_executor(
|
||||
None, self._handle.resume
|
||||
)
|
||||
@@ -0,0 +1,140 @@
|
||||
import asyncio
|
||||
import binascii
|
||||
import logging
|
||||
import os
|
||||
import typing
|
||||
from typing import Optional
|
||||
from aiohttp.web import Request
|
||||
from lbry.file.source_manager import SourceManager
|
||||
from lbry.file.source import ManagedDownloadSource
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.torrent.session import TorrentSession
|
||||
from lbry.conf import Config
|
||||
from lbry.wallet.transaction import Transaction
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim
|
||||
from lbry.extras.daemon.storage import StoredContentClaim
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def path_or_none(encoded_path) -> Optional[str]:
|
||||
if not encoded_path:
|
||||
return
|
||||
return binascii.unhexlify(encoded_path).decode()
|
||||
|
||||
|
||||
class TorrentSource(ManagedDownloadSource):
|
||||
STATUS_STOPPED = "stopped"
|
||||
filter_fields = SourceManager.filter_fields
|
||||
filter_fields.update({
|
||||
'bt_infohash'
|
||||
})
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage', identifier: str,
|
||||
file_name: Optional[str] = None, download_directory: Optional[str] = None,
|
||||
status: Optional[str] = STATUS_STOPPED, claim: Optional['StoredContentClaim'] = None,
|
||||
download_id: Optional[str] = None, rowid: Optional[int] = None,
|
||||
content_fee: Optional['Transaction'] = None,
|
||||
analytics_manager: Optional['AnalyticsManager'] = None,
|
||||
added_on: Optional[int] = None, torrent_session: Optional['TorrentSession'] = None):
|
||||
super().__init__(loop, config, storage, identifier, file_name, download_directory, status, claim, download_id,
|
||||
rowid, content_fee, analytics_manager, added_on)
|
||||
self.torrent_session = torrent_session
|
||||
|
||||
@property
|
||||
def full_path(self) -> Optional[str]:
|
||||
full_path = self.torrent_session.full_path(self.identifier)
|
||||
self.download_directory = os.path.dirname(full_path)
|
||||
return full_path
|
||||
|
||||
async def start(self, timeout: Optional[float] = None, save_now: Optional[bool] = False):
|
||||
await self.torrent_session.add_torrent(self.identifier, self.download_directory)
|
||||
|
||||
async def stop(self, finished: bool = False):
|
||||
await self.torrent_session.remove_torrent(self.identifier)
|
||||
|
||||
async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
|
||||
await self.torrent_session.save_file(self.identifier, download_directory)
|
||||
|
||||
@property
|
||||
def torrent_length(self):
|
||||
return self.torrent_session.get_size(self.identifier)
|
||||
|
||||
@property
|
||||
def written_bytes(self):
|
||||
return self.torrent_session.get_downloaded(self.identifier)
|
||||
|
||||
@property
|
||||
def torrent_name(self):
|
||||
return self.torrent_session.get_name(self.identifier)
|
||||
|
||||
@property
|
||||
def bt_infohash(self):
|
||||
return self.identifier
|
||||
|
||||
def stop_tasks(self):
|
||||
pass
|
||||
|
||||
@property
|
||||
def completed(self):
|
||||
return self.torrent_session.is_completed(self.identifier)
|
||||
|
||||
|
||||
class TorrentManager(SourceManager):
|
||||
_sources: typing.Dict[str, ManagedDownloadSource]
|
||||
|
||||
filter_fields = set(SourceManager.filter_fields)
|
||||
filter_fields.update({
|
||||
'bt_infohash',
|
||||
'blobs_remaining', # TODO: here they call them "parts", but its pretty much the same concept
|
||||
'blobs_in_stream'
|
||||
})
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', torrent_session: 'TorrentSession',
|
||||
storage: 'SQLiteStorage', analytics_manager: Optional['AnalyticsManager'] = None):
|
||||
super().__init__(loop, config, storage, analytics_manager)
|
||||
self.torrent_session: 'TorrentSession' = torrent_session
|
||||
|
||||
async def recover_streams(self, file_infos: typing.List[typing.Dict]):
|
||||
raise NotImplementedError
|
||||
|
||||
async def _load_stream(self, rowid: int, bt_infohash: str, file_name: Optional[str],
|
||||
download_directory: Optional[str], status: str,
|
||||
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
|
||||
added_on: Optional[int]):
|
||||
stream = TorrentSource(
|
||||
self.loop, self.config, self.storage, identifier=bt_infohash, file_name=file_name,
|
||||
download_directory=download_directory, status=status, claim=claim, rowid=rowid,
|
||||
content_fee=content_fee, analytics_manager=self.analytics_manager, added_on=added_on,
|
||||
torrent_session=self.torrent_session
|
||||
)
|
||||
self.add(stream)
|
||||
|
||||
async def initialize_from_database(self):
|
||||
pass
|
||||
|
||||
async def start(self):
|
||||
await super().start()
|
||||
|
||||
def stop(self):
|
||||
super().stop()
|
||||
log.info("finished stopping the torrent manager")
|
||||
|
||||
async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
|
||||
await super().delete(source, delete_file)
|
||||
self.torrent_session.remove_torrent(source.identifier, delete_file)
|
||||
|
||||
async def create(self, file_path: str, key: Optional[bytes] = None,
|
||||
iv_generator: Optional[typing.Generator[bytes, None, None]] = None):
|
||||
raise NotImplementedError
|
||||
|
||||
async def _delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
|
||||
raise NotImplementedError
|
||||
# blob_hashes = [source.sd_hash] + [b.blob_hash for b in source.descriptor.blobs[:-1]]
|
||||
# await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False)
|
||||
# await self.storage.delete_stream(source.descriptor)
|
||||
|
||||
async def stream_partial_content(self, request: Request, sd_hash: str):
|
||||
raise NotImplementedError
|
||||
+217
-19
@@ -3,7 +3,9 @@ import codecs
|
||||
import datetime
|
||||
import random
|
||||
import socket
|
||||
import time
|
||||
import string
|
||||
import sys
|
||||
import json
|
||||
import typing
|
||||
import asyncio
|
||||
@@ -18,7 +20,9 @@ import pkg_resources
|
||||
|
||||
import certifi
|
||||
import aiohttp
|
||||
from lbry.schema.claim import Claim
|
||||
from prometheus_client import Counter
|
||||
from scribe.schema.claim import Claim
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -100,10 +104,6 @@ def check_connection(server="lbry.com", port=80, timeout=5) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
async def async_check_connection(server="lbry.com", port=80, timeout=1) -> bool:
|
||||
return await asyncio.get_event_loop().run_in_executor(None, check_connection, server, port, timeout)
|
||||
|
||||
|
||||
def random_string(length=10, chars=string.ascii_lowercase):
|
||||
return ''.join([random.choice(chars) for _ in range(length)])
|
||||
|
||||
@@ -155,7 +155,7 @@ def async_timed_cache(duration: int):
|
||||
async def _inner(*args, **kwargs) -> typing.Any:
|
||||
loop = asyncio.get_running_loop()
|
||||
time_now = loop.time()
|
||||
key = tuple([args, tuple([tuple([k, kwargs[k]]) for k in kwargs])])
|
||||
key = (args, tuple(kwargs.items()))
|
||||
if key in cache and (time_now - cache[key][1] < duration):
|
||||
return cache[key][0]
|
||||
to_cache = await func(*args, **kwargs)
|
||||
@@ -173,7 +173,7 @@ def cache_concurrent(async_fn):
|
||||
|
||||
@functools.wraps(async_fn)
|
||||
async def wrapper(*args, **kwargs):
|
||||
key = tuple([args, tuple([tuple([k, kwargs[k]]) for k in kwargs])])
|
||||
key = (args, tuple(kwargs.items()))
|
||||
cache[key] = cache.get(key) or asyncio.create_task(async_fn(*args, **kwargs))
|
||||
try:
|
||||
return await cache[key]
|
||||
@@ -187,6 +187,8 @@ def cache_concurrent(async_fn):
|
||||
async def resolve_host(url: str, port: int, proto: str) -> str:
|
||||
if proto not in ['udp', 'tcp']:
|
||||
raise Exception("invalid protocol")
|
||||
if url.lower() == 'localhost':
|
||||
return '127.0.0.1'
|
||||
try:
|
||||
if ipaddress.ip_address(url):
|
||||
return url
|
||||
@@ -201,18 +203,44 @@ async def resolve_host(url: str, port: int, proto: str) -> str:
|
||||
))[0][4][0]
|
||||
|
||||
|
||||
class LRUCache:
|
||||
class LRUCacheWithMetrics:
|
||||
__slots__ = [
|
||||
'capacity',
|
||||
'cache'
|
||||
'cache',
|
||||
'_track_metrics',
|
||||
'hits',
|
||||
'misses'
|
||||
]
|
||||
|
||||
def __init__(self, capacity):
|
||||
def __init__(self, capacity: int, metric_name: typing.Optional[str] = None, namespace: str = "daemon_cache"):
|
||||
self.capacity = capacity
|
||||
self.cache = collections.OrderedDict()
|
||||
if metric_name is None:
|
||||
self._track_metrics = False
|
||||
self.hits = self.misses = None
|
||||
else:
|
||||
self._track_metrics = True
|
||||
try:
|
||||
self.hits = Counter(
|
||||
f"{metric_name}_cache_hit_count", "Number of cache hits", namespace=namespace
|
||||
)
|
||||
self.misses = Counter(
|
||||
f"{metric_name}_cache_miss_count", "Number of cache misses", namespace=namespace
|
||||
)
|
||||
except ValueError as err:
|
||||
log.debug("failed to set up prometheus %s_cache_miss_count metric: %s", metric_name, err)
|
||||
self._track_metrics = False
|
||||
self.hits = self.misses = None
|
||||
|
||||
def get(self, key):
|
||||
value = self.cache.pop(key)
|
||||
def get(self, key, default=None):
|
||||
try:
|
||||
value = self.cache.pop(key)
|
||||
if self._track_metrics:
|
||||
self.hits.inc()
|
||||
except KeyError:
|
||||
if self._track_metrics:
|
||||
self.misses.inc()
|
||||
return default
|
||||
self.cache[key] = value
|
||||
return value
|
||||
|
||||
@@ -224,22 +252,97 @@ class LRUCache:
|
||||
self.cache.popitem(last=False)
|
||||
self.cache[key] = value
|
||||
|
||||
def clear(self):
|
||||
self.cache.clear()
|
||||
|
||||
def pop(self, key):
|
||||
return self.cache.pop(key)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
return self.set(key, value)
|
||||
|
||||
def __getitem__(self, item):
|
||||
return self.get(item)
|
||||
|
||||
def __contains__(self, item) -> bool:
|
||||
return item in self.cache
|
||||
|
||||
def __len__(self):
|
||||
return len(self.cache)
|
||||
|
||||
def __delitem__(self, key):
|
||||
self.cache.pop(key)
|
||||
|
||||
def __del__(self):
|
||||
self.clear()
|
||||
|
||||
|
||||
class LRUCache:
|
||||
__slots__ = [
|
||||
'capacity',
|
||||
'cache'
|
||||
]
|
||||
|
||||
def __init__(self, capacity: int):
|
||||
self.capacity = capacity
|
||||
self.cache = collections.OrderedDict()
|
||||
|
||||
def get(self, key, default=None):
|
||||
try:
|
||||
value = self.cache.pop(key)
|
||||
except KeyError:
|
||||
return default
|
||||
self.cache[key] = value
|
||||
return value
|
||||
|
||||
def set(self, key, value):
|
||||
try:
|
||||
self.cache.pop(key)
|
||||
except KeyError:
|
||||
if len(self.cache) >= self.capacity:
|
||||
self.cache.popitem(last=False)
|
||||
self.cache[key] = value
|
||||
|
||||
def items(self):
|
||||
return self.cache.items()
|
||||
|
||||
def clear(self):
|
||||
self.cache.clear()
|
||||
|
||||
def pop(self, key, default=None):
|
||||
return self.cache.pop(key, default)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
return self.set(key, value)
|
||||
|
||||
def __getitem__(self, item):
|
||||
return self.get(item)
|
||||
|
||||
def __contains__(self, item) -> bool:
|
||||
return item in self.cache
|
||||
|
||||
def __len__(self):
|
||||
return len(self.cache)
|
||||
|
||||
def __delitem__(self, key):
|
||||
self.cache.pop(key)
|
||||
|
||||
def __del__(self):
|
||||
self.clear()
|
||||
|
||||
|
||||
def lru_cache_concurrent(cache_size: typing.Optional[int] = None,
|
||||
override_lru_cache: typing.Optional[LRUCache] = None):
|
||||
override_lru_cache: typing.Optional[LRUCacheWithMetrics] = None):
|
||||
if not cache_size and override_lru_cache is None:
|
||||
raise ValueError("invalid cache size")
|
||||
concurrent_cache = {}
|
||||
lru_cache = override_lru_cache or LRUCache(cache_size)
|
||||
lru_cache = override_lru_cache if override_lru_cache is not None else LRUCacheWithMetrics(cache_size)
|
||||
|
||||
def wrapper(async_fn):
|
||||
|
||||
@functools.wraps(async_fn)
|
||||
async def _inner(*args, **kwargs):
|
||||
key = tuple([args, tuple([tuple([k, kwargs[k]]) for k in kwargs])])
|
||||
key = (args, tuple(kwargs.items()))
|
||||
if key in lru_cache:
|
||||
return lru_cache.get(key)
|
||||
|
||||
@@ -264,15 +367,110 @@ def get_ssl_context() -> ssl.SSLContext:
|
||||
@contextlib.asynccontextmanager
|
||||
async def aiohttp_request(method, url, **kwargs) -> typing.AsyncContextManager[aiohttp.ClientResponse]:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.request(method, url, ssl=get_ssl_context(), **kwargs) as response:
|
||||
async with session.request(method, url, **kwargs) as response:
|
||||
yield response
|
||||
|
||||
|
||||
async def get_external_ip() -> typing.Optional[str]: # used if upnp is disabled or non-functioning
|
||||
# the ipaddress module does not show these subnets as reserved
|
||||
CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
|
||||
IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
|
||||
|
||||
|
||||
def is_valid_public_ipv4(address, allow_localhost: bool = False, allow_lan: bool = False):
|
||||
try:
|
||||
parsed_ip = ipaddress.ip_address(address)
|
||||
if parsed_ip.is_loopback and allow_localhost:
|
||||
return True
|
||||
if allow_lan and parsed_ip.is_private:
|
||||
return True
|
||||
if any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local, parsed_ip.is_loopback,
|
||||
parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private)):
|
||||
return False
|
||||
else:
|
||||
return not any((CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
|
||||
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
|
||||
except (ipaddress.AddressValueError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
async def fallback_get_external_ip(): # used if spv servers can't be used for ip detection
|
||||
try:
|
||||
async with aiohttp_request("get", "https://api.lbry.com/ip") as resp:
|
||||
response = await resp.json()
|
||||
if response['success']:
|
||||
return response['data']['ip']
|
||||
return response['data']['ip'], None
|
||||
except Exception:
|
||||
return
|
||||
return None, None
|
||||
|
||||
|
||||
async def _get_external_ip(default_servers) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]:
|
||||
# used if upnp is disabled or non-functioning
|
||||
from lbry.wallet.udp import SPVStatusClientProtocol # pylint: disable=C0415
|
||||
|
||||
hostname_to_ip = {}
|
||||
ip_to_hostnames = collections.defaultdict(list)
|
||||
|
||||
async def resolve_spv(server, port):
|
||||
try:
|
||||
server_addr = await resolve_host(server, port, 'udp')
|
||||
hostname_to_ip[server] = (server_addr, port)
|
||||
ip_to_hostnames[(server_addr, port)].append(server)
|
||||
except Exception:
|
||||
log.exception("error looking up dns for spv servers")
|
||||
|
||||
# accumulate the dns results
|
||||
await asyncio.gather(*(resolve_spv(server, port) for (server, port) in default_servers))
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
pong_responses = asyncio.Queue()
|
||||
connection = SPVStatusClientProtocol(pong_responses)
|
||||
try:
|
||||
await loop.create_datagram_endpoint(lambda: connection, ('0.0.0.0', 0))
|
||||
# could raise OSError if it cant bind
|
||||
randomized_servers = list(ip_to_hostnames.keys())
|
||||
random.shuffle(randomized_servers)
|
||||
for server in randomized_servers:
|
||||
connection.ping(server)
|
||||
try:
|
||||
_, pong = await asyncio.wait_for(pong_responses.get(), 1)
|
||||
if is_valid_public_ipv4(pong.ip_address):
|
||||
return pong.ip_address, ip_to_hostnames[server][0]
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
return None, None
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
async def get_external_ip(default_servers) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]:
|
||||
ip_from_spv_servers = await _get_external_ip(default_servers)
|
||||
if not ip_from_spv_servers[1]:
|
||||
return await fallback_get_external_ip()
|
||||
return ip_from_spv_servers
|
||||
|
||||
|
||||
def is_running_from_bundle():
|
||||
# see https://pyinstaller.readthedocs.io/en/stable/runtime-information.html
|
||||
return getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS')
|
||||
|
||||
|
||||
class LockWithMetrics(asyncio.Lock):
|
||||
def __init__(self, acquire_metric, held_time_metric, loop=None):
|
||||
super().__init__(loop=loop)
|
||||
self._acquire_metric = acquire_metric
|
||||
self._lock_held_time_metric = held_time_metric
|
||||
self._lock_acquired_time = None
|
||||
|
||||
async def acquire(self):
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
return await super().acquire()
|
||||
finally:
|
||||
self._lock_acquired_time = time.perf_counter()
|
||||
self._acquire_metric.observe(self._lock_acquired_time - start)
|
||||
|
||||
def release(self):
|
||||
try:
|
||||
return super().release()
|
||||
finally:
|
||||
self._lock_held_time_metric.observe(time.perf_counter() - self._lock_acquired_time)
|
||||
+20
-14
@@ -1,17 +1,23 @@
|
||||
__node_daemon__ = 'lbrycrdd'
|
||||
__node_cli__ = 'lbrycrd-cli'
|
||||
__node_bin__ = ''
|
||||
__node_url__ = (
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.3/lbrycrd-linux-1743.zip'
|
||||
__lbcd__ = 'lbcd'
|
||||
__lbcctl__ = 'lbcctl'
|
||||
__lbcwallet__ = 'lbcwallet'
|
||||
__lbcd_url__ = (
|
||||
'https://github.com/lbryio/lbcd/releases/download/' +
|
||||
'v0.22.200-beta/lbcd_0.22.200-beta_TARGET_PLATFORM.tar.gz'
|
||||
)
|
||||
__lbcwallet_url__ = (
|
||||
'https://github.com/lbryio/lbcwallet/releases/download/' +
|
||||
'v0.13.100-alpha-rc2/lbcwallet_0.13.100-alpha-rc2_TARGET_PLATFORM.tar.gz'
|
||||
)
|
||||
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
|
||||
|
||||
from .wallet import Wallet, WalletStorage, TimestampedPreferences, ENCRYPT_ON_DISK
|
||||
from .manager import WalletManager
|
||||
from .network import Network
|
||||
from .ledger import Ledger, RegTestLedger, TestNetLedger, BlockHeightEvent
|
||||
from .account import Account, AddressManager, SingleKey, HierarchicalDeterministic
|
||||
from .transaction import Transaction, Output, Input
|
||||
from .script import OutputScript, InputScript
|
||||
from .database import SQLiteMixin, Database
|
||||
from .header import Headers
|
||||
from lbry.wallet.wallet import Wallet, WalletStorage, TimestampedPreferences, ENCRYPT_ON_DISK
|
||||
from lbry.wallet.manager import WalletManager
|
||||
from lbry.wallet.network import Network
|
||||
from lbry.wallet.ledger import Ledger, RegTestLedger, TestNetLedger, BlockHeightEvent
|
||||
from lbry.wallet.account import Account, AddressManager, SingleKey, HierarchicalDeterministic, \
|
||||
DeterministicChannelKeyManager
|
||||
from lbry.wallet.transaction import Transaction, Output, Input
|
||||
from lbry.wallet.script import OutputScript, InputScript
|
||||
from lbry.wallet.database import SQLiteMixin, Database
|
||||
from lbry.wallet.header import Headers
|
||||
+106
-75
@@ -5,18 +5,16 @@ import logging
|
||||
import typing
|
||||
import asyncio
|
||||
import random
|
||||
from functools import partial
|
||||
from hashlib import sha256
|
||||
from string import hexdigits
|
||||
from typing import Type, Dict, Tuple, Optional, Any, List
|
||||
|
||||
import ecdsa
|
||||
from lbry.error import InvalidPasswordError
|
||||
from lbry.crypto.crypt import aes_encrypt, aes_decrypt
|
||||
|
||||
from .bip32 import PrivateKey, PubKey, from_extended_key_string
|
||||
from scribe.schema.bip32 import PrivateKey, PublicKey, KeyPath, from_extended_key_string
|
||||
from .mnemonic import Mnemonic
|
||||
from .constants import COIN, CLAIM_TYPES, TXO_TYPES
|
||||
from .constants import COIN, TXO_TYPES
|
||||
from .transaction import Transaction, Input, Output
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
@@ -35,6 +33,49 @@ def validate_claim_id(claim_id):
|
||||
raise Exception("Claim id is not hex encoded")
|
||||
|
||||
|
||||
class DeterministicChannelKeyManager:
|
||||
|
||||
def __init__(self, account: 'Account'):
|
||||
self.account = account
|
||||
self.last_known = 0
|
||||
self.cache = {}
|
||||
self._private_key: Optional[PrivateKey] = None
|
||||
|
||||
@property
|
||||
def private_key(self):
|
||||
if self._private_key is None:
|
||||
if self.account.private_key is not None:
|
||||
self._private_key = self.account.private_key.child(KeyPath.CHANNEL)
|
||||
return self._private_key
|
||||
|
||||
def maybe_generate_deterministic_key_for_channel(self, txo):
|
||||
if self.private_key is None:
|
||||
return
|
||||
next_private_key = self.private_key.child(self.last_known)
|
||||
public_key = next_private_key.public_key
|
||||
public_key_bytes = public_key.pubkey_bytes
|
||||
if txo.claim.channel.public_key_bytes == public_key_bytes:
|
||||
self.cache[public_key.address] = next_private_key
|
||||
self.last_known += 1
|
||||
|
||||
async def ensure_cache_primed(self):
|
||||
if self.private_key is not None:
|
||||
await self.generate_next_key()
|
||||
|
||||
async def generate_next_key(self) -> PrivateKey:
|
||||
db = self.account.ledger.db
|
||||
while True:
|
||||
next_private_key = self.private_key.child(self.last_known)
|
||||
public_key = next_private_key.public_key
|
||||
self.cache[public_key.address] = next_private_key
|
||||
if not await db.is_channel_key_used(self.account, public_key):
|
||||
return next_private_key
|
||||
self.last_known += 1
|
||||
|
||||
def get_private_key_from_pubkey_hash(self, pubkey_hash) -> PrivateKey:
|
||||
return self.cache.get(pubkey_hash)
|
||||
|
||||
|
||||
class AddressManager:
|
||||
|
||||
name: str
|
||||
@@ -71,6 +112,7 @@ class AddressManager:
|
||||
|
||||
def _query_addresses(self, **constraints):
|
||||
return self.account.ledger.db.get_addresses(
|
||||
read_only=constraints.pop("read_only", False),
|
||||
accounts=[self.account],
|
||||
chain=self.chain_number,
|
||||
**constraints
|
||||
@@ -79,7 +121,7 @@ class AddressManager:
|
||||
def get_private_key(self, index: int) -> PrivateKey:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_public_key(self, index: int) -> PubKey:
|
||||
def get_public_key(self, index: int) -> PublicKey:
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_max_gap(self):
|
||||
@@ -96,7 +138,8 @@ class AddressManager:
|
||||
return [r['address'] for r in records]
|
||||
|
||||
async def get_or_create_usable_address(self) -> str:
|
||||
addresses = await self.get_addresses(only_usable=True, limit=10)
|
||||
async with self.address_generator_lock:
|
||||
addresses = await self.get_addresses(only_usable=True, limit=10)
|
||||
if addresses:
|
||||
return random.choice(addresses)
|
||||
addresses = await self.ensure_address_gap()
|
||||
@@ -118,8 +161,8 @@ class HierarchicalDeterministic(AddressManager):
|
||||
@classmethod
|
||||
def from_dict(cls, account: 'Account', d: dict) -> Tuple[AddressManager, AddressManager]:
|
||||
return (
|
||||
cls(account, 0, **d.get('receiving', {'gap': 20, 'maximum_uses_per_address': 1})),
|
||||
cls(account, 1, **d.get('change', {'gap': 6, 'maximum_uses_per_address': 1}))
|
||||
cls(account, KeyPath.RECEIVE, **d.get('receiving', {'gap': 20, 'maximum_uses_per_address': 1})),
|
||||
cls(account, KeyPath.CHANGE, **d.get('change', {'gap': 6, 'maximum_uses_per_address': 1}))
|
||||
)
|
||||
|
||||
def merge(self, d: dict):
|
||||
@@ -132,7 +175,7 @@ class HierarchicalDeterministic(AddressManager):
|
||||
def get_private_key(self, index: int) -> PrivateKey:
|
||||
return self.account.private_key.child(self.chain_number).child(index)
|
||||
|
||||
def get_public_key(self, index: int) -> PubKey:
|
||||
def get_public_key(self, index: int) -> PublicKey:
|
||||
return self.account.public_key.child(self.chain_number).child(index)
|
||||
|
||||
async def get_max_gap(self) -> int:
|
||||
@@ -192,7 +235,7 @@ class SingleKey(AddressManager):
|
||||
@classmethod
|
||||
def from_dict(cls, account: 'Account', d: dict) \
|
||||
-> Tuple[AddressManager, AddressManager]:
|
||||
same_address_manager = cls(account, account.public_key, 0)
|
||||
same_address_manager = cls(account, account.public_key, KeyPath.RECEIVE)
|
||||
return same_address_manager, same_address_manager
|
||||
|
||||
def to_dict_instance(self):
|
||||
@@ -201,7 +244,7 @@ class SingleKey(AddressManager):
|
||||
def get_private_key(self, index: int) -> PrivateKey:
|
||||
return self.account.private_key
|
||||
|
||||
def get_public_key(self, index: int) -> PubKey:
|
||||
def get_public_key(self, index: int) -> PublicKey:
|
||||
return self.account.public_key
|
||||
|
||||
async def get_max_gap(self) -> int:
|
||||
@@ -223,9 +266,6 @@ class SingleKey(AddressManager):
|
||||
|
||||
class Account:
|
||||
|
||||
mnemonic_class = Mnemonic
|
||||
private_key_class = PrivateKey
|
||||
public_key_class = PubKey
|
||||
address_generators: Dict[str, Type[AddressManager]] = {
|
||||
SingleKey.name: SingleKey,
|
||||
HierarchicalDeterministic.name: HierarchicalDeterministic,
|
||||
@@ -233,7 +273,7 @@ class Account:
|
||||
|
||||
def __init__(self, ledger: 'Ledger', wallet: 'Wallet', name: str,
|
||||
seed: str, private_key_string: str, encrypted: bool,
|
||||
private_key: Optional[PrivateKey], public_key: PubKey,
|
||||
private_key: Optional[PrivateKey], public_key: PublicKey,
|
||||
address_generator: dict, modified_on: float, channel_keys: dict) -> None:
|
||||
self.ledger = ledger
|
||||
self.wallet = wallet
|
||||
@@ -244,13 +284,14 @@ class Account:
|
||||
self.private_key_string = private_key_string
|
||||
self.init_vectors: Dict[str, bytes] = {}
|
||||
self.encrypted = encrypted
|
||||
self.private_key = private_key
|
||||
self.public_key = public_key
|
||||
self.private_key: Optional[PrivateKey] = private_key
|
||||
self.public_key: PublicKey = public_key
|
||||
generator_name = address_generator.get('name', HierarchicalDeterministic.name)
|
||||
self.address_generator = self.address_generators[generator_name]
|
||||
self.receiving, self.change = self.address_generator.from_dict(self, address_generator)
|
||||
self.address_managers = {am.chain_number: am for am in {self.receiving, self.change}}
|
||||
self.address_managers = {am.chain_number: am for am in (self.receiving, self.change)}
|
||||
self.channel_keys = channel_keys
|
||||
self.deterministic_channel_keys = DeterministicChannelKeyManager(self)
|
||||
ledger.add_account(self)
|
||||
wallet.add_account(self)
|
||||
|
||||
@@ -265,19 +306,19 @@ class Account:
|
||||
name: str = None, address_generator: dict = None):
|
||||
return cls.from_dict(ledger, wallet, {
|
||||
'name': name,
|
||||
'seed': cls.mnemonic_class().make_seed(),
|
||||
'seed': Mnemonic().make_seed(),
|
||||
'address_generator': address_generator or {}
|
||||
})
|
||||
|
||||
@classmethod
|
||||
def get_private_key_from_seed(cls, ledger: 'Ledger', seed: str, password: str):
|
||||
return cls.private_key_class.from_seed(
|
||||
ledger, cls.mnemonic_class.mnemonic_to_seed(seed, password or 'lbryum')
|
||||
return PrivateKey.from_seed(
|
||||
ledger, Mnemonic.mnemonic_to_seed(seed, password or 'lbryum')
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def keys_from_dict(cls, ledger: 'Ledger', d: dict) \
|
||||
-> Tuple[str, Optional[PrivateKey], PubKey]:
|
||||
-> Tuple[str, Optional[PrivateKey], PublicKey]:
|
||||
seed = d.get('seed', '')
|
||||
private_key_string = d.get('private_key', '')
|
||||
private_key = None
|
||||
@@ -310,7 +351,7 @@ class Account:
|
||||
private_key=private_key,
|
||||
public_key=public_key,
|
||||
address_generator=d.get('address_generator', {}),
|
||||
modified_on=d.get('modified_on', time.time()),
|
||||
modified_on=int(d.get('modified_on', time.time())),
|
||||
channel_keys=d.get('certificates', {})
|
||||
)
|
||||
|
||||
@@ -342,7 +383,7 @@ class Account:
|
||||
def merge(self, d: dict):
|
||||
if d.get('modified_on', 0) > self.modified_on:
|
||||
self.name = d['name']
|
||||
self.modified_on = d.get('modified_on', time.time())
|
||||
self.modified_on = int(d.get('modified_on', time.time()))
|
||||
assert self.address_generator.name == d['address_generator']['name']
|
||||
for chain_name in ('change', 'receiving'):
|
||||
if chain_name in d['address_generator']:
|
||||
@@ -434,9 +475,9 @@ class Account:
|
||||
addresses.extend(new_addresses)
|
||||
return addresses
|
||||
|
||||
async def get_addresses(self, **constraints) -> List[str]:
|
||||
rows = await self.ledger.db.select_addresses('address', accounts=[self], **constraints)
|
||||
return [r[0] for r in rows]
|
||||
async def get_addresses(self, read_only=False, **constraints) -> List[str]:
|
||||
rows = await self.ledger.db.select_addresses('address', read_only=read_only, accounts=[self], **constraints)
|
||||
return [r['address'] for r in rows]
|
||||
|
||||
def get_address_records(self, **constraints):
|
||||
return self.ledger.db.get_addresses(accounts=[self], **constraints)
|
||||
@@ -448,16 +489,16 @@ class Account:
|
||||
assert not self.encrypted, "Cannot get private key on encrypted wallet account."
|
||||
return self.address_managers[chain].get_private_key(index)
|
||||
|
||||
def get_public_key(self, chain: int, index: int) -> PubKey:
|
||||
def get_public_key(self, chain: int, index: int) -> PublicKey:
|
||||
return self.address_managers[chain].get_public_key(index)
|
||||
|
||||
def get_balance(self, confirmations: int = 0, include_claims=False, **constraints):
|
||||
def get_balance(self, confirmations=0, include_claims=False, read_only=False, **constraints):
|
||||
if not include_claims:
|
||||
constraints.update({'txo_type__in': (0, TXO_TYPES['purchase'])})
|
||||
constraints.update({'txo_type__in': (TXO_TYPES['other'], TXO_TYPES['purchase'])})
|
||||
if confirmations > 0:
|
||||
height = self.ledger.headers.height - (confirmations-1)
|
||||
constraints.update({'height__lte': height, 'height__gt': 0})
|
||||
return self.ledger.db.get_balance(accounts=[self], **constraints)
|
||||
return self.ledger.db.get_balance(accounts=[self], read_only=read_only, **constraints)
|
||||
|
||||
async def get_max_gap(self):
|
||||
change_gap = await self.change.get_max_gap()
|
||||
@@ -467,6 +508,12 @@ class Account:
|
||||
'max_receiving_gap': receiving_gap,
|
||||
}
|
||||
|
||||
def get_txos(self, **constraints):
|
||||
return self.ledger.get_txos(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_txo_count(self, **constraints):
|
||||
return self.ledger.get_txo_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_utxos(self, **constraints):
|
||||
return self.ledger.get_utxos(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
@@ -513,16 +560,18 @@ class Account:
|
||||
|
||||
return tx
|
||||
|
||||
def add_channel_private_key(self, private_key):
|
||||
public_key_bytes = private_key.get_verifying_key().to_der()
|
||||
channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes)
|
||||
self.channel_keys[channel_pubkey_hash] = private_key.to_pem().decode()
|
||||
async def generate_channel_private_key(self):
|
||||
return await self.deterministic_channel_keys.generate_next_key()
|
||||
|
||||
def get_channel_private_key(self, public_key_bytes):
|
||||
def add_channel_private_key(self, private_key: PrivateKey):
|
||||
self.channel_keys[private_key.address] = private_key.to_pem().decode()
|
||||
|
||||
async def get_channel_private_key(self, public_key_bytes) -> PrivateKey:
|
||||
channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes)
|
||||
private_key_pem = self.channel_keys.get(channel_pubkey_hash)
|
||||
if private_key_pem:
|
||||
return ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=sha256)
|
||||
return PrivateKey.from_pem(self.ledger, private_key_pem)
|
||||
return self.deterministic_channel_keys.get_private_key_from_pubkey_hash(channel_pubkey_hash)
|
||||
|
||||
async def maybe_migrate_certificates(self):
|
||||
if not self.channel_keys:
|
||||
@@ -531,11 +580,10 @@ class Account:
|
||||
for private_key_pem in self.channel_keys.values():
|
||||
if not isinstance(private_key_pem, str):
|
||||
continue
|
||||
if "-----BEGIN EC PRIVATE KEY-----" not in private_key_pem:
|
||||
if not private_key_pem.startswith("-----BEGIN"):
|
||||
continue
|
||||
private_key = ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=sha256)
|
||||
public_key_der = private_key.get_verifying_key().to_der()
|
||||
channel_keys[self.ledger.public_key_to_address(public_key_der)] = private_key_pem
|
||||
private_key = PrivateKey.from_pem(self.ledger, private_key_pem)
|
||||
channel_keys[private_key.address] = private_key_pem
|
||||
if self.channel_keys != channel_keys:
|
||||
self.channel_keys = channel_keys
|
||||
self.wallet.save()
|
||||
@@ -555,41 +603,24 @@ class Account:
|
||||
if gap_changed:
|
||||
self.wallet.save()
|
||||
|
||||
async def get_detailed_balance(self, confirmations=0, reserved_subtotals=False):
|
||||
tips_balance, supports_balance, claims_balance = 0, 0, 0
|
||||
get_total_balance = partial(self.get_balance, confirmations=confirmations, include_claims=True)
|
||||
total = await get_total_balance()
|
||||
if reserved_subtotals:
|
||||
claims_balance = await get_total_balance(txo_type__in=CLAIM_TYPES)
|
||||
for amount, spent, from_me, to_me, height in await self.get_support_summary():
|
||||
if confirmations > 0 and not 0 < height <= self.ledger.headers.height - (confirmations - 1):
|
||||
continue
|
||||
if not spent and to_me:
|
||||
if from_me:
|
||||
supports_balance += amount
|
||||
else:
|
||||
tips_balance += amount
|
||||
reserved = claims_balance + supports_balance + tips_balance
|
||||
else:
|
||||
reserved = await self.get_balance(
|
||||
confirmations=confirmations, include_claims=True, txo_type__gt=0
|
||||
)
|
||||
return {
|
||||
'total': total,
|
||||
'available': total - reserved,
|
||||
'reserved': reserved,
|
||||
'reserved_subtotals': {
|
||||
'claims': claims_balance,
|
||||
'supports': supports_balance,
|
||||
'tips': tips_balance
|
||||
} if reserved_subtotals else None
|
||||
}
|
||||
async def get_detailed_balance(self, confirmations=0, read_only=False):
|
||||
constraints = {}
|
||||
if confirmations > 0:
|
||||
height = self.ledger.headers.height - (confirmations-1)
|
||||
constraints.update({'height__lte': height, 'height__gt': 0})
|
||||
return await self.ledger.db.get_detailed_balance(
|
||||
accounts=[self], read_only=read_only, **constraints
|
||||
)
|
||||
|
||||
def get_transaction_history(self, **constraints):
|
||||
return self.ledger.get_transaction_history(wallet=self.wallet, accounts=[self], **constraints)
|
||||
def get_transaction_history(self, read_only=False, **constraints):
|
||||
return self.ledger.get_transaction_history(
|
||||
read_only=read_only, wallet=self.wallet, accounts=[self], **constraints
|
||||
)
|
||||
|
||||
def get_transaction_history_count(self, **constraints):
|
||||
return self.ledger.get_transaction_history_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
def get_transaction_history_count(self, read_only=False, **constraints):
|
||||
return self.ledger.get_transaction_history_count(
|
||||
read_only=read_only, wallet=self.wallet, accounts=[self], **constraints
|
||||
)
|
||||
|
||||
def get_claims(self, **constraints):
|
||||
return self.ledger.get_claims(wallet=self.wallet, accounts=[self], **constraints)
|
||||
@@ -622,7 +653,7 @@ class Account:
|
||||
return self.ledger.get_support_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_support_summary(self):
|
||||
return self.ledger.db.get_supports_summary(account_id=self.id)
|
||||
return self.ledger.db.get_supports_summary(wallet=self.wallet, accounts=[self])
|
||||
|
||||
async def release_all_outputs(self):
|
||||
await self.ledger.db.release_all_outputs(self)
|
||||
@@ -1,253 +0,0 @@
|
||||
from coincurve import PublicKey, PrivateKey as _PrivateKey
|
||||
|
||||
from lbry.crypto.hash import hmac_sha512, hash160, double_sha256
|
||||
from lbry.crypto.base58 import Base58
|
||||
from .util import cachedproperty
|
||||
|
||||
|
||||
class DerivationError(Exception):
|
||||
""" Raised when an invalid derivation occurs. """
|
||||
|
||||
|
||||
class _KeyBase:
|
||||
""" A BIP32 Key, public or private. """
|
||||
|
||||
def __init__(self, ledger, chain_code, n, depth, parent):
|
||||
if not isinstance(chain_code, (bytes, bytearray)):
|
||||
raise TypeError('chain code must be raw bytes')
|
||||
if len(chain_code) != 32:
|
||||
raise ValueError('invalid chain code')
|
||||
if not 0 <= n < 1 << 32:
|
||||
raise ValueError('invalid child number')
|
||||
if not 0 <= depth < 256:
|
||||
raise ValueError('invalid depth')
|
||||
if parent is not None:
|
||||
if not isinstance(parent, type(self)):
|
||||
raise TypeError('parent key has bad type')
|
||||
self.ledger = ledger
|
||||
self.chain_code = chain_code
|
||||
self.n = n
|
||||
self.depth = depth
|
||||
self.parent = parent
|
||||
|
||||
def _hmac_sha512(self, msg):
|
||||
""" Use SHA-512 to provide an HMAC, returned as a pair of 32-byte objects. """
|
||||
hmac = hmac_sha512(self.chain_code, msg)
|
||||
return hmac[:32], hmac[32:]
|
||||
|
||||
def _extended_key(self, ver_bytes, raw_serkey):
|
||||
""" Return the 78-byte extended key given prefix version bytes and serialized key bytes. """
|
||||
if not isinstance(ver_bytes, (bytes, bytearray)):
|
||||
raise TypeError('ver_bytes must be raw bytes')
|
||||
if len(ver_bytes) != 4:
|
||||
raise ValueError('ver_bytes must have length 4')
|
||||
if not isinstance(raw_serkey, (bytes, bytearray)):
|
||||
raise TypeError('raw_serkey must be raw bytes')
|
||||
if len(raw_serkey) != 33:
|
||||
raise ValueError('raw_serkey must have length 33')
|
||||
|
||||
return (ver_bytes + bytes((self.depth,))
|
||||
+ self.parent_fingerprint() + self.n.to_bytes(4, 'big')
|
||||
+ self.chain_code + raw_serkey)
|
||||
|
||||
def identifier(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def extended_key(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def fingerprint(self):
|
||||
""" Return the key's fingerprint as 4 bytes. """
|
||||
return self.identifier()[:4]
|
||||
|
||||
def parent_fingerprint(self):
|
||||
""" Return the parent key's fingerprint as 4 bytes. """
|
||||
return self.parent.fingerprint() if self.parent else bytes((0,)*4)
|
||||
|
||||
def extended_key_string(self):
|
||||
""" Return an extended key as a base58 string. """
|
||||
return Base58.encode_check(self.extended_key())
|
||||
|
||||
|
||||
class PubKey(_KeyBase):
|
||||
""" A BIP32 public key. """
|
||||
|
||||
def __init__(self, ledger, pubkey, chain_code, n, depth, parent=None):
|
||||
super().__init__(ledger, chain_code, n, depth, parent)
|
||||
if isinstance(pubkey, PublicKey):
|
||||
self.verifying_key = pubkey
|
||||
else:
|
||||
self.verifying_key = self._verifying_key_from_pubkey(pubkey)
|
||||
|
||||
@classmethod
|
||||
def _verifying_key_from_pubkey(cls, pubkey):
|
||||
""" Converts a 33-byte compressed pubkey into an PublicKey object. """
|
||||
if not isinstance(pubkey, (bytes, bytearray)):
|
||||
raise TypeError('pubkey must be raw bytes')
|
||||
if len(pubkey) != 33:
|
||||
raise ValueError('pubkey must be 33 bytes')
|
||||
if pubkey[0] not in (2, 3):
|
||||
raise ValueError('invalid pubkey prefix byte')
|
||||
return PublicKey(pubkey)
|
||||
|
||||
@cachedproperty
|
||||
def pubkey_bytes(self):
|
||||
""" Return the compressed public key as 33 bytes. """
|
||||
return self.verifying_key.format(True)
|
||||
|
||||
@cachedproperty
|
||||
def address(self):
|
||||
""" The public key as a P2PKH address. """
|
||||
return self.ledger.public_key_to_address(self.pubkey_bytes)
|
||||
|
||||
def ec_point(self):
|
||||
return self.verifying_key.point()
|
||||
|
||||
def child(self, n: int):
|
||||
""" Return the derived child extended pubkey at index N. """
|
||||
if not 0 <= n < (1 << 31):
|
||||
raise ValueError('invalid BIP32 public key child number')
|
||||
|
||||
msg = self.pubkey_bytes + n.to_bytes(4, 'big')
|
||||
L_b, R_b = self._hmac_sha512(msg) # pylint: disable=invalid-name
|
||||
derived_key = self.verifying_key.add(L_b)
|
||||
return PubKey(self.ledger, derived_key, R_b, n, self.depth + 1, self)
|
||||
|
||||
def identifier(self):
|
||||
""" Return the key's identifier as 20 bytes. """
|
||||
return hash160(self.pubkey_bytes)
|
||||
|
||||
def extended_key(self):
|
||||
""" Return a raw extended public key. """
|
||||
return self._extended_key(
|
||||
self.ledger.extended_public_key_prefix,
|
||||
self.pubkey_bytes
|
||||
)
|
||||
|
||||
|
||||
class PrivateKey(_KeyBase):
|
||||
"""A BIP32 private key."""
|
||||
|
||||
HARDENED = 1 << 31
|
||||
|
||||
def __init__(self, ledger, privkey, chain_code, n, depth, parent=None):
|
||||
super().__init__(ledger, chain_code, n, depth, parent)
|
||||
if isinstance(privkey, _PrivateKey):
|
||||
self.signing_key = privkey
|
||||
else:
|
||||
self.signing_key = self._signing_key_from_privkey(privkey)
|
||||
|
||||
@classmethod
|
||||
def _signing_key_from_privkey(cls, private_key):
|
||||
""" Converts a 32-byte private key into an coincurve.PrivateKey object. """
|
||||
return _PrivateKey.from_int(PrivateKey._private_key_secret_exponent(private_key))
|
||||
|
||||
@classmethod
|
||||
def _private_key_secret_exponent(cls, private_key):
|
||||
""" Return the private key as a secret exponent if it is a valid private key. """
|
||||
if not isinstance(private_key, (bytes, bytearray)):
|
||||
raise TypeError('private key must be raw bytes')
|
||||
if len(private_key) != 32:
|
||||
raise ValueError('private key must be 32 bytes')
|
||||
return int.from_bytes(private_key, 'big')
|
||||
|
||||
@classmethod
|
||||
def from_seed(cls, ledger, seed):
|
||||
# This hard-coded message string seems to be coin-independent...
|
||||
hmac = hmac_sha512(b'Bitcoin seed', seed)
|
||||
privkey, chain_code = hmac[:32], hmac[32:]
|
||||
return cls(ledger, privkey, chain_code, 0, 0)
|
||||
|
||||
@cachedproperty
|
||||
def private_key_bytes(self):
|
||||
""" Return the serialized private key (no leading zero byte). """
|
||||
return self.signing_key.secret
|
||||
|
||||
@cachedproperty
|
||||
def public_key(self):
|
||||
""" Return the corresponding extended public key. """
|
||||
verifying_key = self.signing_key.public_key
|
||||
parent_pubkey = self.parent.public_key if self.parent else None
|
||||
return PubKey(self.ledger, verifying_key, self.chain_code, self.n, self.depth,
|
||||
parent_pubkey)
|
||||
|
||||
def ec_point(self):
|
||||
return self.public_key.ec_point()
|
||||
|
||||
def secret_exponent(self):
|
||||
""" Return the private key as a secret exponent. """
|
||||
return self.signing_key.to_int()
|
||||
|
||||
def wif(self):
|
||||
""" Return the private key encoded in Wallet Import Format. """
|
||||
return self.ledger.private_key_to_wif(self.private_key_bytes)
|
||||
|
||||
def address(self):
|
||||
""" The public key as a P2PKH address. """
|
||||
return self.public_key.address
|
||||
|
||||
def child(self, n):
|
||||
""" Return the derived child extended private key at index N."""
|
||||
if not 0 <= n < (1 << 32):
|
||||
raise ValueError('invalid BIP32 private key child number')
|
||||
|
||||
if n >= self.HARDENED:
|
||||
serkey = b'\0' + self.private_key_bytes
|
||||
else:
|
||||
serkey = self.public_key.pubkey_bytes
|
||||
|
||||
msg = serkey + n.to_bytes(4, 'big')
|
||||
L_b, R_b = self._hmac_sha512(msg) # pylint: disable=invalid-name
|
||||
derived_key = self.signing_key.add(L_b)
|
||||
return PrivateKey(self.ledger, derived_key, R_b, n, self.depth + 1, self)
|
||||
|
||||
def sign(self, data):
|
||||
""" Produce a signature for piece of data by double hashing it and signing the hash. """
|
||||
return self.signing_key.sign(data, hasher=double_sha256)
|
||||
|
||||
def identifier(self):
|
||||
"""Return the key's identifier as 20 bytes."""
|
||||
return self.public_key.identifier()
|
||||
|
||||
def extended_key(self):
|
||||
"""Return a raw extended private key."""
|
||||
return self._extended_key(
|
||||
self.ledger.extended_private_key_prefix,
|
||||
b'\0' + self.private_key_bytes
|
||||
)
|
||||
|
||||
|
||||
def _from_extended_key(ledger, ekey):
|
||||
"""Return a PubKey or PrivateKey from an extended key raw bytes."""
|
||||
if not isinstance(ekey, (bytes, bytearray)):
|
||||
raise TypeError('extended key must be raw bytes')
|
||||
if len(ekey) != 78:
|
||||
raise ValueError('extended key must have length 78')
|
||||
|
||||
depth = ekey[4]
|
||||
n = int.from_bytes(ekey[9:13], 'big')
|
||||
chain_code = ekey[13:45]
|
||||
|
||||
if ekey[:4] == ledger.extended_public_key_prefix:
|
||||
pubkey = ekey[45:]
|
||||
key = PubKey(ledger, pubkey, chain_code, n, depth)
|
||||
elif ekey[:4] == ledger.extended_private_key_prefix:
|
||||
if ekey[45] != 0:
|
||||
raise ValueError('invalid extended private key prefix byte')
|
||||
privkey = ekey[46:]
|
||||
key = PrivateKey(ledger, privkey, chain_code, n, depth)
|
||||
else:
|
||||
raise ValueError('version bytes unrecognised')
|
||||
|
||||
return key
|
||||
|
||||
|
||||
def from_extended_key_string(ledger, ekey_str):
|
||||
"""Given an extended key string, such as
|
||||
|
||||
xpub6BsnM1W2Y7qLMiuhi7f7dbAwQZ5Cz5gYJCRzTNainXzQXYjFwtuQXHd
|
||||
3qfi3t3KJtHxshXezfjft93w4UE7BGMtKwhqEHae3ZA7d823DVrL
|
||||
|
||||
return a PubKey or PrivateKey.
|
||||
"""
|
||||
return _from_extended_key(ledger, Base58.decode_check(ekey_str))
|
||||
File diff suppressed because it is too large.
Load diff
@@ -5,7 +5,7 @@ from lbry.wallet.transaction import OutputEffectiveAmountEstimator
|
||||
|
||||
MAXIMUM_TRIES = 100000
|
||||
|
||||
STRATEGIES = []
|
||||
STRATEGIES = ['sqlite'] # sqlite coin chooser is in database.py
|
||||
|
||||
|
||||
def strategy(method):
|
||||
|
||||
@@ -2,10 +2,12 @@ NULL_HASH32 = b'\x00'*32
|
||||
|
||||
CENT = 1000000
|
||||
COIN = 100*CENT
|
||||
DUST = 1000
|
||||
|
||||
TIMEOUT = 30.0
|
||||
|
||||
TXO_TYPES = {
|
||||
"other": 0,
|
||||
"stream": 1,
|
||||
"channel": 2,
|
||||
"support": 3,
|
||||
@@ -14,9 +16,13 @@ TXO_TYPES = {
|
||||
"repost": 6,
|
||||
}
|
||||
|
||||
CLAIM_TYPE_NAMES = [
|
||||
'stream',
|
||||
'channel',
|
||||
'collection',
|
||||
'repost',
|
||||
]
|
||||
|
||||
CLAIM_TYPES = [
|
||||
TXO_TYPES['stream'],
|
||||
TXO_TYPES['channel'],
|
||||
TXO_TYPES['collection'],
|
||||
TXO_TYPES['repost'],
|
||||
TXO_TYPES[name] for name in CLAIM_TYPE_NAMES
|
||||
]
|
||||
Loaded 100 of 227 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user