Compare commits
1163
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d14eb24141 | ||
|
|
820015872b | ||
|
|
f57b1b3c3d | ||
|
|
a85ceb3cca | ||
|
|
254ae66ed3 | ||
|
|
3206fe873e | ||
|
|
63046fce58 | ||
|
|
b76863d055 | ||
|
|
d7ed74ab94 | ||
|
|
7388f5f1cd | ||
|
|
c211338218 | ||
|
|
8ac89af8bd | ||
|
|
bbbaf59591 | ||
|
|
169419896f | ||
|
|
0543dca502 | ||
|
|
cc6011d57a | ||
|
|
fc4407ef7e | ||
|
|
03735a125f | ||
|
|
5baeda9ff1 | ||
|
|
9b9794b5e0 | ||
|
|
0697d60a48 | ||
|
|
cfe6c82a31 | ||
|
|
3e30228d95 | ||
|
|
7264b53e5f | ||
|
|
60836d8523 | ||
|
|
ef89c2e47a | ||
|
|
2d9e3e1847 | ||
|
|
30136a9697 | ||
|
|
db7ccd66d3 | ||
|
|
cfe6483102 | ||
|
|
561566e723 | ||
|
|
c2dcc4c898 | ||
|
|
d09bfdc4ff | ||
|
|
358ef4536f | ||
|
|
5061a35e66 | ||
|
|
cd9a1e8c9e | ||
|
|
646902e75e | ||
|
|
40d26cb868 | ||
|
|
b64aa51c0c | ||
|
|
8206441834 | ||
|
|
d713783736 | ||
|
|
57dffaa2ce | ||
|
|
9e81dd2360 | ||
|
|
e2798969d7 | ||
|
|
1c31ec66f2 | ||
|
|
241f9fc7b0 | ||
|
|
270192486a | ||
|
|
a799503c97 | ||
|
|
9685928087 | ||
|
|
0e4b2fad99 | ||
|
|
3c4571a4e0 | ||
|
|
046147eb1d | ||
|
|
7834520e54 | ||
|
|
8e5b4d4b6f | ||
|
|
4544a074d9 | ||
|
|
9b78501392 | ||
|
|
f59ddcc88d | ||
|
|
a4955a2b79 | ||
|
|
92ae1a565b | ||
|
|
15a56ca25e | ||
|
|
9dcaa829ea | ||
|
|
9f65799a3d | ||
|
|
886587848b | ||
|
|
a97fc6dba8 | ||
|
|
c124e88d12 | ||
|
|
17f3870296 | ||
|
|
4626d42d08 | ||
|
|
e1e760055c | ||
|
|
45bf6c3bf3 | ||
|
|
fef0cc764d | ||
|
|
72049afcf6 | ||
|
|
d26c06dbf3 | ||
|
|
268decd655 | ||
|
|
7ae246c839 | ||
|
|
c7c454e4fb | ||
|
|
8e27297a81 | ||
|
|
2cdec72985 | ||
|
|
0085ac534d | ||
|
|
7828a79a96 | ||
|
|
5576c21e67 | ||
|
|
e49cfb1d2b | ||
|
|
1e541d0225 | ||
|
|
0974afd26d | ||
|
|
8d93594771 | ||
|
|
1136ac70e8 | ||
|
|
dc8d5a39ea | ||
|
|
8329e649b0 | ||
|
|
66da8b164f | ||
|
|
ea48577864 | ||
|
|
597146b136 | ||
|
|
30dd0c1e11 | ||
|
|
88772c4266 | ||
|
|
dc1d9e1c84 | ||
|
|
69ea65835d | ||
|
|
d5bae3a8c6 | ||
|
|
f14010bd5b | ||
|
|
87094fc83f | ||
|
|
7c179cfeab | ||
|
|
7582c221d1 | ||
|
|
c109895848 | ||
|
|
eccedada40 | ||
|
|
25d54accf8 | ||
|
|
d07685f0e9 | ||
|
|
2445c00c7e | ||
|
|
4c1d3ef514 | ||
|
|
4614c7d4c2 | ||
|
|
bbf1ef0dc3 | ||
|
|
3433c9e708 | ||
|
|
2cd5d75a2e | ||
|
|
2535b8adef | ||
|
|
4edab7bb7f | ||
|
|
fd8658e317 | ||
|
|
51d21d8c86 | ||
|
|
b4c3307cdf | ||
|
|
4e8d10cb44 | ||
|
|
e96875a425 | ||
|
|
5ab0035348 | ||
|
|
4ddff96b1e | ||
|
|
a08d84c1df | ||
|
|
21c71bfac1 | ||
|
|
6baaed3581 | ||
|
|
152dbfd5d1 | ||
|
|
a56d14086b | ||
|
|
aee87693f8 | ||
|
|
976b4affd9 | ||
|
|
e222b6ad9c | ||
|
|
19b17374e8 | ||
|
|
43989122bb | ||
|
|
72712d6047 | ||
|
|
0b52d2cc15 | ||
|
|
8304102136 | ||
|
|
3381aefcfa | ||
|
|
279a365cb1 | ||
|
|
2c9e00da56 | ||
|
|
f7cae69704 | ||
|
|
b7d58bcdbc | ||
|
|
13a856b843 | ||
|
|
8da38985c3 | ||
|
|
60cf6c6b97 | ||
|
|
35c2b34564 | ||
|
|
ef2e048efc | ||
|
|
6b3261aa33 | ||
|
|
1849c02cb6 | ||
|
|
1ec74a89e2 | ||
|
|
c591792de9 | ||
|
|
3108543ae5 | ||
|
|
1eb221c743 | ||
|
|
bebf6bc2e7 | ||
|
|
9e91cc2138 | ||
|
|
c5b939cfb7 | ||
|
|
5bd411ca27 | ||
|
|
a533cda6f0 | ||
|
|
fe4b07b8ae | ||
|
|
f9f2ccd904 | ||
|
|
d9e87d7c32 | ||
|
|
a0092c0770 | ||
|
|
3100131125 | ||
|
|
988880cf83 | ||
|
|
c3fb9672c4 | ||
|
|
0a2d94e425 | ||
|
|
8d9073cd31 | ||
|
|
d075961ffa | ||
|
|
7a72409b61 | ||
|
|
34fc530fba | ||
|
|
f257ff2f97 | ||
|
|
7ad5822c5b | ||
|
|
9a8f9f0a94 | ||
|
|
6421cecafb | ||
|
|
be544d6d89 | ||
|
|
3c89ecafdd | ||
|
|
35ec4eec52 | ||
|
|
e47f737a2f | ||
|
|
ac671a065b | ||
|
|
74116cc550 | ||
|
|
406070a5c3 | ||
|
|
0ccafd5b53 | ||
|
|
940f517aa3 | ||
|
|
216e5f65ad | ||
|
|
a74685d66d | ||
|
|
b7791d2845 | ||
|
|
d151a82d78 | ||
|
|
8ce61fbd52 | ||
|
|
90c24aade3 | ||
|
|
6b3f787fee | ||
|
|
4ebe4ce1b7 | ||
|
|
8c79740ee8 | ||
|
|
59d027ca02 | ||
|
|
37a7345a90 | ||
|
|
c519d4651b | ||
|
|
9b3b609e40 | ||
|
|
6254f53716 | ||
|
|
f05dc46432 | ||
|
|
3de0982a4a | ||
|
|
c2184fb3bf | ||
|
|
919c09fcb0 | ||
|
|
1d9dbd40ec | ||
|
|
0cd953a6f3 | ||
|
|
4db2b72351 | ||
|
|
dd54fcbdbd | ||
|
|
3123cf7ac6 | ||
|
|
6b579dd4ce | ||
|
|
16dfaa3e27 | ||
|
|
d7842b9f84 | ||
|
|
115034fccb | ||
|
|
309e957a85 | ||
|
|
d7007e402e | ||
|
|
91323a21cf | ||
|
|
fea893d76c | ||
|
|
761bc6ba4c | ||
|
|
75172feb4e | ||
|
|
3285fb1608 | ||
|
|
03a4c6910d | ||
|
|
485b958599 | ||
|
|
da47ba2f67 | ||
|
|
c39195488a | ||
|
|
227fb0ae9b | ||
|
|
b12ff5b503 | ||
|
|
0946c72b88 | ||
|
|
7d49b046d4 | ||
|
|
5f0426c840 | ||
|
|
73e239cc5f | ||
|
|
ad670f721a | ||
|
|
028a4a70cf | ||
|
|
77d7960347 | ||
|
|
39821146bd | ||
|
|
7d505a41ac | ||
|
|
e457b2f0d6 | ||
|
|
c9cf7fd4d4 | ||
|
|
b0371dd33d | ||
|
|
25e16c3565 | ||
|
|
7b39527863 | ||
|
|
d861b08866 | ||
|
|
fb438dc108 | ||
|
|
4e6b4f179b | ||
|
|
00d038c8f3 | ||
|
|
a9f6a68952 | ||
|
|
b9142bbc5a | ||
|
|
6c812f663e | ||
|
|
a93ec9783a | ||
|
|
2d184d77b6 | ||
|
|
bce299ccc7 | ||
|
|
235cebd14a | ||
|
|
a638aa9d53 | ||
|
|
67cce0ef7e | ||
|
|
82f4267bf6 | ||
|
|
45a9ca29c4 | ||
|
|
7f4e813277 | ||
|
|
3805ff4a0c | ||
|
|
464cfd475e | ||
|
|
fe469ae57f | ||
|
|
550ef9a1c4 | ||
|
|
935adfb51a | ||
|
|
3974df4a62 | ||
|
|
4870974161 | ||
|
|
8c4b0037f5 | ||
|
|
2c6f763ef2 | ||
|
|
ca28de02d8 | ||
|
|
bfc15ea029 | ||
|
|
6e8b8a5920 | ||
|
|
099f3b6a62 | ||
|
|
142d182bc1 | ||
|
|
1437871d88 | ||
|
|
352bf69409 | ||
|
|
9bdf3d23e1 | ||
|
|
be8ecfa707 | ||
|
|
51da0d0259 | ||
|
|
f55b78a994 | ||
|
|
e1a44c93f8 | ||
|
|
07e7087a09 | ||
|
|
2c79c7e2f6 | ||
|
|
09f6637fe0 | ||
|
|
3784db3308 | ||
|
|
2b950ff5dd | ||
|
|
09339c9cfb | ||
|
|
ccadd88af5 | ||
|
|
cc02a0efc2 | ||
|
|
43a1385b79 | ||
|
|
5101464e3b | ||
|
|
3d71478d38 | ||
|
|
4989ed445e | ||
|
|
d9413039ec | ||
|
|
eba0c9be34 | ||
|
|
48c9e9f3cc | ||
|
|
81ebde88db | ||
|
|
79ced9d0f8 | ||
|
|
a4058b84ce | ||
|
|
7bf211a52b | ||
|
|
d5f722792f | ||
|
|
0f02906c9b | ||
|
|
9582e228b1 | ||
|
|
45f20431f9 | ||
|
|
7554e6d7f9 | ||
|
|
cb8f26f177 | ||
|
|
b5dfce7861 | ||
|
|
2ca5a65544 | ||
|
|
17deb136db | ||
|
|
8c9710c76c | ||
|
|
32f7ecb261 | ||
|
|
fb77fde710 | ||
|
|
3c67bb90d7 | ||
|
|
dabb168853 | ||
|
|
45e5b3b219 | ||
|
|
a6b7469923 | ||
|
|
cb5dab3033 | ||
|
|
21d0038ff2 | ||
|
|
c094d8f2e8 | ||
|
|
c465d6a6c2 | ||
|
|
73d35bc985 | ||
|
|
2a8ccb065b | ||
|
|
8f04a50ce1 | ||
|
|
888aa5586b | ||
|
|
99f56f5d22 | ||
|
|
ad6281090d | ||
|
|
f0d334d3e2 | ||
|
|
5f829b048f | ||
|
|
1a961e66ff | ||
|
|
fdb0e22656 | ||
|
|
132ee1915f | ||
|
|
44bf4f3c8f | ||
|
|
6237767d5a | ||
|
|
dec9d96417 | ||
|
|
b167c87267 | ||
|
|
2280fe8e8e | ||
|
|
575d6dcd2d | ||
|
|
f729490c6b | ||
|
|
b32124cdd6 | ||
|
|
3d4321ee38 | ||
|
|
85034b382e | ||
|
|
77a51d1ad4 | ||
|
|
33e0cdc2d7 | ||
|
|
6519faa2fe | ||
|
|
5e3a234cbe | ||
|
|
e54c31d2d5 | ||
|
|
66c0537251 | ||
|
|
ac58516593 | ||
|
|
c3da6322b5 | ||
|
|
3d241500cf | ||
|
|
ded8224f66 | ||
|
|
f8814881a1 | ||
|
|
cc2852cd48 | ||
|
|
467637a9eb | ||
|
|
3cfc292d84 | ||
|
|
6acf94a810 | ||
|
|
31367fb4c4 | ||
|
|
12d6074e3b | ||
|
|
ff30386051 | ||
|
|
601f99ac16 | ||
|
|
87fe5c6101 | ||
|
|
68399ca31c | ||
|
|
2a6d7fd80f | ||
|
|
4725f510d8 | ||
|
|
be0ba22222 | ||
|
|
c8781392be | ||
|
|
b97164fcfb | ||
|
|
0dfb92281b | ||
|
|
4fe80c40da | ||
|
|
f0fac5115a | ||
|
|
46dd389d0d | ||
|
|
1e28e21ab5 | ||
|
|
7832c62c5d | ||
|
|
d025ee9dbe | ||
|
|
a9a9cb4319 | ||
|
|
aa727cb9b1 | ||
|
|
b8c9a99f20 | ||
|
|
aff995b0d0 | ||
|
|
2cc7e5dfdc | ||
|
|
5235a150b1 | ||
|
|
c6372ea9de | ||
|
|
7df4cc44c4 | ||
|
|
d47cf40544 | ||
|
|
7f5d88e95c | ||
|
|
d09663c066 | ||
|
|
ef97c9b69f | ||
|
|
d855e6c8b1 | ||
|
|
cd66f7eb43 | ||
|
|
6a35a7ba4c | ||
|
|
a3e146dc68 | ||
|
|
b81305a4a9 | ||
|
|
73884b34bc | ||
|
|
6166a34db2 | ||
|
|
6fa7da4b1c | ||
|
|
c3e426c491 | ||
|
|
21e023f0db | ||
|
|
063be001b3 | ||
|
|
5dff02e8bc | ||
|
|
60a59407d8 | ||
|
|
20a5aecfca | ||
|
|
c2e7b5a67d | ||
|
|
8f32303d07 | ||
|
|
891b1e7782 | ||
|
|
f26394fd3b | ||
|
|
4d83d42b4c | ||
|
|
57f1108df2 | ||
|
|
2641a9abe5 | ||
|
|
6b193ab350 | ||
|
|
b1bb37511c | ||
|
|
319187d6d6 | ||
|
|
02eb789f84 | ||
|
|
5a9338a27f | ||
|
|
eb6924277f | ||
|
|
325419404d | ||
|
|
bd8f371fd5 | ||
|
|
1783ff2845 | ||
|
|
d388527ffa | ||
|
|
19494088bd | ||
|
|
920dad524a | ||
|
|
ec89bcac8e | ||
|
|
a916c1f4ad | ||
|
|
a9a0ac92d7 | ||
|
|
da8a8bd1ef | ||
|
|
d9c746891d | ||
|
|
67817005b5 | ||
|
|
24d11de5a7 | ||
|
|
9251c87323 | ||
|
|
e12fab90d1 | ||
|
|
0a194b5b01 | ||
|
|
8d028adc53 | ||
|
|
dfca15395e | ||
|
|
e21f2362fe | ||
|
|
1ce328e8a9 | ||
|
|
038a5f999f | ||
|
|
5d3704c7ea | ||
|
|
87037c06c9 | ||
|
|
dd412c0f50 | ||
|
|
bf44befff6 | ||
|
|
e61874bb6f | ||
|
|
1e5331768f | ||
|
|
ec9a3a4f7c | ||
|
|
e439a3a8dc | ||
|
|
19f70d7a11 | ||
|
|
afe7ed5b05 | ||
|
|
d4bf004d74 | ||
|
|
e4d06a088b | ||
|
|
0929088b12 | ||
|
|
7b4838fc9b | ||
|
|
0cf9533248 | ||
|
|
84ff0b8a9f | ||
|
|
d467dcfeaf | ||
|
|
8e68ba4751 | ||
|
|
0f2a85ba9f | ||
|
|
7674a0a91e | ||
|
|
5bc1a66572 | ||
|
|
9b56067213 | ||
|
|
9a9df2fc3c | ||
|
|
9989d8d1d4 | ||
|
|
f9471f297e | ||
|
|
146b693e4a | ||
|
|
7295b7e329 | ||
|
|
e2441ea3e7 | ||
|
|
119e51912e | ||
|
|
dd950f5b0d | ||
|
|
78a9bad1e1 | ||
|
|
0c6eaf5484 | ||
|
|
1010068ddb | ||
|
|
82eec3d8d7 | ||
|
|
ee7b37d3f3 | ||
|
|
143d82d242 | ||
|
|
8b91b38855 | ||
|
|
1098f0d2a3 | ||
|
|
ab53cec022 | ||
|
|
6f5f8e5648 | ||
|
|
edfd707c22 | ||
|
|
1870f30af8 | ||
|
|
90106f5f08 | ||
|
|
9924b7b438 | ||
|
|
aa37faab0a | ||
|
|
dc10f8ce72 | ||
|
|
996686c1da | ||
|
|
488785d013 | ||
|
|
3abdc01230 | ||
|
|
8da04a584f | ||
|
|
27cc61d45e | ||
|
|
7371c30064 | ||
|
|
140d163895 | ||
|
|
dc33bdc1dc | ||
|
|
74df4fab83 | ||
|
|
1e5cd3d7a1 | ||
|
|
a54e9b64aa | ||
|
|
74660704e3 | ||
|
|
7439893a2a | ||
|
|
e27e49e9dc | ||
|
|
34ed729c59 | ||
|
|
adaeeca3fd | ||
|
|
dac75563d3 | ||
|
|
cbc76adcaa | ||
|
|
69a9cb383d | ||
|
|
4343073c00 | ||
|
|
fe60d4be88 | ||
|
|
ae337807f5 | ||
|
|
9ae30ac08e | ||
|
|
62fa85c0a4 | ||
|
|
7bb873dad9 | ||
|
|
5f6c1c14cb | ||
|
|
d43189ad33 | ||
|
|
fcad76fc51 | ||
|
|
97e6e1684e | ||
|
|
67a0d3e926 | ||
|
|
183fb9f9ff | ||
|
|
9815ddef1f | ||
|
|
f6d0847453 | ||
|
|
b0b9f0d65f | ||
|
|
0cec80f676 | ||
|
|
48c64143e3 | ||
|
|
a8712422bc | ||
|
|
97f65bd283 | ||
|
|
fd3c1c50f1 | ||
|
|
b153e4bb9f | ||
|
|
db9856a8db | ||
|
|
75ecea265d | ||
|
|
be8751cb73 | ||
|
|
fb25ecb4a1 | ||
|
|
f1cb7d27ac | ||
|
|
dee494e12f | ||
|
|
b13a121915 | ||
|
|
7486ee9537 | ||
|
|
4a20ccc28e | ||
|
|
f80dd2b307 | ||
|
|
b208cf6d32 | ||
|
|
39e78ff17e | ||
|
|
a8177ea7fe | ||
|
|
bedcfc154b | ||
|
|
4c38b4aa3c | ||
|
|
f6cfe266e0 | ||
|
|
4905e65f14 | ||
|
|
ccb250b410 | ||
|
|
aca57ffc62 | ||
|
|
7f375f42d8 | ||
|
|
eedcc2034d | ||
|
|
24c9a167d7 | ||
|
|
909df8ef1f | ||
|
|
3b27cb3671 | ||
|
|
3fe0db4a7d | ||
|
|
8b55814ab2 | ||
|
|
575e471553 | ||
|
|
0f5f1aebed | ||
|
|
50e17eb1ab | ||
|
|
158cc2f660 | ||
|
|
1066a31acd | ||
|
|
1f9d0f4582 | ||
|
|
a6d65233f1 | ||
|
|
eff2fe7a1b | ||
|
|
20efdc70b3 | ||
|
|
f0d8fb8f1a | ||
|
|
f7a380e9b7 | ||
|
|
e9c7cf6f63 | ||
|
|
68f1661452 | ||
|
|
36fd1b91ae | ||
|
|
a4ec430ac0 | ||
|
|
519614b2fd | ||
|
|
bf0118c8ef | ||
|
|
a4db0820bc | ||
|
|
ee7528413e | ||
|
|
7952fc8324 | ||
|
|
652773d2cf | ||
|
|
2a17787242 | ||
|
|
0a53ad5721 | ||
|
|
6da6bdc863 | ||
|
|
42ad2bb83f | ||
|
|
f309a65cb4 | ||
|
|
77e19ab1a4 | ||
|
|
1a996b6ef3 | ||
|
|
b882f1a010 | ||
|
|
82a030e6ff | ||
|
|
0758b85179 | ||
|
|
ab3d9bd080 | ||
|
|
8ff813f689 | ||
|
|
88ae72c0d3 | ||
|
|
312aa4be26 | ||
|
|
cbb06fce9d | ||
|
|
f259e497c4 | ||
|
|
dd4172ac66 | ||
|
|
66029e60d3 | ||
|
|
364f484f04 | ||
|
|
9dd5159414 | ||
|
|
13e38d6fd8 | ||
|
|
10dcb64715 | ||
|
|
7551b51e7d | ||
|
|
adb418aafc | ||
|
|
270da80d64 | ||
|
|
b2027cfd66 | ||
|
|
7f1f4eeac6 | ||
|
|
7a7446c8bd | ||
|
|
ddbae294e6 | ||
|
|
8c71b744f3 | ||
|
|
479b5d31a9 | ||
|
|
4cbf4230e8 | ||
|
|
6a610187e0 | ||
|
|
eb2a4aebba | ||
|
|
21a2e67755 | ||
|
|
3b9e312615 | ||
|
|
26dab04c9e | ||
|
|
00713c0d11 | ||
|
|
751b5f3027 | ||
|
|
e8261b000e | ||
|
|
41ecb70297 | ||
|
|
09ee104b8c | ||
|
|
e3a4964787 | ||
|
|
9bf72910a4 | ||
|
|
ee39e20e6d | ||
|
|
399d6db6f6 | ||
|
|
0821ce44b5 | ||
|
|
ea279111c6 | ||
|
|
674ce02e58 | ||
|
|
8dfa2767ec | ||
|
|
20dad7f07f | ||
|
|
751cc4c44d | ||
|
|
2318e6d8e9 | ||
|
|
61b4a492c3 | ||
|
|
9db3d01e09 | ||
|
|
8da73ad3dd | ||
|
|
b8c16d8ac5 | ||
|
|
429c0951f3 | ||
|
|
74e103c791 | ||
|
|
f941950ee2 | ||
|
|
846df2eef1 | ||
|
|
34ed058c97 | ||
|
|
eae0290978 | ||
|
|
561368570e | ||
|
|
3467d1fed0 | ||
|
|
d02ff232e5 | ||
|
|
2d1c6a5402 | ||
|
|
eab3b65629 | ||
|
|
20b435732a | ||
|
|
929617273d | ||
|
|
2717bf7d49 | ||
|
|
5cd2ebc960 | ||
|
|
9b4afe9816 | ||
|
|
23bb5598d5 | ||
|
|
af1d7813e9 | ||
|
|
16c2e5a585 | ||
|
|
c02750edbd | ||
|
|
7204ddafec | ||
|
|
faeba9a7e4 | ||
|
|
190d238a1f | ||
|
|
715451b5fb | ||
|
|
87f1895405 | ||
|
|
923d817751 | ||
|
|
0728209b66 | ||
|
|
b8b9dcc2ee | ||
|
|
f35e879852 | ||
|
|
34f4f12eb9 | ||
|
|
fa63bf758d | ||
|
|
f6b396ae64 | ||
|
|
2c7fd58e34 | ||
|
|
982f2c9634 | ||
|
|
f2fd42b47a | ||
|
|
1b4ccad938 | ||
|
|
a9de1ce8e0 | ||
|
|
ac752d5ec2 | ||
|
|
632d8d02d2 | ||
|
|
48aeb26e02 | ||
|
|
1694af8b5e | ||
|
|
83bcab9cd2 | ||
|
|
bdc7f4b3f5 | ||
|
|
39202a3d79 | ||
|
|
912065a121 | ||
|
|
c8466afac2 | ||
|
|
2619e162c1 | ||
|
|
e1112e17f8 | ||
|
|
92b2ead74c | ||
|
|
bbed9b94c1 | ||
|
|
73d07311db | ||
|
|
1cdff47477 | ||
|
|
511a5c3f82 | ||
|
|
853885e2ff | ||
|
|
d83936a66a | ||
|
|
5517d2bf56 | ||
|
|
f21ab49ac5 | ||
|
|
925a458abe | ||
|
|
76946c447f | ||
|
|
2faa29b1c4 | ||
|
|
6826cc311d | ||
|
|
5e17ce0a0b | ||
|
|
e8d299d3b6 | ||
|
|
7637aa2ab6 | ||
|
|
ab067d1d3a | ||
|
|
e6f84666c7 | ||
|
|
4c5429af15 | ||
|
|
0a0ac3b7c9 | ||
|
|
24833ce9fb | ||
|
|
ec2c18dc87 | ||
|
|
7384609e74 | ||
|
|
3047649650 | ||
|
|
d298dac3f3 | ||
|
|
1574bca8a8 | ||
|
|
ec2f6c6b80 | ||
|
|
838cc60161 | ||
|
|
310c61a5cc | ||
|
|
318cc15323 | ||
|
|
3a64ceb4d6 | ||
|
|
d0f21c0095 | ||
|
|
46dc15dd29 | ||
|
|
8dc654b513 | ||
|
|
7000ac3f3f | ||
|
|
43c2e8d8e9 | ||
|
|
0231139b01 | ||
|
|
d6ee6446dd | ||
|
|
7b666efcf8 | ||
|
|
eb5d2198fc | ||
|
|
34e44ebd1c | ||
|
|
bf2f4bc040 | ||
|
|
9dc4559aba | ||
|
|
eba8856261 | ||
|
|
a8c5aa471a | ||
|
|
52f6dcf092 | ||
|
|
dec79f3742 | ||
|
|
8bdcac0f3e | ||
|
|
8426b674a3 | ||
|
|
da391bcc8d | ||
|
|
2d7443d454 | ||
|
|
991987ed76 | ||
|
|
ec24ebf2cf | ||
|
|
6ba0976085 | ||
|
|
2b88d01a01 | ||
|
|
0c09f24cbf | ||
|
|
61d22afeba | ||
|
|
9f1ed6e8c3 | ||
|
|
bbc4113cac | ||
|
|
91194bf422 | ||
|
|
9c5f940b00 | ||
|
|
455b4043b8 | ||
|
|
bd83ee7931 | ||
|
|
f6bdf7c09a | ||
|
|
2db8afb8c2 | ||
|
|
e033129dd3 | ||
|
|
f9dc590100 | ||
|
|
8996aafe0d | ||
|
|
9dc25ef7af | ||
|
|
2b84f4d407 | ||
|
|
097c8b674c | ||
|
|
ba649d4b94 | ||
|
|
6ed1614db0 | ||
|
|
df5b6a8380 | ||
|
|
1f82a8b99e | ||
|
|
0c95d96f32 | ||
|
|
c2f5f84118 | ||
|
|
b3b5e3d8f0 | ||
|
|
506d3f3cd9 | ||
|
|
516a8c5ee5 | ||
|
|
2f81e9d374 | ||
|
|
2d8703bb8d | ||
|
|
76e60d9bc3 | ||
|
|
9d5370be5f | ||
|
|
fc1a06bc45 | ||
|
|
fce80374f4 | ||
|
|
420c9f10c2 | ||
|
|
5a39681a2e | ||
|
|
7a1b7db7c8 | ||
|
|
03a643da52 | ||
|
|
383f0ce450 | ||
|
|
c6c668676c | ||
|
|
7b01dde063 | ||
|
|
8c25f65024 | ||
|
|
1e478e3545 | ||
|
|
6c75a8978b | ||
|
|
640267fc2b | ||
|
|
33c7c3ee12 | ||
|
|
f9b41d34ae | ||
|
|
d4bec79451 | ||
|
|
ac1a8b4daf | ||
|
|
28838c1759 | ||
|
|
50ecb0dac9 | ||
|
|
e22bc01cbd | ||
|
|
6c28713a4c | ||
|
|
fc9023386c | ||
|
|
e6cae9bcc3 | ||
|
|
a9eeca1302 | ||
|
|
8c695e42ca | ||
|
|
0aa7fd47d5 | ||
|
|
70596042d6 | ||
|
|
caf616234b | ||
|
|
375187aa70 | ||
|
|
71eccdc0e3 | ||
|
|
639b1e48f5 | ||
|
|
0bb4cb4472 | ||
|
|
cc51543851 | ||
|
|
22540390e1 | ||
|
|
98565eb67c | ||
|
|
cf6a47ecb7 | ||
|
|
fa60b9f9d3 | ||
|
|
644120ca31 | ||
|
|
a50a625b3b | ||
|
|
7297c13331 | ||
|
|
f73399bfac | ||
|
|
a056cd78f7 | ||
|
|
a30f3c86c2 | ||
|
|
e70bdd86a7 | ||
|
|
bc9f33c2e0 | ||
|
|
872b89ee93 | ||
|
|
ae53062518 | ||
|
|
17f76c9cb3 | ||
|
|
5de944146a | ||
|
|
9dc6092cb0 | ||
|
|
a32a2ef04e | ||
|
|
ecfa0ae3da | ||
|
|
03595052ce | ||
|
|
1f94c53dd2 | ||
|
|
9c426373f2 | ||
|
|
c03e30a01f | ||
|
|
07f7a77ac0 | ||
|
|
3b9ea2c9a4 | ||
|
|
1beb13dd80 | ||
|
|
ddae84abb3 | ||
|
|
863b9a2c98 | ||
|
|
9d44bbdb48 | ||
|
|
8d93dd5adc | ||
|
|
48502961cf | ||
|
|
3c8bec61d3 | ||
|
|
7296c7df1a | ||
|
|
f3ee6603de | ||
|
|
ee0aabda1d | ||
|
|
08d37a4b0f | ||
|
|
f975ea99cb | ||
|
|
f030d41dc7 | ||
|
|
8d079bfcd1 | ||
|
|
d39b8654a6 | ||
|
|
ce7816a968 | ||
|
|
2ee572e68f | ||
|
|
4bbd850898 | ||
|
|
34eae6e608 | ||
|
|
6a0302fec6 | ||
|
|
c94cc293c2 | ||
|
|
cae7792a1e | ||
|
|
7f6b2fe4f1 | ||
|
|
eba430bbc0 | ||
|
|
01280c8d04 | ||
|
|
ec141ac9c9 | ||
|
|
590c892a6a | ||
|
|
ff8a50c366 | ||
|
|
46ef6c8ab7 | ||
|
|
b09eabc478 | ||
|
|
e49fcea6e3 | ||
|
|
68ed9f4ffc | ||
|
|
af94687d45 | ||
|
|
26964ecf0f | ||
|
|
77d19af359 | ||
|
|
120b82f243 | ||
|
|
a0fea30a11 | ||
|
|
8bb3e0a64d | ||
|
|
bbded12923 | ||
|
|
e8ba5d7606 | ||
|
|
af66b31a44 | ||
|
|
b000a40f28 | ||
|
|
3c85322523 | ||
|
|
a469b8bc04 | ||
|
|
78b8261a3a | ||
|
|
f20ca70c01 | ||
|
|
4e4148fc1c | ||
|
|
c22482f907 | ||
|
|
870e139fce | ||
|
|
4d58648c02 | ||
|
|
ebbb182537 | ||
|
|
1cd5377b45 | ||
|
|
1d1f0527ee | ||
|
|
37a5f77415 | ||
|
|
ced3c7efe4 | ||
|
|
c3b8f366ed | ||
|
|
de78876b1a | ||
|
|
64c25b049c | ||
|
|
8a4fe4f3ad | ||
|
|
8811b8c1fd | ||
|
|
190b01fdf9 | ||
|
|
f145d08c10 | ||
|
|
53382b7e15 | ||
|
|
6ad0242617 | ||
|
|
a7c2408c0a | ||
|
|
ce1eabaed6 | ||
|
|
f602541ede | ||
|
|
3f718e6efc | ||
|
|
ce7a985df6 | ||
|
|
6d83f7e7bd | ||
|
|
b73c00943c | ||
|
|
abaac8ef48 | ||
|
|
a2f8e7068e | ||
|
|
4d47873219 | ||
|
|
cf985486e5 | ||
|
|
b930c3fc93 | ||
|
|
dd26a96828 | ||
|
|
6865ddfc12 | ||
|
|
e888e69d4d | ||
|
|
2089059792 | ||
|
|
27739e0364 | ||
|
|
698ee271d6 | ||
|
|
543c75b293 | ||
|
|
b09c46f6f7 | ||
|
|
f2cc19e6aa | ||
|
|
814a0a123f | ||
|
|
179383540f | ||
|
|
8be1c8310d | ||
|
|
ef02d776ca | ||
|
|
750ff448ad | ||
|
|
e3abab6d4d | ||
|
|
d3ffae72fb | ||
|
|
87f751188e | ||
|
|
3469abaefd | ||
|
|
797364ee5c | ||
|
|
36c05fc4b9 | ||
|
|
79624febc0 | ||
|
|
0a9d4de126 | ||
|
|
b6f6994db4 | ||
|
|
ff7bed720a | ||
|
|
00c0f48b02 | ||
|
|
94c45cf2a0 | ||
|
|
58f77b2a1c | ||
|
|
7170e69b22 | ||
|
|
239ee2437c | ||
|
|
ced368db31 | ||
|
|
6a991e5c15 | ||
|
|
523d22262b | ||
|
|
08197a327e | ||
|
|
b7cb2a7aa5 | ||
|
|
decc5c74ef | ||
|
|
fbe0f886b6 | ||
|
|
7e23d6e2ef | ||
|
|
49458d1085 | ||
|
|
289c12bd9a | ||
|
|
9a6326b027 | ||
|
|
51f573f1ea | ||
|
|
c8b7cd8862 | ||
|
|
21c112d059 | ||
|
|
9432e1b5b2 | ||
|
|
8269d2f83c | ||
|
|
5a4b6be974 | ||
|
|
35e8ce60a9 | ||
|
|
8b6dd9f603 | ||
|
|
084f0ebdab | ||
|
|
58640c1521 | ||
|
|
7ffdfd12f8 | ||
|
|
cb9a30f285 | ||
|
|
e48bef809f | ||
|
|
f5d7570102 | ||
|
|
a600c60cf8 | ||
|
|
695eabd026 | ||
|
|
e81b51a647 | ||
|
|
b7e95ff090 | ||
|
|
3ca41be686 | ||
|
|
3152046173 | ||
|
|
3a98dc8a95 | ||
|
|
d737b28916 | ||
|
|
97c0dac876 | ||
|
|
006494b1fa | ||
|
|
149d343201 | ||
|
|
496cc79ba8 | ||
|
|
3591768745 | ||
|
|
d615f6761a | ||
|
|
5b29894048 | ||
|
|
d17f0cfa40 | ||
|
|
b2f70c7120 | ||
|
|
238707bd93 | ||
|
|
87cdf1e3a0 | ||
|
|
57fd47022e | ||
|
|
5eafd3bf6b | ||
|
|
640b5b0ea9 | ||
|
|
e4fb2f4680 | ||
|
|
a4909f54e4 | ||
|
|
f7065c6f0c | ||
|
|
64f7f837e7 | ||
|
|
130d36acd9 | ||
|
|
e94c28cfa2 | ||
|
|
a697dec30c | ||
|
|
bac09e9b9f | ||
|
|
b179f8f86a | ||
|
|
962d04ae17 | ||
|
|
f28e3bfe37 | ||
|
|
6474c86d32 | ||
|
|
769ea8cdfe | ||
|
|
052e77dd5a | ||
|
|
1a4b0cb12a | ||
|
|
f9aa95c987 | ||
|
|
a5d06fb4a4 | ||
|
|
d5e5d90bdc | ||
|
|
e62678e4e6 | ||
|
|
558ac24f7e | ||
|
|
6f22f6a59f | ||
|
|
5ec74f8abe | ||
|
|
16d7547e03 | ||
|
|
767112dcda | ||
|
|
81e23d1d8c | ||
|
|
ace90a4354 | ||
|
|
267e7096cc | ||
|
|
e7cded7511 | ||
|
|
33fbd715c0 | ||
|
|
25ba5b867c | ||
|
|
c8f431447c | ||
|
|
1b83a1d09a | ||
|
|
d2fb7a7151 | ||
|
|
7ad3447598 | ||
|
|
5e5bc8e705 | ||
|
|
886d1e8a19 | ||
|
|
6de7a035fa | ||
|
|
48d2497eb2 | ||
|
|
6494754ab9 | ||
|
|
a8153627c6 | ||
|
|
ed38966edb | ||
|
|
85551d1e54 | ||
|
|
ca31363180 | ||
|
|
a8f11eb3c3 | ||
|
|
151805121c | ||
|
|
0422d2a021 | ||
|
|
8586762dde | ||
|
|
ffdf70257b | ||
|
|
6079d60aa1 | ||
|
|
d46d012f8c | ||
|
|
baf926b360 | ||
|
|
82339869dc | ||
|
|
f70343bb63 | ||
|
|
4ab56ae3d1 | ||
|
|
39b4031684 | ||
|
|
71f8965393 | ||
|
|
66857e72a4 | ||
|
|
15eb5d47eb | ||
|
|
86242139da | ||
|
|
460bdc4148 | ||
|
|
1052126522 | ||
|
|
fdb42ac876 | ||
|
|
6486f986e8 | ||
|
|
bc4075e2ed | ||
|
|
11525c7d0d | ||
|
|
bb12ae9ce6 | ||
|
|
66b4ad0c44 | ||
|
|
4bff2d718e | ||
|
|
af7a7b4dc6 | ||
|
|
ec20d9a2a8 | ||
|
|
b0aee3d335 | ||
|
|
5e2ddbfd86 | ||
|
|
f086ebbb8e | ||
|
|
e5bf6a5bfc | ||
|
|
76fa86d54b | ||
|
|
60d89506a5 | ||
|
|
2893f1eb9e | ||
|
|
a08cbf412d | ||
|
|
0bd65356f9 | ||
|
|
2e2b39455f | ||
|
|
1f9fbe34e4 | ||
|
|
506d7ae50e | ||
|
|
342cb00625 | ||
|
|
952fc01efd | ||
|
|
3eebe301fe | ||
|
|
19c0a81c42 | ||
|
|
9fc7f9904b | ||
|
|
b04a516063 | ||
|
|
241e946d91 | ||
|
|
e45375dc26 | ||
|
|
ec8e243323 | ||
|
|
af0e9368d4 | ||
|
|
285483b81a | ||
|
|
73a91d5569 | ||
|
|
e24c78be32 | ||
|
|
b11184de68 | ||
|
|
61d02fc5d7 | ||
|
|
f0217f6821 | ||
|
|
09644914a6 | ||
|
|
b5c24d6a48 | ||
|
|
bdd2ac2c25 | ||
|
|
76376f0d33 | ||
|
|
9749da46ae | ||
|
|
1c05295e89 | ||
|
|
4d0f28215a | ||
|
|
0bba72bc5a | ||
|
|
87089b8e83 | ||
|
|
6a58148a89 | ||
|
|
15091052be | ||
|
|
5cd7e9a9b8 | ||
|
|
7cb530c334 | ||
|
|
5e0324cc91 | ||
|
|
6293e227ea | ||
|
|
d6d83a5c76 | ||
|
|
93fc883b90 | ||
|
|
dd21803598 | ||
|
|
af2f2282c2 | ||
|
|
869a76c9bb | ||
|
|
44b2964a6a | ||
|
|
8fddb57e0a | ||
|
|
c7a5a0cab0 | ||
|
|
c13aab3ffc | ||
|
|
95a9a76598 | ||
|
|
923cfa3d50 | ||
|
|
2c489168c2 | ||
|
|
f3292b4d34 | ||
|
|
9769829b72 | ||
|
|
36243d15cc | ||
|
|
f4645f570c | ||
|
|
61603ccfce | ||
|
|
a26cfc639c | ||
|
|
90602931d8 | ||
|
|
d1b330028c | ||
|
|
7a6b1930bf | ||
|
|
c271361552 | ||
|
|
17789bc814 | ||
|
|
d7c16e161a | ||
|
|
e71c17d7e7 | ||
|
|
17d87eb157 | ||
|
|
bf11bcc084 | ||
|
|
bd291109df | ||
|
|
ac89ba9b8d | ||
|
|
5ab634e375 | ||
|
|
bb1978d976 | ||
|
|
bea94ce8ac | ||
|
|
9561f93594 | ||
|
|
d44d5c3304 | ||
|
|
ec541e2057 | ||
|
|
8169bf6b97 | ||
|
|
56c8ad1221 | ||
|
|
6814f2e38c | ||
|
|
5f043b9a78 | ||
|
|
c9092cd1c7 | ||
|
|
86cc65d894 | ||
|
|
8ef2647fa9 | ||
|
|
3ff9e99416 | ||
|
|
1731046011 | ||
|
|
c3d96184b6 | ||
|
|
c255c606a7 | ||
|
|
cc69faa1fd | ||
|
|
0f53cd86c8 | ||
|
|
212c8f188d | ||
|
|
dec248adec | ||
|
|
98e6a066f4 | ||
|
|
c90db54a3d | ||
|
|
927d156933 | ||
|
|
c71f91073f | ||
|
|
ab28387692 | ||
|
|
c4905d02b9 | ||
|
|
ee39880fb5 | ||
|
|
926b3e56b9 | ||
|
|
e060df5367 | ||
|
|
a3294d4a0d | ||
|
|
19ce0ab246 | ||
|
|
75a1cc0d33 | ||
|
|
05228529b0 | ||
|
|
3cbeadfbc3 | ||
|
|
3d4938c0e2 | ||
|
|
1b850b8a2b | ||
|
|
ebd33f1869 | ||
|
|
ac846b4df3 | ||
|
|
0d427c9b90 | ||
|
|
dc6194f862 | ||
|
|
de50214e1f | ||
|
|
5ec4a88c35 | ||
|
|
31c141e757 | ||
|
|
9d5760d899 | ||
|
|
7945e1ea3c | ||
|
|
d94c40e371 | ||
|
|
31f22122e8 | ||
|
|
506582aa2b | ||
|
|
a02b251c9b | ||
|
|
34cbb6fa79 | ||
|
|
33679b56fd | ||
|
|
8897c4d560 | ||
|
|
f0c18ccbe7 | ||
|
|
e560d83c51 | ||
|
|
eca7addc67 | ||
|
|
263f3ba5c9 | ||
|
|
f514123ef0 | ||
|
|
9f461db0d0 | ||
|
|
7de8670616 | ||
|
|
8c7908b200 | ||
|
|
e7722e039f | ||
|
|
039bc0208a | ||
|
|
d57900a069 | ||
|
|
c1153302aa | ||
|
|
1086a3297f | ||
|
|
5c613934ca | ||
|
|
6d7e9092f8 |
No files matched your search
+145
-22
@@ -1,31 +1,74 @@
|
|||||||
name: ci
|
name: ci
|
||||||
on: pull_request
|
on: ["push", "pull_request"]
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
|
|
||||||
lint:
|
lint:
|
||||||
name: lint
|
name: lint
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v1
|
- uses: actions/checkout@v2
|
||||||
- uses: actions/setup-python@v1
|
- uses: actions/setup-python@v1
|
||||||
with:
|
with:
|
||||||
python-version: '3.7'
|
python-version: '3.7'
|
||||||
- run: make install tools
|
- name: extract pip cache
|
||||||
|
uses: actions/cache@v2
|
||||||
|
with:
|
||||||
|
path: ~/.cache/pip
|
||||||
|
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
|
||||||
|
restore-keys: ${{ runner.os }}-pip-
|
||||||
|
- run: pip install --user --upgrade pip wheel
|
||||||
|
- run: pip install -e .[torrent,lint]
|
||||||
- run: make lint
|
- run: make lint
|
||||||
|
|
||||||
tests-unit:
|
tests-unit:
|
||||||
name: "tests / unit"
|
name: "tests / unit"
|
||||||
runs-on: ubuntu-latest
|
strategy:
|
||||||
|
matrix:
|
||||||
|
os:
|
||||||
|
- ubuntu-latest
|
||||||
|
- macos-latest
|
||||||
|
- windows-latest
|
||||||
|
runs-on: ${{ matrix.os }}
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v1
|
- uses: actions/checkout@v2
|
||||||
- uses: actions/setup-python@v1
|
- uses: actions/setup-python@v1
|
||||||
with:
|
with:
|
||||||
python-version: '3.7'
|
python-version: '3.7'
|
||||||
- run: make install tools
|
- name: set pip cache dir
|
||||||
- working-directory: lbry
|
id: pip-cache
|
||||||
|
run: echo "::set-output name=dir::$(pip cache dir)"
|
||||||
|
- name: extract pip cache
|
||||||
|
uses: actions/cache@v2
|
||||||
|
with:
|
||||||
|
path: ${{ steps.pip-cache.outputs.dir }}
|
||||||
|
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
|
||||||
|
restore-keys: ${{ runner.os }}-pip-
|
||||||
|
- id: os-name
|
||||||
|
uses: ASzc/change-string-case-action@v1
|
||||||
|
with:
|
||||||
|
string: ${{ runner.os }}
|
||||||
|
- run: pip install --user --upgrade pip wheel
|
||||||
|
- if: startsWith(runner.os, 'linux')
|
||||||
|
run: pip install -e .[torrent,test]
|
||||||
|
- if: startsWith(runner.os, 'linux')
|
||||||
env:
|
env:
|
||||||
HOME: /tmp
|
HOME: /tmp
|
||||||
run: coverage run -p --source=lbry -m unittest discover -vv tests.unit
|
run: make test-unit-coverage
|
||||||
|
- if: startsWith(runner.os, 'linux') != true
|
||||||
|
run: pip install -e .[test]
|
||||||
|
- if: startsWith(runner.os, 'linux') != true
|
||||||
|
env:
|
||||||
|
HOME: /tmp
|
||||||
|
run: coverage run --source=lbry -m unittest tests/unit/test_conf.py
|
||||||
|
- name: submit coverage report
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
COVERALLS_FLAG_NAME: tests-unit-${{ steps.os-name.outputs.lowercase }}
|
||||||
|
COVERALLS_PARALLEL: true
|
||||||
|
run: |
|
||||||
|
pip install coveralls
|
||||||
|
coveralls --service=github
|
||||||
|
|
||||||
tests-integration:
|
tests-integration:
|
||||||
name: "tests / integration"
|
name: "tests / integration"
|
||||||
@@ -35,45 +78,125 @@ jobs:
|
|||||||
test:
|
test:
|
||||||
- datanetwork
|
- datanetwork
|
||||||
- blockchain
|
- blockchain
|
||||||
|
- blockchain_legacy_search
|
||||||
- other
|
- other
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v1
|
- name: Configure sysctl limits
|
||||||
|
run: |
|
||||||
|
sudo swapoff -a
|
||||||
|
sudo sysctl -w vm.swappiness=1
|
||||||
|
sudo sysctl -w fs.file-max=262144
|
||||||
|
sudo sysctl -w vm.max_map_count=262144
|
||||||
|
- name: Runs Elasticsearch
|
||||||
|
uses: elastic/elastic-github-actions/elasticsearch@master
|
||||||
|
with:
|
||||||
|
stack-version: 7.6.0
|
||||||
|
- uses: actions/checkout@v2
|
||||||
- uses: actions/setup-python@v1
|
- uses: actions/setup-python@v1
|
||||||
with:
|
with:
|
||||||
python-version: '3.7'
|
python-version: '3.7'
|
||||||
- if: matrix.test == 'other'
|
- if: matrix.test == 'other'
|
||||||
run: sudo apt install -y --no-install-recommends ffmpeg
|
run: |
|
||||||
- run: pip install tox-travis
|
sudo apt-get update
|
||||||
|
sudo apt-get install -y --no-install-recommends ffmpeg
|
||||||
|
- name: extract pip cache
|
||||||
|
uses: actions/cache@v2
|
||||||
|
with:
|
||||||
|
path: ./.tox
|
||||||
|
key: tox-integration-${{ matrix.test }}-${{ hashFiles('setup.py') }}
|
||||||
|
restore-keys: txo-integration-${{ matrix.test }}-
|
||||||
|
- run: pip install tox coverage coveralls
|
||||||
- run: tox -e ${{ matrix.test }}
|
- run: tox -e ${{ matrix.test }}
|
||||||
|
- name: submit coverage report
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
COVERALLS_FLAG_NAME: tests-integration-${{ matrix.test }}
|
||||||
|
COVERALLS_PARALLEL: true
|
||||||
|
run: |
|
||||||
|
coverage combine tests
|
||||||
|
coveralls --service=github
|
||||||
|
|
||||||
|
|
||||||
|
coverage:
|
||||||
|
needs: ["tests-unit", "tests-integration"]
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: finalize coverage report submission
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
run: |
|
||||||
|
pip install coveralls
|
||||||
|
coveralls --service=github --finish
|
||||||
|
|
||||||
build:
|
build:
|
||||||
needs: ["lint", "tests-unit", "tests-integration"]
|
needs: ["lint", "tests-unit", "tests-integration"]
|
||||||
name: "build"
|
name: "build / binary"
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
os:
|
os:
|
||||||
- ubuntu-latest
|
- ubuntu-18.04
|
||||||
- macos-latest
|
- macos-latest
|
||||||
- windows-latest
|
- windows-latest
|
||||||
runs-on: ${{ matrix.os }}
|
runs-on: ${{ matrix.os }}
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v1
|
- uses: actions/checkout@v2
|
||||||
- uses: actions/setup-python@v1
|
- uses: actions/setup-python@v1
|
||||||
with:
|
with:
|
||||||
python-version: '3.7'
|
python-version: '3.7'
|
||||||
- name: Setup
|
- id: os-name
|
||||||
run: |
|
uses: ASzc/change-string-case-action@v1
|
||||||
pip install pyinstaller
|
with:
|
||||||
pip install -e .
|
string: ${{ runner.os }}
|
||||||
- if: startsWith(matrix.os, 'windows') == false
|
- name: set pip cache dir
|
||||||
|
id: pip-cache
|
||||||
|
run: echo "::set-output name=dir::$(pip cache dir)"
|
||||||
|
- name: extract pip cache
|
||||||
|
uses: actions/cache@v2
|
||||||
|
with:
|
||||||
|
path: ${{ steps.pip-cache.outputs.dir }}
|
||||||
|
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
|
||||||
|
restore-keys: ${{ runner.os }}-pip-
|
||||||
|
- run: pip install pyinstaller==4.4
|
||||||
|
- run: pip install -e .
|
||||||
|
- if: startsWith(github.ref, 'refs/tags/v')
|
||||||
|
run: python docker/set_build.py
|
||||||
|
- if: startsWith(runner.os, 'linux') || startsWith(runner.os, 'mac')
|
||||||
name: Build & Run (Unix)
|
name: Build & Run (Unix)
|
||||||
run: |
|
run: |
|
||||||
pyinstaller --onefile --name lbrynet lbry/extras/cli.py
|
pyinstaller --onefile --name lbrynet lbry/extras/cli.py
|
||||||
chmod +x dist/lbrynet
|
|
||||||
dist/lbrynet --version
|
dist/lbrynet --version
|
||||||
- if: startsWith(matrix.os, 'windows')
|
- if: startsWith(runner.os, 'windows')
|
||||||
name: Build & Run (Windows)
|
name: Build & Run (Windows)
|
||||||
run: |
|
run: |
|
||||||
pip install pywin32
|
pip install pywin32==301
|
||||||
pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico --onefile --name lbrynet lbry/extras/cli.py
|
pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico --onefile --name lbrynet lbry/extras/cli.py
|
||||||
dist/lbrynet.exe --version
|
dist/lbrynet.exe --version
|
||||||
|
- uses: actions/upload-artifact@v2
|
||||||
|
with:
|
||||||
|
name: lbrynet-${{ steps.os-name.outputs.lowercase }}
|
||||||
|
path: dist/
|
||||||
|
|
||||||
|
release:
|
||||||
|
name: "release"
|
||||||
|
if: startsWith(github.ref, 'refs/tags/v')
|
||||||
|
needs: ["build"]
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v1
|
||||||
|
- uses: actions/download-artifact@v2
|
||||||
|
- name: upload binaries
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ secrets.RELEASE_API_TOKEN }}
|
||||||
|
run: |
|
||||||
|
pip install githubrelease
|
||||||
|
chmod +x lbrynet-macos/lbrynet
|
||||||
|
chmod +x lbrynet-linux/lbrynet
|
||||||
|
zip --junk-paths lbrynet-mac.zip lbrynet-macos/lbrynet
|
||||||
|
zip --junk-paths lbrynet-linux.zip lbrynet-linux/lbrynet
|
||||||
|
zip --junk-paths lbrynet-windows.zip lbrynet-windows/lbrynet.exe
|
||||||
|
ls -lh
|
||||||
|
githubrelease release lbryio/lbry-sdk info ${GITHUB_REF#refs/tags/}
|
||||||
|
githubrelease asset lbryio/lbry-sdk upload ${GITHUB_REF#refs/tags/} \
|
||||||
|
lbrynet-mac.zip lbrynet-linux.zip lbrynet-windows.zip
|
||||||
|
githubrelease release lbryio/lbry-sdk publish ${GITHUB_REF#refs/tags/}
|
||||||
|
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
name: slack
|
||||||
|
|
||||||
|
on:
|
||||||
|
release:
|
||||||
|
types: [published]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
release:
|
||||||
|
name: "slack notification"
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: LoveToKnow/slackify-markdown-action@v1.0.0
|
||||||
|
id: markdown
|
||||||
|
with:
|
||||||
|
text: "There is a new SDK release: ${{github.event.release.html_url}}\n${{ github.event.release.body }}"
|
||||||
|
- uses: slackapi/slack-github-action@v1.14.0
|
||||||
|
env:
|
||||||
|
CHANGELOG: '<!channel> ${{ steps.markdown.outputs.text }}'
|
||||||
|
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_RELEASE_BOT_WEBHOOK }}
|
||||||
|
with:
|
||||||
|
payload: '{"type": "mrkdwn", "text": ${{ toJSON(env.CHANGELOG) }} }'
|
||||||
|
|
||||||
+6
-1
@@ -6,12 +6,17 @@
|
|||||||
/.coverage*
|
/.coverage*
|
||||||
/lbry-venv
|
/lbry-venv
|
||||||
/venv
|
/venv
|
||||||
|
/lbry/blockchain
|
||||||
|
|
||||||
lbry.egg-info
|
lbry.egg-info
|
||||||
__pycache__
|
__pycache__
|
||||||
_trial_temp/
|
_trial_temp/
|
||||||
|
trending*.log
|
||||||
|
|
||||||
/tests/integration/blockchain/files
|
/tests/integration/blockchain/files
|
||||||
/tests/.coverage.*
|
/tests/.coverage.*
|
||||||
|
|
||||||
/lbry/wallet/bin
|
/lbry/wallet/bin
|
||||||
|
|
||||||
|
/.vscode
|
||||||
|
/.gitignore
|
||||||
-210
@@ -1,210 +0,0 @@
|
|||||||
default:
|
|
||||||
image: python:3.7
|
|
||||||
|
|
||||||
|
|
||||||
#cache:
|
|
||||||
# directories:
|
|
||||||
# - $HOME/venv
|
|
||||||
# - $HOME/.cache/pip
|
|
||||||
# - $HOME/Library/Caches/pip
|
|
||||||
# - $HOME/Library/Caches/Homebrew
|
|
||||||
# - $TRAVIS_BUILD_DIR/.tox
|
|
||||||
|
|
||||||
|
|
||||||
stages:
|
|
||||||
- test
|
|
||||||
- build
|
|
||||||
- assets
|
|
||||||
- release
|
|
||||||
|
|
||||||
|
|
||||||
.tagged:
|
|
||||||
rules:
|
|
||||||
- if: '$CI_COMMIT_TAG =~ /^v[0-9\.]+$/'
|
|
||||||
when: on_success
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
test:lint:
|
|
||||||
stage: test
|
|
||||||
script:
|
|
||||||
- make install tools
|
|
||||||
- make lint
|
|
||||||
|
|
||||||
test:unit:
|
|
||||||
stage: test
|
|
||||||
script:
|
|
||||||
- make install tools
|
|
||||||
- HOME=/tmp coverage run -p --source=lbry -m unittest discover -vv tests.unit
|
|
||||||
|
|
||||||
test:datanetwork-integration:
|
|
||||||
stage: test
|
|
||||||
script:
|
|
||||||
- pip install tox-travis
|
|
||||||
- tox -e datanetwork
|
|
||||||
|
|
||||||
test:blockchain-integration:
|
|
||||||
stage: test
|
|
||||||
script:
|
|
||||||
- pip install tox-travis
|
|
||||||
- tox -e blockchain
|
|
||||||
|
|
||||||
test:other-integration:
|
|
||||||
stage: test
|
|
||||||
script:
|
|
||||||
- apt-get update
|
|
||||||
- apt-get install -y --no-install-recommends ffmpeg
|
|
||||||
- pip install tox-travis
|
|
||||||
- tox -e other
|
|
||||||
|
|
||||||
test:json-api:
|
|
||||||
stage: test
|
|
||||||
script:
|
|
||||||
- make install tools
|
|
||||||
- HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
.build:
|
|
||||||
stage: build
|
|
||||||
artifacts:
|
|
||||||
expire_in: 1 day
|
|
||||||
paths:
|
|
||||||
- lbrynet-${OS}.zip
|
|
||||||
script:
|
|
||||||
- pip install --upgrade 'setuptools<45.0.0'
|
|
||||||
- pip install pyinstaller
|
|
||||||
- pip install -e .
|
|
||||||
- python3.7 docker/set_build.py # must come after lbry is installed because it imports lbry
|
|
||||||
- pyinstaller --onefile --name lbrynet lbry/extras/cli.py
|
|
||||||
- chmod +x dist/lbrynet
|
|
||||||
- zip --junk-paths ${CI_PROJECT_DIR}/lbrynet-${OS}.zip dist/lbrynet # gitlab expects artifacts to be in $CI_PROJECT_DIR
|
|
||||||
- openssl dgst -sha256 ${CI_PROJECT_DIR}/lbrynet-${OS}.zip | egrep -o [0-9a-f]+$ # get sha256 of asset. works on mac and ubuntu
|
|
||||||
- dist/lbrynet --version
|
|
||||||
|
|
||||||
build:linux:
|
|
||||||
extends: .build
|
|
||||||
image: ubuntu:16.04
|
|
||||||
variables:
|
|
||||||
OS: linux
|
|
||||||
before_script:
|
|
||||||
- apt-get update
|
|
||||||
- apt-get install -y --no-install-recommends software-properties-common zip curl build-essential
|
|
||||||
- add-apt-repository -y ppa:deadsnakes/ppa
|
|
||||||
- apt-get update
|
|
||||||
- apt-get install -y --no-install-recommends python3.7-dev
|
|
||||||
- python3.7 <(curl -q https://bootstrap.pypa.io/get-pip.py) # make sure we get pip with python3.7
|
|
||||||
|
|
||||||
build:mac:
|
|
||||||
extends: .build
|
|
||||||
tags: [macos] # makes gitlab use the mac runner
|
|
||||||
variables:
|
|
||||||
OS: mac
|
|
||||||
GIT_DEPTH: 5
|
|
||||||
VENV: /tmp/gitlab-lbry-sdk-venv
|
|
||||||
before_script:
|
|
||||||
# - brew upgrade python || true
|
|
||||||
- python3 --version | grep -q '^Python 3\.7\.' # dont upgrade python on every run. just make sure we're on the right Python
|
|
||||||
# - pip3 install --user --upgrade pip virtualenv
|
|
||||||
- pip3 --version | grep -q '\(python 3\.7\)'
|
|
||||||
- virtualenv --python=python3.7 "${VENV}"
|
|
||||||
- source "${VENV}/bin/activate"
|
|
||||||
after_script:
|
|
||||||
- rm -rf "${VENV}"
|
|
||||||
|
|
||||||
build:windows:
|
|
||||||
extends: .build
|
|
||||||
tags: [windows] # makes gitlab use the windows runner
|
|
||||||
variables:
|
|
||||||
OS: windows
|
|
||||||
GIT_DEPTH: 5
|
|
||||||
before_script:
|
|
||||||
- ./docker/install_choco.ps1
|
|
||||||
- choco install -y --x86 python3 7zip checksum
|
|
||||||
- python --version # | findstr /B "Python 3\.7\." # dont upgrade python on every run. just make sure we're on the right Python
|
|
||||||
- pip --version # | findstr /E '\(python 3\.7\)'
|
|
||||||
- pip install virtualenv pywin32
|
|
||||||
- virtualenv venv
|
|
||||||
- venv/Scripts/activate.ps1
|
|
||||||
- pip install pip==19.3.1; $true # $true ignores errors. need this to get the correct coincurve wheel. see commit notes for details.
|
|
||||||
after_script:
|
|
||||||
- rmdir -Recurse venv
|
|
||||||
script:
|
|
||||||
- pip install --upgrade 'setuptools<45.0.0'
|
|
||||||
- pip install pyinstaller==3.5
|
|
||||||
- pip install -e .
|
|
||||||
- python docker/set_build.py # must come after lbry is installed because it imports lbry
|
|
||||||
- pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico -F -n lbrynet lbry/extras/cli.py
|
|
||||||
- 7z a -tzip $env:CI_PROJECT_DIR/lbrynet-${OS}.zip ./dist/lbrynet.exe
|
|
||||||
- checksum --type=sha256 --file=$env:CI_PROJECT_DIR/lbrynet-${OS}.zip
|
|
||||||
- dist/lbrynet.exe --version
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# s3 = upload asset to s3 (build.lbry.io)
|
|
||||||
.s3:
|
|
||||||
stage: assets
|
|
||||||
variables:
|
|
||||||
GIT_STRATEGY: none
|
|
||||||
script:
|
|
||||||
- "[ -f lbrynet-${OS}.zip ]" # check that asset exists before trying to upload
|
|
||||||
- pip install awscli
|
|
||||||
- S3_PATH="daemon/gitlab-build-${CI_PIPELINE_ID}_commit-${CI_COMMIT_SHA:0:7}$( if [ ! -z ${CI_COMMIT_TAG} ]; then echo _tag-${CI_COMMIT_TAG}; else echo _branch-${CI_COMMIT_REF_NAME}; fi )"
|
|
||||||
- AWS_ACCESS_KEY_ID=${ARTIFACTS_KEY} AWS_SECRET_ACCESS_KEY=${ARTIFACTS_SECRET} AWS_REGION=${ARTIFACTS_REGION}
|
|
||||||
aws s3 cp lbrynet-${OS}.zip s3://${ARTIFACTS_BUCKET}/${S3_PATH}/lbrynet-${OS}.zip
|
|
||||||
|
|
||||||
s3:linux:
|
|
||||||
extends: .s3
|
|
||||||
variables: {OS: linux}
|
|
||||||
needs: ["build:linux"]
|
|
||||||
|
|
||||||
s3:mac:
|
|
||||||
extends: .s3
|
|
||||||
variables: {OS: mac}
|
|
||||||
needs: ["build:mac"]
|
|
||||||
|
|
||||||
s3:windows:
|
|
||||||
extends: .s3
|
|
||||||
variables: {OS: windows}
|
|
||||||
needs: ["build:windows"]
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# github = upload assets to github when there's a tagged release
|
|
||||||
.github:
|
|
||||||
extends: .tagged
|
|
||||||
stage: assets
|
|
||||||
variables:
|
|
||||||
GIT_STRATEGY: none
|
|
||||||
script:
|
|
||||||
- "[ -f lbrynet-${OS}.zip ]" # check that asset exists before trying to upload. githubrelease won't error if its missing
|
|
||||||
- pip install githubrelease
|
|
||||||
- githubrelease --no-progress --github-token ${GITHUB_CI_USER_ACCESS_TOKEN} asset lbryio/lbry-sdk upload ${CI_COMMIT_TAG} lbrynet-${OS}.zip
|
|
||||||
|
|
||||||
github:linux:
|
|
||||||
extends: .github
|
|
||||||
variables: {OS: linux}
|
|
||||||
needs: ["build:linux"]
|
|
||||||
|
|
||||||
github:mac:
|
|
||||||
extends: .github
|
|
||||||
variables: {OS: mac}
|
|
||||||
needs: ["build:mac"]
|
|
||||||
|
|
||||||
github:windows:
|
|
||||||
extends: .github
|
|
||||||
variables: {OS: windows}
|
|
||||||
needs: ["build:windows"]
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
publish:
|
|
||||||
extends: .tagged
|
|
||||||
stage: release
|
|
||||||
variables:
|
|
||||||
GIT_STRATEGY: none
|
|
||||||
script:
|
|
||||||
- pip install githubrelease
|
|
||||||
- githubrelease --no-progress --github-token ${GITHUB_CI_USER_ACCESS_TOKEN} release lbryio/lbry-sdk publish ${CI_COMMIT_TAG}
|
|
||||||
- >
|
|
||||||
curl -X POST -H 'Content-type: application/json' --data '{"text":"<!channel> There is a new SDK release: https://github.com/lbryio/lbry-sdk/releases/tag/'"${CI_COMMIT_TAG}"'\n'"$(curl -s "https://api.github.com/repos/lbryio/lbry-sdk/releases/tags/${CI_COMMIT_TAG}" | egrep '\w*\"body\":' | cut -d':' -f 2- | tail -c +3 | head -c -2)"'", "channel":"tech"}' "$(echo ${SLACK_WEBHOOK_URL_BASE64} | base64 -d)"
|
|
||||||
+113
-44
@@ -9,20 +9,29 @@ Here's a video walkthrough of this setup, which is itself hosted by the LBRY net
|
|||||||
|
|
||||||
## Prerequisites
|
## Prerequisites
|
||||||
|
|
||||||
Running `lbrynet` from source requires Python 3.7 or higher. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/).
|
Running `lbrynet` from source requires Python 3.7. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/).
|
||||||
|
|
||||||
After installing python 3, you'll need to install some additional libraries depending on your operating system.
|
After installing Python 3.7, you'll need to install some additional libraries depending on your operating system.
|
||||||
|
|
||||||
|
Because of [issue #2769](https://github.com/lbryio/lbry-sdk/issues/2769)
|
||||||
|
at the moment the `lbrynet` daemon will only work correctly with Python 3.7.
|
||||||
|
If Python 3.8+ is used, the daemon will start but the RPC server
|
||||||
|
may not accept messages, returning the following:
|
||||||
|
```
|
||||||
|
Could not connect to daemon. Are you sure it's running?
|
||||||
|
```
|
||||||
|
|
||||||
### macOS
|
### macOS
|
||||||
|
|
||||||
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
|
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
|
||||||
|
|
||||||
These environment variables also need to be set:
|
These environment variables also need to be set:
|
||||||
1. PYTHONUNBUFFERED=1
|
```
|
||||||
2. EVENT_NOKQUEUE=1
|
PYTHONUNBUFFERED=1
|
||||||
|
EVENT_NOKQUEUE=1
|
||||||
|
```
|
||||||
|
|
||||||
Remaining dependencies can then be installed by running:
|
Remaining dependencies can then be installed by running:
|
||||||
|
|
||||||
```
|
```
|
||||||
brew install python protobuf
|
brew install python protobuf
|
||||||
```
|
```
|
||||||
@@ -31,14 +40,17 @@ Assistance installing Python3: https://docs.python-guide.org/starting/install3/o
|
|||||||
|
|
||||||
### Linux
|
### Linux
|
||||||
|
|
||||||
On Ubuntu (16.04 minimum, we recommend 18.04), install the following:
|
On Ubuntu (we recommend 18.04 or 20.04), install the following:
|
||||||
|
|
||||||
```
|
```
|
||||||
sudo add-apt-repository ppa:deadsnakes/ppa
|
sudo add-apt-repository ppa:deadsnakes/ppa
|
||||||
sudo apt-get update
|
sudo apt-get update
|
||||||
sudo apt-get install build-essential python3.7 python3.7-dev git python3.7-venv libssl-dev python-protobuf
|
sudo apt-get install build-essential python3.7 python3.7-dev git python3.7-venv libssl-dev python-protobuf
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The [deadsnakes PPA](https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa) provides Python 3.7
|
||||||
|
for those Ubuntu distributions that no longer have it in their
|
||||||
|
official repositories.
|
||||||
|
|
||||||
On Raspbian, you will also need to install `python-pyparsing`.
|
On Raspbian, you will also need to install `python-pyparsing`.
|
||||||
|
|
||||||
If you're running another Linux distro, install the equivalent of the above packages for your system.
|
If you're running another Linux distro, install the equivalent of the above packages for your system.
|
||||||
@@ -47,62 +59,119 @@ If you're running another Linux distro, install the equivalent of the above pack
|
|||||||
|
|
||||||
### Linux/Mac
|
### Linux/Mac
|
||||||
|
|
||||||
To install on Linux/Mac:
|
Clone the repository:
|
||||||
|
```bash
|
||||||
|
git clone https://github.com/lbryio/lbry-sdk.git
|
||||||
|
cd lbry-sdk
|
||||||
|
```
|
||||||
|
|
||||||
```
|
Create a Python virtual environment for lbry-sdk:
|
||||||
Clone the repository:
|
```bash
|
||||||
$ git clone https://github.com/lbryio/lbry-sdk.git
|
python3.7 -m venv lbry-venv
|
||||||
$ cd lbry-sdk
|
```
|
||||||
|
|
||||||
Create a Python virtual environment for lbry-sdk:
|
Activate virtual environment:
|
||||||
$ python3.7 -m venv lbry-venv
|
```bash
|
||||||
|
source lbry-venv/bin/activate
|
||||||
Activating lbry-sdk virtual environment:
|
```
|
||||||
$ source lbry-venv/bin/activate
|
|
||||||
|
|
||||||
Make sure you're on Python 3.7+ (as the default Python in virtual environment):
|
|
||||||
$ python --version
|
|
||||||
|
|
||||||
Install packages:
|
Make sure you're on Python 3.7+ as default in the virtual environment:
|
||||||
$ make install
|
```bash
|
||||||
|
python --version
|
||||||
|
```
|
||||||
|
|
||||||
If you are on Linux and using PyCharm, generates initial configs:
|
Install packages:
|
||||||
$ make idea
|
```bash
|
||||||
```
|
make install
|
||||||
|
```
|
||||||
|
|
||||||
To verify your installation, `which lbrynet` should return a path inside of the `lbry-venv` folder created by the `python3.7 -m venv lbry-venv` command.
|
If you are on Linux and using PyCharm, generates initial configs:
|
||||||
|
```bash
|
||||||
|
make idea
|
||||||
|
```
|
||||||
|
|
||||||
|
To verify your installation, `which lbrynet` should return a path inside
|
||||||
|
of the `lbry-venv` folder.
|
||||||
|
```bash
|
||||||
|
(lbry-venv) $ which lbrynet
|
||||||
|
/opt/lbry-sdk/lbry-venv/bin/lbrynet
|
||||||
|
```
|
||||||
|
|
||||||
|
To exit the virtual environment simply use the command `deactivate`.
|
||||||
|
|
||||||
### Windows
|
### Windows
|
||||||
|
|
||||||
To install on Windows:
|
Clone the repository:
|
||||||
|
```bash
|
||||||
|
git clone https://github.com/lbryio/lbry-sdk.git
|
||||||
|
cd lbry-sdk
|
||||||
|
```
|
||||||
|
|
||||||
```
|
Create a Python virtual environment for lbry-sdk:
|
||||||
Clone the repository:
|
```bash
|
||||||
> git clone https://github.com/lbryio/lbry-sdk.git
|
python -m venv lbry-venv
|
||||||
> cd lbry-sdk
|
```
|
||||||
|
|
||||||
Create a Python virtual environment for lbry-sdk:
|
Activate virtual environment:
|
||||||
> python -m venv lbry-venv
|
```bash
|
||||||
|
lbry-venv\Scripts\activate
|
||||||
|
```
|
||||||
|
|
||||||
Activating lbry-sdk virtual environment:
|
Install packages:
|
||||||
> lbry-venv\Scripts\activate
|
```bash
|
||||||
|
pip install -e .
|
||||||
Install packages:
|
```
|
||||||
> pip install -e .
|
|
||||||
```
|
|
||||||
|
|
||||||
## Run the tests
|
## Run the tests
|
||||||
|
### Elasticsearch
|
||||||
|
|
||||||
To run the unit tests from the repo directory:
|
For running integration tests, Elasticsearch is required to be available at localhost:9200/
|
||||||
|
|
||||||
```
|
The easiest way to start it is using docker with:
|
||||||
python -m unittest discover tests.unit
|
```bash
|
||||||
```
|
make elastic-docker
|
||||||
|
```
|
||||||
|
|
||||||
|
Alternative installation methods are available [at Elasticsearch website](https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html).
|
||||||
|
|
||||||
|
To run the unit and integration tests from the repo directory:
|
||||||
|
```
|
||||||
|
python -m unittest discover tests.unit
|
||||||
|
python -m unittest discover tests.integration
|
||||||
|
```
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
To start the API server:
|
To start the API server:
|
||||||
`lbrynet start`
|
```
|
||||||
|
lbrynet start
|
||||||
|
```
|
||||||
|
|
||||||
|
Whenever the code inside [lbry-sdk/lbry](./lbry)
|
||||||
|
is modified we should run `make install` to recompile the `lbrynet`
|
||||||
|
executable with the newest code.
|
||||||
|
|
||||||
|
## Development
|
||||||
|
|
||||||
|
When developing, remember to enter the environment,
|
||||||
|
and if you wish start the server interactively.
|
||||||
|
```bash
|
||||||
|
$ source lbry-venv/bin/activate
|
||||||
|
|
||||||
|
(lbry-venv) $ python lbry/extras/cli.py start
|
||||||
|
```
|
||||||
|
|
||||||
|
Parameters can be passed in the same way.
|
||||||
|
```bash
|
||||||
|
(lbry-venv) $ python lbry/extras/cli.py wallet balance
|
||||||
|
```
|
||||||
|
|
||||||
|
If a Python debugger (`pdb` or `ipdb`) is installed we can also start it
|
||||||
|
in this way, set up break points, and step through the code.
|
||||||
|
```bash
|
||||||
|
(lbry-venv) $ pip install ipdb
|
||||||
|
|
||||||
|
(lbry-venv) $ ipdb lbry/extras/cli.py
|
||||||
|
```
|
||||||
|
|
||||||
Happy hacking!
|
Happy hacking!
|
||||||
@@ -1,23 +1,26 @@
|
|||||||
.PHONY: install tools lint test idea
|
.PHONY: install tools lint test test-unit test-unit-coverage test-integration idea
|
||||||
|
|
||||||
install:
|
install:
|
||||||
CFLAGS="-DSQLITE_MAX_VARIABLE_NUMBER=2500000" pip install -U https://github.com/rogerbinns/apsw/releases/download/3.30.1-r1/apsw-3.30.1-r1.zip \
|
|
||||||
--global-option=fetch \
|
|
||||||
--global-option=--version --global-option=3.30.1 --global-option=--all \
|
|
||||||
--global-option=build --global-option=--enable --global-option=fts5
|
|
||||||
pip install -e .
|
pip install -e .
|
||||||
|
|
||||||
tools:
|
|
||||||
pip install mypy==0.701
|
|
||||||
pip install coverage astroid pylint
|
|
||||||
|
|
||||||
lint:
|
lint:
|
||||||
pylint --rcfile=setup.cfg lbry
|
pylint --rcfile=setup.cfg lbry
|
||||||
#mypy --ignore-missing-imports lbry
|
#mypy --ignore-missing-imports lbry
|
||||||
|
|
||||||
test:
|
test: test-unit test-integration
|
||||||
|
|
||||||
|
test-unit:
|
||||||
|
python -m unittest discover tests.unit
|
||||||
|
|
||||||
|
test-unit-coverage:
|
||||||
|
coverage run --source=lbry -m unittest discover -vv tests.unit
|
||||||
|
|
||||||
|
test-integration:
|
||||||
tox
|
tox
|
||||||
|
|
||||||
idea:
|
idea:
|
||||||
mkdir -p .idea
|
mkdir -p .idea
|
||||||
cp -r scripts/idea/* .idea
|
cp -r scripts/idea/* .idea
|
||||||
|
|
||||||
|
elastic-docker:
|
||||||
|
docker run -d -v lbryhub:/usr/share/elasticsearch/data -p 9200:9200 -p 9300:9300 -e"ES_JAVA_OPTS=-Xms512m -Xmx512m" -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.12.1
|
||||||
@@ -1,10 +1,10 @@
|
|||||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry-sdk/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://ci.lbry.tech/lbry/lbry-sdk)
|
# <img src="https://raw.githubusercontent.com/lbryio/lbry-sdk/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://github.com/lbryio/lbry-sdk/actions/workflows/main.yml) [](https://coveralls.io/github/lbryio/lbry-sdk)
|
||||||
|
|
||||||
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||||
|
|
||||||
LBRY SDK for Python is currently the most fully featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components include:
|
LBRY SDK for Python is currently the most fully featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components include:
|
||||||
|
|
||||||
* Built on Python 3.7+ and `asyncio`.
|
* Built on Python 3.7 and `asyncio`.
|
||||||
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbry.dht](https://github.com/lbryio/lbry-sdk/tree/master/lbry/dht)).
|
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbry.dht](https://github.com/lbryio/lbry-sdk/tree/master/lbry/dht)).
|
||||||
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbry.blob_exchange](https://github.com/lbryio/lbry-sdk/tree/master/lbry/blob_exchange)).
|
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbry.blob_exchange](https://github.com/lbryio/lbry-sdk/tree/master/lbry/blob_exchange)).
|
||||||
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbry.schema](https://github.com/lbryio/lbry-sdk/tree/master/lbry/schema)).
|
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbry.schema](https://github.com/lbryio/lbry-sdk/tree/master/lbry/schema)).
|
||||||
@@ -41,7 +41,7 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
|
|||||||
|
|
||||||
## Security
|
## Security
|
||||||
|
|
||||||
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.com/faq/gpg-key) if you need it.
|
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our PGP key is here](https://lbry.com/faq/pgp-key) if you need it.
|
||||||
|
|
||||||
## Contact
|
## Contact
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,9 @@
|
|||||||
|
# Security Policy
|
||||||
|
|
||||||
|
## Supported Versions
|
||||||
|
|
||||||
|
While we are not at v1.0 yet, only the latest release will be supported.
|
||||||
|
|
||||||
|
## Reporting a Vulnerability
|
||||||
|
|
||||||
|
See https://lbry.com/faq/security
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
FROM debian:10-slim
|
||||||
|
|
||||||
|
ARG user=lbry
|
||||||
|
ARG projects_dir=/home/$user
|
||||||
|
|
||||||
|
ARG DOCKER_TAG
|
||||||
|
ARG DOCKER_COMMIT=docker
|
||||||
|
ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT
|
||||||
|
|
||||||
|
RUN apt-get update && \
|
||||||
|
apt-get -y --no-install-recommends install \
|
||||||
|
wget \
|
||||||
|
automake libtool \
|
||||||
|
tar unzip \
|
||||||
|
build-essential \
|
||||||
|
pkg-config \
|
||||||
|
libleveldb-dev \
|
||||||
|
python3.7 \
|
||||||
|
python3-dev \
|
||||||
|
python3-pip \
|
||||||
|
python3-wheel \
|
||||||
|
python3-setuptools && \
|
||||||
|
update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
RUN groupadd -g 999 $user && useradd -m -u 999 -g $user $user
|
||||||
|
|
||||||
|
COPY . $projects_dir
|
||||||
|
RUN chown -R $user:$user $projects_dir
|
||||||
|
|
||||||
|
USER $user
|
||||||
|
WORKDIR $projects_dir
|
||||||
|
|
||||||
|
RUN make install
|
||||||
|
RUN python3 docker/set_build.py
|
||||||
|
RUN rm ~/.cache -rf
|
||||||
|
ENTRYPOINT ["python3", "scripts/dht_node.py"]
|
||||||
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
FROM ubuntu:19.10
|
FROM debian:10-slim
|
||||||
|
|
||||||
ARG user=lbry
|
ARG user=lbry
|
||||||
ARG db_dir=/database
|
ARG db_dir=/database
|
||||||
@@ -13,7 +13,10 @@ RUN apt-get update && \
|
|||||||
wget \
|
wget \
|
||||||
tar unzip \
|
tar unzip \
|
||||||
build-essential \
|
build-essential \
|
||||||
python3 \
|
automake libtool \
|
||||||
|
pkg-config \
|
||||||
|
libleveldb-dev \
|
||||||
|
python3.7 \
|
||||||
python3-dev \
|
python3-dev \
|
||||||
python3-pip \
|
python3-pip \
|
||||||
python3-wheel \
|
python3-wheel \
|
||||||
|
|||||||
@@ -0,0 +1,45 @@
|
|||||||
|
FROM debian:10-slim
|
||||||
|
|
||||||
|
ARG user=lbry
|
||||||
|
ARG downloads_dir=/database
|
||||||
|
ARG projects_dir=/home/$user
|
||||||
|
|
||||||
|
ARG DOCKER_TAG
|
||||||
|
ARG DOCKER_COMMIT=docker
|
||||||
|
ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT
|
||||||
|
|
||||||
|
RUN apt-get update && \
|
||||||
|
apt-get -y --no-install-recommends install \
|
||||||
|
wget \
|
||||||
|
automake libtool \
|
||||||
|
tar unzip \
|
||||||
|
build-essential \
|
||||||
|
pkg-config \
|
||||||
|
libleveldb-dev \
|
||||||
|
python3.7 \
|
||||||
|
python3-dev \
|
||||||
|
python3-pip \
|
||||||
|
python3-wheel \
|
||||||
|
python3-setuptools && \
|
||||||
|
update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
RUN groupadd -g 999 $user && useradd -m -u 999 -g $user $user
|
||||||
|
RUN mkdir -p $downloads_dir
|
||||||
|
RUN chown -R $user:$user $downloads_dir
|
||||||
|
|
||||||
|
COPY . $projects_dir
|
||||||
|
RUN chown -R $user:$user $projects_dir
|
||||||
|
|
||||||
|
USER $user
|
||||||
|
WORKDIR $projects_dir
|
||||||
|
|
||||||
|
RUN pip install uvloop
|
||||||
|
RUN make install
|
||||||
|
RUN python3 docker/set_build.py
|
||||||
|
RUN rm ~/.cache -rf
|
||||||
|
|
||||||
|
# entry point
|
||||||
|
VOLUME $downloads_dir
|
||||||
|
COPY ./docker/webconf.yaml /webconf.yaml
|
||||||
|
ENTRYPOINT ["/home/lbry/.local/bin/lbrynet", "start", "--config=/webconf.yaml"]
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
### How to run with docker-compose
|
||||||
|
1. Edit config file and after that fix permissions with
|
||||||
|
```
|
||||||
|
sudo chown -R 999:999 webconf.yaml
|
||||||
|
```
|
||||||
|
2. Start SDK with
|
||||||
|
```
|
||||||
|
docker-compose up -d
|
||||||
|
```
|
||||||
@@ -1,34 +1,45 @@
|
|||||||
version: "3"
|
version: "3"
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
lbrycrd:
|
|
||||||
wallet_server:
|
wallet_server:
|
||||||
|
es01:
|
||||||
|
|
||||||
services:
|
services:
|
||||||
lbrycrd:
|
|
||||||
image: lbry/lbrycrd:${LBRYCRD_TAG:-latest-release}
|
|
||||||
restart: always
|
|
||||||
ports: # accessible from host
|
|
||||||
- "9246:9246" # rpc port
|
|
||||||
expose: # internal to docker network. also this doesn't do anything. its for documentation only.
|
|
||||||
- "9245" # node-to-node comms port
|
|
||||||
volumes:
|
|
||||||
- "lbrycrd:/data/.lbrycrd"
|
|
||||||
environment:
|
|
||||||
- RUN_MODE=default
|
|
||||||
- SNAPSHOT_URL=${LBRYCRD_SNAPSHOT_URL-https://lbry.com/snapshot/blockchain}
|
|
||||||
- RPC_ALLOW_IP=0.0.0.0/0
|
|
||||||
wallet_server:
|
wallet_server:
|
||||||
|
depends_on:
|
||||||
|
- es01
|
||||||
image: lbry/wallet-server:${WALLET_SERVER_TAG:-latest-release}
|
image: lbry/wallet-server:${WALLET_SERVER_TAG:-latest-release}
|
||||||
depends_on:
|
|
||||||
- lbrycrd
|
|
||||||
restart: always
|
restart: always
|
||||||
|
network_mode: host
|
||||||
ports:
|
ports:
|
||||||
- "50001:50001" # rpc port
|
- "50001:50001" # rpc port
|
||||||
- "50005:50005" # websocket port
|
- "2112:2112" # uncomment to enable prometheus
|
||||||
#- "2112:2112" # uncomment to enable prometheus
|
|
||||||
volumes:
|
volumes:
|
||||||
- "wallet_server:/database"
|
- "wallet_server:/database"
|
||||||
environment:
|
environment:
|
||||||
- SNAPSHOT_URL=${WALLET_SERVER_SNAPSHOT_URL-https://lbry.com/snapshot/wallet}
|
- DAEMON_URL=http://lbry:lbry@127.0.0.1:9245
|
||||||
- DAEMON_URL=http://lbry:lbry@lbrycrd:9245
|
- TCP_PORT=50001
|
||||||
|
- PROMETHEUS_PORT=2112
|
||||||
|
- QUERY_TIMEOUT_MS=3000 # how long search queries allowed to run before cancelling, in milliseconds
|
||||||
|
- TRENDING_ALGORITHMS=variable_decay
|
||||||
|
- MAX_SEND=10000000000000 # deprecated. leave it high until its removed
|
||||||
|
- MAX_SUBS=1000000000000 # deprecated. leave it high until its removed
|
||||||
|
- FILTERING_CHANNEL_IDS=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8
|
||||||
|
- BLOCKING_CHANNEL_IDS=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6 e4e230b131082f6b10c8f7994bbb83f29e8e6fb9
|
||||||
|
es01:
|
||||||
|
image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0
|
||||||
|
container_name: es01
|
||||||
|
environment:
|
||||||
|
- node.name=es01
|
||||||
|
- discovery.type=single-node
|
||||||
|
- indices.query.bool.max_clause_count=4096
|
||||||
|
- bootstrap.memory_lock=true
|
||||||
|
- "ES_JAVA_OPTS=-Xms8g -Xmx8g" # no more than 32, remember to disable swap
|
||||||
|
ulimits:
|
||||||
|
memlock:
|
||||||
|
soft: -1
|
||||||
|
hard: -1
|
||||||
|
volumes:
|
||||||
|
- es01:/usr/share/elasticsearch/data
|
||||||
|
ports:
|
||||||
|
- 127.0.0.1:9200:9200
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
version: '3'
|
||||||
|
services:
|
||||||
|
websdk:
|
||||||
|
image: vshyba/websdk
|
||||||
|
ports:
|
||||||
|
- '5279:5279'
|
||||||
|
- '5280:5280'
|
||||||
|
volumes:
|
||||||
|
- ./webconf.yaml:/webconf.yaml
|
||||||
+5
-3
@@ -20,7 +20,7 @@ def _check_and_set(d: dict, key: str, value: str):
|
|||||||
def main():
|
def main():
|
||||||
build_info = {item: build_info_mod.__dict__[item] for item in dir(build_info_mod) if not item.startswith("__")}
|
build_info = {item: build_info_mod.__dict__[item] for item in dir(build_info_mod) if not item.startswith("__")}
|
||||||
|
|
||||||
commit_hash = os.getenv('DOCKER_COMMIT', os.getenv('CI_COMMIT_SHA', os.getenv('TRAVIS_COMMIT')))
|
commit_hash = os.getenv('DOCKER_COMMIT', os.getenv('GITHUB_SHA'))
|
||||||
if commit_hash is None:
|
if commit_hash is None:
|
||||||
raise ValueError("Commit hash not found in env vars")
|
raise ValueError("Commit hash not found in env vars")
|
||||||
_check_and_set(build_info, "COMMIT_HASH", commit_hash[:6])
|
_check_and_set(build_info, "COMMIT_HASH", commit_hash[:6])
|
||||||
@@ -30,8 +30,10 @@ def main():
|
|||||||
_check_and_set(build_info, "DOCKER_TAG", docker_tag)
|
_check_and_set(build_info, "DOCKER_TAG", docker_tag)
|
||||||
_check_and_set(build_info, "BUILD", "docker")
|
_check_and_set(build_info, "BUILD", "docker")
|
||||||
else:
|
else:
|
||||||
ci_tag = os.getenv('CI_COMMIT_TAG', os.getenv('TRAVIS_TAG'))
|
if re.match(r'refs/tags/v\d+\.\d+\.\d+$', str(os.getenv('GITHUB_REF'))):
|
||||||
_check_and_set(build_info, "BUILD", "release" if re.match(r'v\d+\.\d+\.\d+$', str(ci_tag)) else "qa")
|
_check_and_set(build_info, "BUILD", "release")
|
||||||
|
else:
|
||||||
|
_check_and_set(build_info, "BUILD", "qa")
|
||||||
|
|
||||||
log.debug("build info: %s", ", ".join([f"{k}={v}" for k, v in build_info.items()]))
|
log.debug("build info: %s", ", ".join([f"{k}={v}" for k, v in build_info.items()]))
|
||||||
with open(build_info_mod.__file__, 'w') as f:
|
with open(build_info_mod.__file__, 'w') as f:
|
||||||
|
|||||||
@@ -20,4 +20,6 @@ if [[ -n "$SNAPSHOT_URL" ]] && [[ ! -f /database/claims.db ]]; then
|
|||||||
rm "$filename"
|
rm "$filename"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
/home/lbry/.local/bin/torba-server "$@"
|
/home/lbry/.local/bin/lbry-hub-elastic-sync /database/claims.db
|
||||||
|
echo 'starting server'
|
||||||
|
/home/lbry/.local/bin/lbry-hub "$@"
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
allowed_origin: "*"
|
||||||
|
max_key_fee: "0.0 USD"
|
||||||
|
save_files: false
|
||||||
|
save_blobs: false
|
||||||
|
streaming_server: "0.0.0.0:5280"
|
||||||
|
api: "0.0.0.0:5279"
|
||||||
|
data_dir: /tmp
|
||||||
|
download_dir: /tmp
|
||||||
|
wallet_dir: /tmp
|
||||||
+1148
-403
File diff suppressed because it is too large.
Load diff
+1
-1
@@ -1,2 +1,2 @@
|
|||||||
__version__ = "0.60.1"
|
__version__ = "0.104.0"
|
||||||
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
|
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
|
||||||
+58
-32
@@ -1,5 +1,6 @@
|
|||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
import time
|
||||||
import asyncio
|
import asyncio
|
||||||
import binascii
|
import binascii
|
||||||
import logging
|
import logging
|
||||||
@@ -70,12 +71,16 @@ class AbstractBlob:
|
|||||||
'writers',
|
'writers',
|
||||||
'verified',
|
'verified',
|
||||||
'writing',
|
'writing',
|
||||||
'readers'
|
'readers',
|
||||||
|
'added_on',
|
||||||
|
'is_mine',
|
||||||
]
|
]
|
||||||
|
|
||||||
def __init__(self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
def __init__(
|
||||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||||
blob_directory: typing.Optional[str] = None):
|
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||||
|
blob_directory: typing.Optional[str] = None, added_on: typing.Optional[int] = None, is_mine: bool = False,
|
||||||
|
):
|
||||||
self.loop = loop
|
self.loop = loop
|
||||||
self.blob_hash = blob_hash
|
self.blob_hash = blob_hash
|
||||||
self.length = length
|
self.length = length
|
||||||
@@ -85,6 +90,8 @@ class AbstractBlob:
|
|||||||
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
|
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||||
self.writing: asyncio.Event = asyncio.Event(loop=self.loop)
|
self.writing: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||||
self.readers: typing.List[typing.BinaryIO] = []
|
self.readers: typing.List[typing.BinaryIO] = []
|
||||||
|
self.added_on = added_on or time.time()
|
||||||
|
self.is_mine = is_mine
|
||||||
|
|
||||||
if not is_valid_blobhash(blob_hash):
|
if not is_valid_blobhash(blob_hash):
|
||||||
raise InvalidBlobHashError(blob_hash)
|
raise InvalidBlobHashError(blob_hash)
|
||||||
@@ -110,7 +117,7 @@ class AbstractBlob:
|
|||||||
if reader in self.readers:
|
if reader in self.readers:
|
||||||
self.readers.remove(reader)
|
self.readers.remove(reader)
|
||||||
|
|
||||||
def _write_blob(self, blob_bytes: bytes):
|
def _write_blob(self, blob_bytes: bytes) -> asyncio.Task:
|
||||||
raise NotImplementedError()
|
raise NotImplementedError()
|
||||||
|
|
||||||
def set_length(self, length) -> None:
|
def set_length(self, length) -> None:
|
||||||
@@ -180,29 +187,36 @@ class AbstractBlob:
|
|||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
async def create_from_unencrypted(
|
async def create_from_unencrypted(
|
||||||
cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
|
cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
|
||||||
unencrypted: bytes, blob_num: int,
|
unencrypted: bytes, blob_num: int, added_on: int, is_mine: bool,
|
||||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None) -> BlobInfo:
|
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None,
|
||||||
|
) -> BlobInfo:
|
||||||
"""
|
"""
|
||||||
Create an encrypted BlobFile from plaintext bytes
|
Create an encrypted BlobFile from plaintext bytes
|
||||||
"""
|
"""
|
||||||
|
|
||||||
blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted)
|
blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted)
|
||||||
length = len(blob_bytes)
|
length = len(blob_bytes)
|
||||||
blob = cls(loop, blob_hash, length, blob_completed_callback, blob_dir)
|
blob = cls(loop, blob_hash, length, blob_completed_callback, blob_dir, added_on, is_mine)
|
||||||
writer = blob.get_blob_writer()
|
writer = blob.get_blob_writer()
|
||||||
writer.write(blob_bytes)
|
writer.write(blob_bytes)
|
||||||
await blob.verified.wait()
|
await blob.verified.wait()
|
||||||
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash)
|
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash, added_on, is_mine)
|
||||||
|
|
||||||
def save_verified_blob(self, verified_bytes: bytes):
|
def save_verified_blob(self, verified_bytes: bytes):
|
||||||
if self.verified.is_set():
|
if self.verified.is_set():
|
||||||
return
|
return
|
||||||
if self.is_writeable():
|
|
||||||
self._write_blob(verified_bytes)
|
def update_events(_):
|
||||||
self.verified.set()
|
self.verified.set()
|
||||||
|
self.writing.clear()
|
||||||
|
|
||||||
|
if self.is_writeable():
|
||||||
|
self.writing.set()
|
||||||
|
task = self._write_blob(verified_bytes)
|
||||||
|
task.add_done_callback(update_events)
|
||||||
if self.blob_completed_callback:
|
if self.blob_completed_callback:
|
||||||
self.blob_completed_callback(self)
|
task.add_done_callback(lambda _: self.blob_completed_callback(self))
|
||||||
|
|
||||||
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
|
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
|
||||||
peer_port: typing.Optional[int] = None) -> HashBlobWriter:
|
peer_port: typing.Optional[int] = None) -> HashBlobWriter:
|
||||||
@@ -242,11 +256,13 @@ class BlobBuffer(AbstractBlob):
|
|||||||
"""
|
"""
|
||||||
An in-memory only blob
|
An in-memory only blob
|
||||||
"""
|
"""
|
||||||
def __init__(self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
def __init__(
|
||||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||||
blob_directory: typing.Optional[str] = None):
|
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||||
|
blob_directory: typing.Optional[str] = None, added_on: typing.Optional[int] = None, is_mine: bool = False
|
||||||
|
):
|
||||||
self._verified_bytes: typing.Optional[BytesIO] = None
|
self._verified_bytes: typing.Optional[BytesIO] = None
|
||||||
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
|
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory, added_on, is_mine)
|
||||||
|
|
||||||
@contextlib.contextmanager
|
@contextlib.contextmanager
|
||||||
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||||
@@ -261,9 +277,11 @@ class BlobBuffer(AbstractBlob):
|
|||||||
self.verified.clear()
|
self.verified.clear()
|
||||||
|
|
||||||
def _write_blob(self, blob_bytes: bytes):
|
def _write_blob(self, blob_bytes: bytes):
|
||||||
if self._verified_bytes:
|
async def write():
|
||||||
raise OSError("already have bytes for blob")
|
if self._verified_bytes:
|
||||||
self._verified_bytes = BytesIO(blob_bytes)
|
raise OSError("already have bytes for blob")
|
||||||
|
self._verified_bytes = BytesIO(blob_bytes)
|
||||||
|
return self.loop.create_task(write())
|
||||||
|
|
||||||
def delete(self):
|
def delete(self):
|
||||||
if self._verified_bytes:
|
if self._verified_bytes:
|
||||||
@@ -281,10 +299,12 @@ class BlobFile(AbstractBlob):
|
|||||||
"""
|
"""
|
||||||
A blob existing on the local file system
|
A blob existing on the local file system
|
||||||
"""
|
"""
|
||||||
def __init__(self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
def __init__(
|
||||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||||
blob_directory: typing.Optional[str] = None):
|
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||||
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
|
blob_directory: typing.Optional[str] = None, added_on: typing.Optional[int] = None, is_mine: bool = False
|
||||||
|
):
|
||||||
|
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory, added_on, is_mine)
|
||||||
if not blob_directory or not os.path.isdir(blob_directory):
|
if not blob_directory or not os.path.isdir(blob_directory):
|
||||||
raise OSError(f"invalid blob directory '{blob_directory}'")
|
raise OSError(f"invalid blob directory '{blob_directory}'")
|
||||||
self.file_path = os.path.join(self.blob_directory, self.blob_hash)
|
self.file_path = os.path.join(self.blob_directory, self.blob_hash)
|
||||||
@@ -319,22 +339,28 @@ class BlobFile(AbstractBlob):
|
|||||||
handle.close()
|
handle.close()
|
||||||
|
|
||||||
def _write_blob(self, blob_bytes: bytes):
|
def _write_blob(self, blob_bytes: bytes):
|
||||||
with open(self.file_path, 'wb') as f:
|
def _write_blob():
|
||||||
f.write(blob_bytes)
|
with open(self.file_path, 'wb') as f:
|
||||||
|
f.write(blob_bytes)
|
||||||
|
|
||||||
|
async def write_blob():
|
||||||
|
await self.loop.run_in_executor(None, _write_blob)
|
||||||
|
|
||||||
|
return self.loop.create_task(write_blob())
|
||||||
|
|
||||||
def delete(self):
|
def delete(self):
|
||||||
|
super().delete()
|
||||||
if os.path.isfile(self.file_path):
|
if os.path.isfile(self.file_path):
|
||||||
os.remove(self.file_path)
|
os.remove(self.file_path)
|
||||||
return super().delete()
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
async def create_from_unencrypted(
|
async def create_from_unencrypted(
|
||||||
cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
|
cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
|
||||||
unencrypted: bytes, blob_num: int,
|
unencrypted: bytes, blob_num: int, added_on: float, is_mine: bool,
|
||||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'],
|
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None
|
||||||
asyncio.Task]] = None) -> BlobInfo:
|
) -> BlobInfo:
|
||||||
if not blob_dir or not os.path.isdir(blob_dir):
|
if not blob_dir or not os.path.isdir(blob_dir):
|
||||||
raise OSError(f"cannot create blob in directory: '{blob_dir}'")
|
raise OSError(f"cannot create blob in directory: '{blob_dir}'")
|
||||||
return await super().create_from_unencrypted(
|
return await super().create_from_unencrypted(
|
||||||
loop, blob_dir, key, iv, unencrypted, blob_num, blob_completed_callback
|
loop, blob_dir, key, iv, unencrypted, blob_num, added_on, is_mine, blob_completed_callback
|
||||||
)
|
)
|
||||||
@@ -7,13 +7,19 @@ class BlobInfo:
|
|||||||
'blob_num',
|
'blob_num',
|
||||||
'length',
|
'length',
|
||||||
'iv',
|
'iv',
|
||||||
|
'added_on',
|
||||||
|
'is_mine'
|
||||||
]
|
]
|
||||||
|
|
||||||
def __init__(self, blob_num: int, length: int, iv: str, blob_hash: typing.Optional[str] = None):
|
def __init__(
|
||||||
|
self, blob_num: int, length: int, iv: str,
|
||||||
|
blob_hash: typing.Optional[str] = None, added_on=0, is_mine=False):
|
||||||
self.blob_hash = blob_hash
|
self.blob_hash = blob_hash
|
||||||
self.blob_num = blob_num
|
self.blob_num = blob_num
|
||||||
self.length = length
|
self.length = length
|
||||||
self.iv = iv
|
self.iv = iv
|
||||||
|
self.added_on = added_on
|
||||||
|
self.is_mine = is_mine
|
||||||
|
|
||||||
def as_dict(self) -> typing.Dict:
|
def as_dict(self) -> typing.Dict:
|
||||||
d = {
|
d = {
|
||||||
|
|||||||
+14
-10
@@ -2,7 +2,7 @@ import os
|
|||||||
import typing
|
import typing
|
||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
from lbry.utils import LRUCache
|
from lbry.utils import LRUCacheWithMetrics
|
||||||
from lbry.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
|
from lbry.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
|
||||||
from lbry.stream.descriptor import StreamDescriptor
|
from lbry.stream.descriptor import StreamDescriptor
|
||||||
from lbry.connection_manager import ConnectionManager
|
from lbry.connection_manager import ConnectionManager
|
||||||
@@ -32,34 +32,34 @@ class BlobManager:
|
|||||||
else self._node_data_store.completed_blobs
|
else self._node_data_store.completed_blobs
|
||||||
self.blobs: typing.Dict[str, AbstractBlob] = {}
|
self.blobs: typing.Dict[str, AbstractBlob] = {}
|
||||||
self.config = config
|
self.config = config
|
||||||
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCache(
|
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCacheWithMetrics(
|
||||||
self.config.blob_lru_cache_size)
|
self.config.blob_lru_cache_size)
|
||||||
self.connection_manager = ConnectionManager(loop)
|
self.connection_manager = ConnectionManager(loop)
|
||||||
|
|
||||||
def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None):
|
def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None, is_mine: bool = False):
|
||||||
if self.config.save_blobs or (
|
if self.config.save_blobs or (
|
||||||
is_valid_blobhash(blob_hash) and os.path.isfile(os.path.join(self.blob_dir, blob_hash))):
|
is_valid_blobhash(blob_hash) and os.path.isfile(os.path.join(self.blob_dir, blob_hash))):
|
||||||
return BlobFile(
|
return BlobFile(
|
||||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
self.loop, blob_hash, length, self.blob_completed, self.blob_dir, is_mine=is_mine
|
||||||
)
|
)
|
||||||
return BlobBuffer(
|
return BlobBuffer(
|
||||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
self.loop, blob_hash, length, self.blob_completed, self.blob_dir, is_mine=is_mine
|
||||||
)
|
)
|
||||||
|
|
||||||
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
|
def get_blob(self, blob_hash, length: typing.Optional[int] = None, is_mine: bool = False):
|
||||||
if blob_hash in self.blobs:
|
if blob_hash in self.blobs:
|
||||||
if self.config.save_blobs and isinstance(self.blobs[blob_hash], BlobBuffer):
|
if self.config.save_blobs and isinstance(self.blobs[blob_hash], BlobBuffer):
|
||||||
buffer = self.blobs.pop(blob_hash)
|
buffer = self.blobs.pop(blob_hash)
|
||||||
if blob_hash in self.completed_blob_hashes:
|
if blob_hash in self.completed_blob_hashes:
|
||||||
self.completed_blob_hashes.remove(blob_hash)
|
self.completed_blob_hashes.remove(blob_hash)
|
||||||
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
|
self.blobs[blob_hash] = self._get_blob(blob_hash, length, is_mine)
|
||||||
if buffer.is_readable():
|
if buffer.is_readable():
|
||||||
with buffer.reader_context() as reader:
|
with buffer.reader_context() as reader:
|
||||||
self.blobs[blob_hash].write_blob(reader.read())
|
self.blobs[blob_hash].write_blob(reader.read())
|
||||||
if length and self.blobs[blob_hash].length is None:
|
if length and self.blobs[blob_hash].length is None:
|
||||||
self.blobs[blob_hash].set_length(length)
|
self.blobs[blob_hash].set_length(length)
|
||||||
else:
|
else:
|
||||||
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
|
self.blobs[blob_hash] = self._get_blob(blob_hash, length, is_mine)
|
||||||
return self.blobs[blob_hash]
|
return self.blobs[blob_hash]
|
||||||
|
|
||||||
def is_blob_verified(self, blob_hash: str, length: typing.Optional[int] = None) -> bool:
|
def is_blob_verified(self, blob_hash: str, length: typing.Optional[int] = None) -> bool:
|
||||||
@@ -105,9 +105,13 @@ class BlobManager:
|
|||||||
if isinstance(blob, BlobFile):
|
if isinstance(blob, BlobFile):
|
||||||
if blob.blob_hash not in self.completed_blob_hashes:
|
if blob.blob_hash not in self.completed_blob_hashes:
|
||||||
self.completed_blob_hashes.add(blob.blob_hash)
|
self.completed_blob_hashes.add(blob.blob_hash)
|
||||||
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=True))
|
return self.loop.create_task(self.storage.add_blobs(
|
||||||
|
(blob.blob_hash, blob.length, blob.added_on, blob.is_mine), finished=True)
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=False))
|
return self.loop.create_task(self.storage.add_blobs(
|
||||||
|
(blob.blob_hash, blob.length, blob.added_on, blob.is_mine), finished=False)
|
||||||
|
)
|
||||||
|
|
||||||
def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]:
|
def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]:
|
||||||
"""Returns of the blobhashes_to_check, which are valid"""
|
"""Returns of the blobhashes_to_check, which are valid"""
|
||||||
|
|||||||
@@ -0,0 +1,53 @@
|
|||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class DiskSpaceManager:
|
||||||
|
|
||||||
|
def __init__(self, config, db, blob_manager, cleaning_interval=30 * 60):
|
||||||
|
self.config = config
|
||||||
|
self.db = db
|
||||||
|
self.blob_manager = blob_manager
|
||||||
|
self.cleaning_interval = cleaning_interval
|
||||||
|
self.running = False
|
||||||
|
self.task = None
|
||||||
|
|
||||||
|
async def get_space_used_bytes(self):
|
||||||
|
return await self.db.get_stored_blob_disk_usage()
|
||||||
|
|
||||||
|
async def get_space_used_mb(self):
|
||||||
|
return int(await self.get_space_used_bytes()/1024.0/1024.0)
|
||||||
|
|
||||||
|
async def clean(self):
|
||||||
|
if not self.config.blob_storage_limit:
|
||||||
|
return 0
|
||||||
|
delete = []
|
||||||
|
available = (self.config.blob_storage_limit*1024*1024) - await self.get_space_used_bytes()
|
||||||
|
if available > 0:
|
||||||
|
return 0
|
||||||
|
for blob_hash, file_size, _ in await self.db.get_stored_blobs(is_mine=False):
|
||||||
|
delete.append(blob_hash)
|
||||||
|
available += file_size
|
||||||
|
if available > 0:
|
||||||
|
break
|
||||||
|
if delete:
|
||||||
|
await self.db.stop_all_files()
|
||||||
|
await self.blob_manager.delete_blobs(delete, delete_from_db=True)
|
||||||
|
return len(delete)
|
||||||
|
|
||||||
|
async def cleaning_loop(self):
|
||||||
|
while self.running:
|
||||||
|
await asyncio.sleep(self.cleaning_interval)
|
||||||
|
await self.clean()
|
||||||
|
|
||||||
|
async def start(self):
|
||||||
|
self.running = True
|
||||||
|
self.task = asyncio.create_task(self.cleaning_loop())
|
||||||
|
self.task.add_done_callback(lambda _: log.info("Stopping blob cleanup service."))
|
||||||
|
|
||||||
|
async def stop(self):
|
||||||
|
if self.running:
|
||||||
|
self.running = False
|
||||||
|
self.task.cancel()
|
||||||
@@ -152,6 +152,8 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
|||||||
log.debug(msg)
|
log.debug(msg)
|
||||||
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
|
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
|
||||||
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
|
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
|
||||||
|
# wait for the io to finish
|
||||||
|
await self.blob.verified.wait()
|
||||||
log.info("%s at %fMB/s", msg,
|
log.info("%s at %fMB/s", msg,
|
||||||
round((float(self._blob_bytes_received) /
|
round((float(self._blob_bytes_received) /
|
||||||
float(time.perf_counter() - start_time)) / 1000000.0, 2))
|
float(time.perf_counter() - start_time)) / 1000000.0, 2))
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ MAX_REQUEST_SIZE = 1200
|
|||||||
|
|
||||||
class BlobServerProtocol(asyncio.Protocol):
|
class BlobServerProtocol(asyncio.Protocol):
|
||||||
def __init__(self, loop: asyncio.AbstractEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str,
|
def __init__(self, loop: asyncio.AbstractEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str,
|
||||||
idle_timeout: float = 30.0, transfer_timeout: float = 60.0):
|
idle_timeout: float = 30.0, transfer_timeout: float = 60.0, blob_callback=None):
|
||||||
self.loop = loop
|
self.loop = loop
|
||||||
self.blob_manager = blob_manager
|
self.blob_manager = blob_manager
|
||||||
self.idle_timeout = idle_timeout
|
self.idle_timeout = idle_timeout
|
||||||
@@ -32,6 +32,7 @@ class BlobServerProtocol(asyncio.Protocol):
|
|||||||
self.started_transfer = asyncio.Event(loop=self.loop)
|
self.started_transfer = asyncio.Event(loop=self.loop)
|
||||||
self.transfer_finished = asyncio.Event(loop=self.loop)
|
self.transfer_finished = asyncio.Event(loop=self.loop)
|
||||||
self.close_on_idle_task: typing.Optional[asyncio.Task] = None
|
self.close_on_idle_task: typing.Optional[asyncio.Task] = None
|
||||||
|
self.blob_handling_callback: typing.Optional[typing.Callable] = blob_callback
|
||||||
|
|
||||||
async def close_on_idle(self):
|
async def close_on_idle(self):
|
||||||
while self.transport:
|
while self.transport:
|
||||||
@@ -92,6 +93,9 @@ class BlobServerProtocol(asyncio.Protocol):
|
|||||||
|
|
||||||
if download_request:
|
if download_request:
|
||||||
blob = self.blob_manager.get_blob(download_request.requested_blob)
|
blob = self.blob_manager.get_blob(download_request.requested_blob)
|
||||||
|
if self.blob_handling_callback:
|
||||||
|
await self.blob_handling_callback(blob)
|
||||||
|
blob = self.blob_manager.get_blob(download_request.requested_blob)
|
||||||
if blob.get_is_verified():
|
if blob.get_is_verified():
|
||||||
incoming_blob = {'blob_hash': blob.blob_hash, 'length': blob.length}
|
incoming_blob = {'blob_hash': blob.blob_hash, 'length': blob.length}
|
||||||
responses.append(BlobDownloadResponse(incoming_blob=incoming_blob))
|
responses.append(BlobDownloadResponse(incoming_blob=incoming_blob))
|
||||||
@@ -152,7 +156,7 @@ class BlobServerProtocol(asyncio.Protocol):
|
|||||||
|
|
||||||
class BlobServer:
|
class BlobServer:
|
||||||
def __init__(self, loop: asyncio.AbstractEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str,
|
def __init__(self, loop: asyncio.AbstractEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str,
|
||||||
idle_timeout: float = 30.0, transfer_timeout: float = 60.0):
|
idle_timeout: float = 30.0, transfer_timeout: float = 60.0, blob_callback=None):
|
||||||
self.loop = loop
|
self.loop = loop
|
||||||
self.blob_manager = blob_manager
|
self.blob_manager = blob_manager
|
||||||
self.server_task: typing.Optional[asyncio.Task] = None
|
self.server_task: typing.Optional[asyncio.Task] = None
|
||||||
@@ -161,6 +165,7 @@ class BlobServer:
|
|||||||
self.idle_timeout = idle_timeout
|
self.idle_timeout = idle_timeout
|
||||||
self.transfer_timeout = transfer_timeout
|
self.transfer_timeout = transfer_timeout
|
||||||
self.server_protocol_class = BlobServerProtocol
|
self.server_protocol_class = BlobServerProtocol
|
||||||
|
self.blob_handling_callback: typing.Optional[typing.Callable] = blob_callback
|
||||||
|
|
||||||
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
|
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
|
||||||
if self.server_task is not None:
|
if self.server_task is not None:
|
||||||
@@ -169,7 +174,8 @@ class BlobServer:
|
|||||||
async def _start_server():
|
async def _start_server():
|
||||||
server = await self.loop.create_server(
|
server = await self.loop.create_server(
|
||||||
lambda: self.server_protocol_class(self.loop, self.blob_manager, self.lbrycrd_address,
|
lambda: self.server_protocol_class(self.loop, self.blob_manager, self.lbrycrd_address,
|
||||||
self.idle_timeout, self.transfer_timeout),
|
self.idle_timeout, self.transfer_timeout,
|
||||||
|
blob_callback=self.blob_handling_callback),
|
||||||
interface, port
|
interface, port
|
||||||
)
|
)
|
||||||
self.started_listening.set()
|
self.started_listening.set()
|
||||||
|
|||||||
+148
-37
@@ -1,13 +1,12 @@
|
|||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
import typing
|
|
||||||
import logging
|
import logging
|
||||||
|
from typing import List, Dict, Tuple, Union, TypeVar, Generic, Optional
|
||||||
from argparse import ArgumentParser
|
from argparse import ArgumentParser
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
|
|
||||||
import yaml
|
|
||||||
from appdirs import user_data_dir, user_config_dir
|
from appdirs import user_data_dir, user_config_dir
|
||||||
|
import yaml
|
||||||
from lbry.error import InvalidCurrencyError
|
from lbry.error import InvalidCurrencyError
|
||||||
from lbry.dht import constants
|
from lbry.dht import constants
|
||||||
from lbry.wallet.coinselection import STRATEGIES
|
from lbry.wallet.coinselection import STRATEGIES
|
||||||
@@ -16,7 +15,7 @@ log = logging.getLogger(__name__)
|
|||||||
|
|
||||||
|
|
||||||
NOT_SET = type('NOT_SET', (object,), {}) # pylint: disable=invalid-name
|
NOT_SET = type('NOT_SET', (object,), {}) # pylint: disable=invalid-name
|
||||||
T = typing.TypeVar('T')
|
T = TypeVar('T')
|
||||||
|
|
||||||
CURRENCIES = {
|
CURRENCIES = {
|
||||||
'BTC': {'type': 'crypto'},
|
'BTC': {'type': 'crypto'},
|
||||||
@@ -25,11 +24,11 @@ CURRENCIES = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
class Setting(typing.Generic[T]):
|
class Setting(Generic[T]):
|
||||||
|
|
||||||
def __init__(self, doc: str, default: typing.Optional[T] = None,
|
def __init__(self, doc: str, default: Optional[T] = None,
|
||||||
previous_names: typing.Optional[typing.List[str]] = None,
|
previous_names: Optional[List[str]] = None,
|
||||||
metavar: typing.Optional[str] = None):
|
metavar: Optional[str] = None):
|
||||||
self.doc = doc
|
self.doc = doc
|
||||||
self.default = default
|
self.default = default
|
||||||
self.previous_names = previous_names or []
|
self.previous_names = previous_names or []
|
||||||
@@ -46,7 +45,7 @@ class Setting(typing.Generic[T]):
|
|||||||
def no_cli_name(self):
|
def no_cli_name(self):
|
||||||
return f"--no-{self.name.replace('_', '-')}"
|
return f"--no-{self.name.replace('_', '-')}"
|
||||||
|
|
||||||
def __get__(self, obj: typing.Optional['BaseConfig'], owner) -> T:
|
def __get__(self, obj: Optional['BaseConfig'], owner) -> T:
|
||||||
if obj is None:
|
if obj is None:
|
||||||
return self
|
return self
|
||||||
for location in obj.search_order:
|
for location in obj.search_order:
|
||||||
@@ -54,7 +53,7 @@ class Setting(typing.Generic[T]):
|
|||||||
return location[self.name]
|
return location[self.name]
|
||||||
return self.default
|
return self.default
|
||||||
|
|
||||||
def __set__(self, obj: 'BaseConfig', val: typing.Union[T, NOT_SET]):
|
def __set__(self, obj: 'BaseConfig', val: Union[T, NOT_SET]):
|
||||||
if val == NOT_SET:
|
if val == NOT_SET:
|
||||||
for location in obj.modify_order:
|
for location in obj.modify_order:
|
||||||
if self.name in location:
|
if self.name in location:
|
||||||
@@ -64,6 +63,18 @@ class Setting(typing.Generic[T]):
|
|||||||
for location in obj.modify_order:
|
for location in obj.modify_order:
|
||||||
location[self.name] = val
|
location[self.name] = val
|
||||||
|
|
||||||
|
def is_set(self, obj: 'BaseConfig') -> bool:
|
||||||
|
for location in obj.search_order:
|
||||||
|
if self.name in location:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def is_set_to_default(self, obj: 'BaseConfig') -> bool:
|
||||||
|
for location in obj.search_order:
|
||||||
|
if self.name in location:
|
||||||
|
return location[self.name] == self.default
|
||||||
|
return False
|
||||||
|
|
||||||
def validate(self, value):
|
def validate(self, value):
|
||||||
raise NotImplementedError()
|
raise NotImplementedError()
|
||||||
|
|
||||||
@@ -88,7 +99,7 @@ class String(Setting[str]):
|
|||||||
f"Setting '{self.name}' must be a string."
|
f"Setting '{self.name}' must be a string."
|
||||||
|
|
||||||
# TODO: removes this after pylint starts to understand generics
|
# TODO: removes this after pylint starts to understand generics
|
||||||
def __get__(self, obj: typing.Optional['BaseConfig'], owner) -> str: # pylint: disable=useless-super-delegation
|
def __get__(self, obj: Optional['BaseConfig'], owner) -> str: # pylint: disable=useless-super-delegation
|
||||||
return super().__get__(obj, owner)
|
return super().__get__(obj, owner)
|
||||||
|
|
||||||
|
|
||||||
@@ -192,7 +203,7 @@ class MaxKeyFee(Setting[dict]):
|
|||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
self.no_cli_name,
|
self.no_cli_name,
|
||||||
help=f"Disable maximum key fee check.",
|
help="Disable maximum key fee check.",
|
||||||
dest=self.name,
|
dest=self.name,
|
||||||
const=None,
|
const=None,
|
||||||
action="store_const",
|
action="store_const",
|
||||||
@@ -201,7 +212,7 @@ class MaxKeyFee(Setting[dict]):
|
|||||||
|
|
||||||
|
|
||||||
class StringChoice(String):
|
class StringChoice(String):
|
||||||
def __init__(self, doc: str, valid_values: typing.List[str], default: str, *args, **kwargs):
|
def __init__(self, doc: str, valid_values: List[str], default: str, *args, **kwargs):
|
||||||
super().__init__(doc, default, *args, **kwargs)
|
super().__init__(doc, default, *args, **kwargs)
|
||||||
if not valid_values:
|
if not valid_values:
|
||||||
raise ValueError("No valid values provided")
|
raise ValueError("No valid values provided")
|
||||||
@@ -274,17 +285,95 @@ class Strings(ListSetting):
|
|||||||
f"'{self.name}' must be a string."
|
f"'{self.name}' must be a string."
|
||||||
|
|
||||||
|
|
||||||
|
class KnownHubsList:
|
||||||
|
|
||||||
|
def __init__(self, config: 'Config' = None, file_name: str = 'known_hubs.yml'):
|
||||||
|
self.file_name = file_name
|
||||||
|
self.path = os.path.join(config.wallet_dir, self.file_name) if config else None
|
||||||
|
self.hubs: Dict[Tuple[str, int], Dict] = {}
|
||||||
|
if self.exists:
|
||||||
|
self.load()
|
||||||
|
|
||||||
|
@property
|
||||||
|
def exists(self):
|
||||||
|
return self.path and os.path.exists(self.path)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def serialized(self) -> Dict[str, Dict]:
|
||||||
|
return {f"{host}:{port}": details for (host, port), details in self.hubs.items()}
|
||||||
|
|
||||||
|
def filter(self, match_none=False, **kwargs):
|
||||||
|
if not kwargs:
|
||||||
|
return self.hubs
|
||||||
|
result = {}
|
||||||
|
for hub, details in self.hubs.items():
|
||||||
|
for key, constraint in kwargs.items():
|
||||||
|
value = details.get(key)
|
||||||
|
if value == constraint or (match_none and value is None):
|
||||||
|
result[hub] = details
|
||||||
|
break
|
||||||
|
return result
|
||||||
|
|
||||||
|
def load(self):
|
||||||
|
if self.path:
|
||||||
|
with open(self.path, 'r') as known_hubs_file:
|
||||||
|
raw = known_hubs_file.read()
|
||||||
|
for hub, details in yaml.safe_load(raw).items():
|
||||||
|
self.set(hub, details)
|
||||||
|
|
||||||
|
def save(self):
|
||||||
|
if self.path:
|
||||||
|
with open(self.path, 'w') as known_hubs_file:
|
||||||
|
known_hubs_file.write(yaml.safe_dump(self.serialized, default_flow_style=False))
|
||||||
|
|
||||||
|
def set(self, hub: str, details: Dict):
|
||||||
|
if hub and hub.count(':') == 1:
|
||||||
|
host, port = hub.split(':')
|
||||||
|
hub_parts = (host, int(port))
|
||||||
|
if hub_parts not in self.hubs:
|
||||||
|
self.hubs[hub_parts] = details
|
||||||
|
return hub
|
||||||
|
|
||||||
|
def add_hubs(self, hubs: List[str]):
|
||||||
|
added = False
|
||||||
|
for hub in hubs:
|
||||||
|
if self.set(hub, {}) is not None:
|
||||||
|
added = True
|
||||||
|
return added
|
||||||
|
|
||||||
|
def items(self):
|
||||||
|
return self.hubs.items()
|
||||||
|
|
||||||
|
def __bool__(self):
|
||||||
|
return len(self) > 0
|
||||||
|
|
||||||
|
def __len__(self):
|
||||||
|
return self.hubs.__len__()
|
||||||
|
|
||||||
|
def __iter__(self):
|
||||||
|
return iter(self.hubs)
|
||||||
|
|
||||||
|
|
||||||
class EnvironmentAccess:
|
class EnvironmentAccess:
|
||||||
PREFIX = 'LBRY_'
|
PREFIX = 'LBRY_'
|
||||||
|
|
||||||
def __init__(self, environ: dict):
|
def __init__(self, config: 'BaseConfig', environ: dict):
|
||||||
self.environ = environ
|
self.configuration = config
|
||||||
|
self.data = {}
|
||||||
|
if environ:
|
||||||
|
self.load(environ)
|
||||||
|
|
||||||
|
def load(self, environ):
|
||||||
|
for setting in self.configuration.get_settings():
|
||||||
|
value = environ.get(f'{self.PREFIX}{setting.name.upper()}', NOT_SET)
|
||||||
|
if value != NOT_SET and not (isinstance(setting, ListSetting) and value is None):
|
||||||
|
self.data[setting.name] = setting.deserialize(value)
|
||||||
|
|
||||||
def __contains__(self, item: str):
|
def __contains__(self, item: str):
|
||||||
return f'{self.PREFIX}{item.upper()}' in self.environ
|
return item in self.data
|
||||||
|
|
||||||
def __getitem__(self, item: str):
|
def __getitem__(self, item: str):
|
||||||
return self.environ[f'{self.PREFIX}{item.upper()}']
|
return self.data[item]
|
||||||
|
|
||||||
|
|
||||||
class ArgumentAccess:
|
class ArgumentAccess:
|
||||||
@@ -325,7 +414,7 @@ class ConfigFileAccess:
|
|||||||
cls = type(self.configuration)
|
cls = type(self.configuration)
|
||||||
with open(self.path, 'r') as config_file:
|
with open(self.path, 'r') as config_file:
|
||||||
raw = config_file.read()
|
raw = config_file.read()
|
||||||
serialized = yaml.load(raw) or {}
|
serialized = yaml.safe_load(raw) or {}
|
||||||
for key, value in serialized.items():
|
for key, value in serialized.items():
|
||||||
attr = getattr(cls, key, None)
|
attr = getattr(cls, key, None)
|
||||||
if attr is None:
|
if attr is None:
|
||||||
@@ -369,7 +458,7 @@ class ConfigFileAccess:
|
|||||||
del self.data[key]
|
del self.data[key]
|
||||||
|
|
||||||
|
|
||||||
TBC = typing.TypeVar('TBC', bound='BaseConfig')
|
TBC = TypeVar('TBC', bound='BaseConfig')
|
||||||
|
|
||||||
|
|
||||||
class BaseConfig:
|
class BaseConfig:
|
||||||
@@ -443,7 +532,7 @@ class BaseConfig:
|
|||||||
self.arguments = ArgumentAccess(self, args)
|
self.arguments = ArgumentAccess(self, args)
|
||||||
|
|
||||||
def set_environment(self, environ=None):
|
def set_environment(self, environ=None):
|
||||||
self.environment = EnvironmentAccess(environ or os.environ)
|
self.environment = EnvironmentAccess(self, environ or os.environ)
|
||||||
|
|
||||||
def set_persisted(self, config_file_path=None):
|
def set_persisted(self, config_file_path=None):
|
||||||
if config_file_path is None:
|
if config_file_path is None:
|
||||||
@@ -464,15 +553,22 @@ class BaseConfig:
|
|||||||
|
|
||||||
class TranscodeConfig(BaseConfig):
|
class TranscodeConfig(BaseConfig):
|
||||||
|
|
||||||
ffmpeg_folder = String('The path to ffmpeg and ffprobe', '')
|
ffmpeg_path = String('A list of places to check for ffmpeg and ffprobe. '
|
||||||
|
f'$data_dir/ffmpeg/bin and $PATH are checked afterward. Separator: {os.pathsep}',
|
||||||
|
'', previous_names=['ffmpeg_folder'])
|
||||||
video_encoder = String('FFmpeg codec and parameters for the video encoding. '
|
video_encoder = String('FFmpeg codec and parameters for the video encoding. '
|
||||||
'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental',
|
'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental',
|
||||||
'libx264 -crf 18 -vf "format=yuv420p"')
|
'libx264 -crf 24 -preset faster -pix_fmt yuv420p')
|
||||||
|
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 5_000_000)
|
||||||
|
video_scaler = String('FFmpeg scaling parameters for reducing bitrate. '
|
||||||
|
'Example: -vf "scale=-2:720,fps=24" -maxrate 5M -bufsize 3M',
|
||||||
|
r'-vf "scale=if(gte(iw\,ih)\,min(1920\,iw)\,-2):if(lt(iw\,ih)\,min(1920\,ih)\,-2)" '
|
||||||
|
r'-maxrate 5500K -bufsize 5000K')
|
||||||
audio_encoder = String('FFmpeg codec and parameters for the audio encoding. '
|
audio_encoder = String('FFmpeg codec and parameters for the audio encoding. '
|
||||||
'Example: libopus -b:a 128k',
|
'Example: libopus -b:a 128k',
|
||||||
'aac -b:a 192k')
|
'aac -b:a 160k')
|
||||||
volume_filter = String('FFmpeg filter for audio normalization.', '-af loudnorm')
|
volume_filter = String('FFmpeg filter for audio normalization. Exmple: -af loudnorm', '')
|
||||||
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', '240')
|
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', 240)
|
||||||
|
|
||||||
|
|
||||||
class CLIConfig(TranscodeConfig):
|
class CLIConfig(TranscodeConfig):
|
||||||
@@ -493,6 +589,9 @@ class CLIConfig(TranscodeConfig):
|
|||||||
|
|
||||||
|
|
||||||
class Config(CLIConfig):
|
class Config(CLIConfig):
|
||||||
|
|
||||||
|
jurisdiction = String("Limit interactions to wallet server in this jurisdiction.")
|
||||||
|
|
||||||
# directories
|
# directories
|
||||||
data_dir = Path("Directory path to store blobs.", metavar='DIR')
|
data_dir = Path("Directory path to store blobs.", metavar='DIR')
|
||||||
download_dir = Path(
|
download_dir = Path(
|
||||||
@@ -529,11 +628,13 @@ class Config(CLIConfig):
|
|||||||
# protocol timeouts
|
# protocol timeouts
|
||||||
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
|
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
|
||||||
blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
|
blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
|
||||||
|
hub_timeout = Float("Timeout when making a hub request", 30.0)
|
||||||
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0)
|
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0)
|
||||||
node_rpc_timeout = Float("Timeout when making a DHT request", constants.RPC_TIMEOUT)
|
node_rpc_timeout = Float("Timeout when making a DHT request", constants.RPC_TIMEOUT)
|
||||||
|
|
||||||
# blob announcement and download
|
# blob announcement and download
|
||||||
save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True)
|
save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True)
|
||||||
|
blob_storage_limit = Integer("Disk space in MB to be allocated for blob storage. 0 = no limit", 0)
|
||||||
blob_lru_cache_size = Integer(
|
blob_lru_cache_size = Integer(
|
||||||
"LRU cache size for decrypted downloaded blobs used to minimize re-downloading the same blobs when "
|
"LRU cache size for decrypted downloaded blobs used to minimize re-downloading the same blobs when "
|
||||||
"replying to a range request. Set to 0 to disable.", 32
|
"replying to a range request. Set to 0 to disable.", 32
|
||||||
@@ -550,6 +651,7 @@ class Config(CLIConfig):
|
|||||||
"Maximum number of peers to connect to while downloading a blob", 4,
|
"Maximum number of peers to connect to while downloading a blob", 4,
|
||||||
previous_names=['max_connections_per_stream']
|
previous_names=['max_connections_per_stream']
|
||||||
)
|
)
|
||||||
|
concurrent_hub_requests = Integer("Maximum number of concurrent hub requests", 32)
|
||||||
fixed_peer_delay = Float(
|
fixed_peer_delay = Float(
|
||||||
"Amount of seconds before adding the reflector servers as potential peers to download from in case dht"
|
"Amount of seconds before adding the reflector servers as potential peers to download from in case dht"
|
||||||
"peers are not found or are slow", 2.0
|
"peers are not found or are slow", 2.0
|
||||||
@@ -570,9 +672,14 @@ class Config(CLIConfig):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# servers
|
# servers
|
||||||
reflector_servers = Servers("Reflector re-hosting servers", [
|
reflector_servers = Servers("Reflector re-hosting servers for mirroring publishes", [
|
||||||
('reflector.lbry.com', 5566)
|
('reflector.lbry.com', 5566)
|
||||||
])
|
])
|
||||||
|
|
||||||
|
fixed_peers = Servers("Fixed peers to fall back to if none are found on P2P for a blob", [
|
||||||
|
('cdn.reflector.lbry.com', 5567)
|
||||||
|
])
|
||||||
|
|
||||||
lbryum_servers = Servers("SPV wallet servers", [
|
lbryum_servers = Servers("SPV wallet servers", [
|
||||||
('spv11.lbry.com', 50001),
|
('spv11.lbry.com', 50001),
|
||||||
('spv12.lbry.com', 50001),
|
('spv12.lbry.com', 50001),
|
||||||
@@ -585,27 +692,28 @@ class Config(CLIConfig):
|
|||||||
('spv19.lbry.com', 50001),
|
('spv19.lbry.com', 50001),
|
||||||
])
|
])
|
||||||
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
|
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
|
||||||
|
('dht.lbry.grin.io', 4444), # Grin
|
||||||
|
('dht.lbry.madiator.com', 4444), # Madiator
|
||||||
('lbrynet1.lbry.com', 4444), # US EAST
|
('lbrynet1.lbry.com', 4444), # US EAST
|
||||||
('lbrynet2.lbry.com', 4444), # US WEST
|
('lbrynet2.lbry.com', 4444), # US WEST
|
||||||
('lbrynet3.lbry.com', 4444), # EU
|
('lbrynet3.lbry.com', 4444), # EU
|
||||||
('lbrynet4.lbry.com', 4444) # ASIA
|
('lbrynet4.lbry.com', 4444) # ASIA
|
||||||
])
|
])
|
||||||
|
|
||||||
comment_server = String("Comment server API URL", "https://comments.lbry.com/api")
|
|
||||||
|
|
||||||
# blockchain
|
# blockchain
|
||||||
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
|
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
|
||||||
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
|
|
||||||
cache_time = Integer("Time to cache resolved claims", 150) # TODO: use this
|
|
||||||
|
|
||||||
# daemon
|
# daemon
|
||||||
save_files = Toggle("Save downloaded files when calling `get` by default", True)
|
save_files = Toggle("Save downloaded files when calling `get` by default", False)
|
||||||
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
|
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
|
||||||
share_usage_data = Toggle(
|
share_usage_data = Toggle(
|
||||||
"Whether to share usage stats and diagnostic info with LBRY.", False,
|
"Whether to share usage stats and diagnostic info with LBRY.", False,
|
||||||
previous_names=['upload_log', 'upload_log', 'share_debug_info']
|
previous_names=['upload_log', 'upload_log', 'share_debug_info']
|
||||||
)
|
)
|
||||||
track_bandwidth = Toggle("Track bandwidth usage", True)
|
track_bandwidth = Toggle("Track bandwidth usage", True)
|
||||||
|
allowed_origin = String(
|
||||||
|
"Allowed `Origin` header value for API request (sent by browser), use * to allow "
|
||||||
|
"all hosts; default is to only allow API requests with no `Origin` value.", "")
|
||||||
|
|
||||||
# media server
|
# media server
|
||||||
streaming_server = String('Host name and port to serve streaming media over range requests',
|
streaming_server = String('Host name and port to serve streaming media over range requests',
|
||||||
@@ -617,6 +725,7 @@ class Config(CLIConfig):
|
|||||||
"Strategy to use when selecting UTXOs for a transaction",
|
"Strategy to use when selecting UTXOs for a transaction",
|
||||||
STRATEGIES, "standard")
|
STRATEGIES, "standard")
|
||||||
|
|
||||||
|
transaction_cache_size = Integer("Transaction cache size", 2 ** 17)
|
||||||
save_resolved_claims = Toggle(
|
save_resolved_claims = Toggle(
|
||||||
"Save content claims to the database when they are resolved to keep file_list up to date, "
|
"Save content claims to the database when they are resolved to keep file_list up to date, "
|
||||||
"only disable this if file_x commands are not needed", True
|
"only disable this if file_x commands are not needed", True
|
||||||
@@ -633,6 +742,7 @@ class Config(CLIConfig):
|
|||||||
def __init__(self, **kwargs):
|
def __init__(self, **kwargs):
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
self.set_default_paths()
|
self.set_default_paths()
|
||||||
|
self.known_hubs = KnownHubsList(self)
|
||||||
|
|
||||||
def set_default_paths(self):
|
def set_default_paths(self):
|
||||||
if 'darwin' in sys.platform.lower():
|
if 'darwin' in sys.platform.lower():
|
||||||
@@ -654,7 +764,7 @@ class Config(CLIConfig):
|
|||||||
return os.path.join(self.data_dir, 'lbrynet.log')
|
return os.path.join(self.data_dir, 'lbrynet.log')
|
||||||
|
|
||||||
|
|
||||||
def get_windows_directories() -> typing.Tuple[str, str, str]:
|
def get_windows_directories() -> Tuple[str, str, str]:
|
||||||
from lbry.winpaths import get_path, FOLDERID, UserHandle, \
|
from lbry.winpaths import get_path, FOLDERID, UserHandle, \
|
||||||
PathNotFoundException # pylint: disable=import-outside-toplevel
|
PathNotFoundException # pylint: disable=import-outside-toplevel
|
||||||
|
|
||||||
@@ -676,19 +786,20 @@ def get_windows_directories() -> typing.Tuple[str, str, str]:
|
|||||||
return data_dir, lbryum_dir, download_dir
|
return data_dir, lbryum_dir, download_dir
|
||||||
|
|
||||||
|
|
||||||
def get_darwin_directories() -> typing.Tuple[str, str, str]:
|
def get_darwin_directories() -> Tuple[str, str, str]:
|
||||||
data_dir = user_data_dir('LBRY')
|
data_dir = user_data_dir('LBRY')
|
||||||
lbryum_dir = os.path.expanduser('~/.lbryum')
|
lbryum_dir = os.path.expanduser('~/.lbryum')
|
||||||
download_dir = os.path.expanduser('~/Downloads')
|
download_dir = os.path.expanduser('~/Downloads')
|
||||||
return data_dir, lbryum_dir, download_dir
|
return data_dir, lbryum_dir, download_dir
|
||||||
|
|
||||||
|
|
||||||
def get_linux_directories() -> typing.Tuple[str, str, str]:
|
def get_linux_directories() -> Tuple[str, str, str]:
|
||||||
try:
|
try:
|
||||||
with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg:
|
with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg:
|
||||||
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()).group(1)
|
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read())
|
||||||
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir)
|
if down_dir:
|
||||||
download_dir = re.sub('\"', '', down_dir)
|
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir.group(1))
|
||||||
|
download_dir = re.sub('\"', '', down_dir)
|
||||||
except OSError:
|
except OSError:
|
||||||
download_dir = os.getenv('XDG_DOWNLOAD_DIR')
|
download_dir = os.getenv('XDG_DOWNLOAD_DIR')
|
||||||
if not download_dir:
|
if not download_dir:
|
||||||
|
|||||||
+3
-4
@@ -1,7 +1,6 @@
|
|||||||
import logging
|
import logging
|
||||||
import asyncio
|
import asyncio
|
||||||
import typing
|
import typing
|
||||||
import binascii
|
|
||||||
import socket
|
import socket
|
||||||
from lbry.utils import resolve_host
|
from lbry.utils import resolve_host
|
||||||
from lbry.dht import constants
|
from lbry.dht import constants
|
||||||
@@ -80,7 +79,7 @@ class Node:
|
|||||||
await fut
|
await fut
|
||||||
|
|
||||||
async def announce_blob(self, blob_hash: str) -> typing.List[bytes]:
|
async def announce_blob(self, blob_hash: str) -> typing.List[bytes]:
|
||||||
hash_value = binascii.unhexlify(blob_hash.encode())
|
hash_value = bytes.fromhex(blob_hash)
|
||||||
assert len(hash_value) == constants.HASH_LENGTH
|
assert len(hash_value) == constants.HASH_LENGTH
|
||||||
peers = await self.peer_search(hash_value)
|
peers = await self.peer_search(hash_value)
|
||||||
|
|
||||||
@@ -95,7 +94,7 @@ class Node:
|
|||||||
stored_to = [node_id for node_id, contacted in stored_to_tup if contacted]
|
stored_to = [node_id for node_id, contacted in stored_to_tup if contacted]
|
||||||
if stored_to:
|
if stored_to:
|
||||||
log.debug(
|
log.debug(
|
||||||
"Stored %s to %i of %i attempted peers", binascii.hexlify(hash_value).decode()[:8],
|
"Stored %s to %i of %i attempted peers", hash_value.hex()[:8],
|
||||||
len(stored_to), len(peers)
|
len(stored_to), len(peers)
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
@@ -223,7 +222,7 @@ class Node:
|
|||||||
# prioritize peers who reply to a dht ping first
|
# prioritize peers who reply to a dht ping first
|
||||||
# this minimizes attempting to make tcp connections that won't work later to dead or unreachable peers
|
# this minimizes attempting to make tcp connections that won't work later to dead or unreachable peers
|
||||||
|
|
||||||
async for results in self.get_iterative_value_finder(binascii.unhexlify(blob_hash.encode())):
|
async for results in self.get_iterative_value_finder(bytes.fromhex(blob_hash)):
|
||||||
to_put = []
|
to_put = []
|
||||||
for peer in results:
|
for peer in results:
|
||||||
if peer.address == self.protocol.external_ip and self.protocol.peer_port == peer.tcp_port:
|
if peer.address == self.protocol.external_ip and self.protocol.peer_port == peer.tcp_port:
|
||||||
|
|||||||
+14
-32
@@ -1,14 +1,13 @@
|
|||||||
import typing
|
import typing
|
||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
import ipaddress
|
|
||||||
from binascii import hexlify
|
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
|
from lbry.utils import is_valid_public_ipv4 as _is_valid_public_ipv4, LRUCache
|
||||||
from lbry.dht import constants
|
from lbry.dht import constants
|
||||||
from lbry.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
|
from lbry.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
|
||||||
|
|
||||||
|
ALLOW_LOCALHOST = False
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
@@ -20,26 +19,9 @@ def make_kademlia_peer(node_id: typing.Optional[bytes], address: typing.Optional
|
|||||||
return KademliaPeer(address, node_id, udp_port, tcp_port=tcp_port, allow_localhost=allow_localhost)
|
return KademliaPeer(address, node_id, udp_port, tcp_port=tcp_port, allow_localhost=allow_localhost)
|
||||||
|
|
||||||
|
|
||||||
# the ipaddress module does not show these subnets as reserved
|
|
||||||
CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
|
|
||||||
IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
|
|
||||||
|
|
||||||
ALLOW_LOCALHOST = False
|
|
||||||
|
|
||||||
|
|
||||||
def is_valid_public_ipv4(address, allow_localhost: bool = False):
|
def is_valid_public_ipv4(address, allow_localhost: bool = False):
|
||||||
allow_localhost = bool(allow_localhost or ALLOW_LOCALHOST)
|
allow_localhost = bool(allow_localhost or ALLOW_LOCALHOST)
|
||||||
try:
|
return _is_valid_public_ipv4(address, allow_localhost)
|
||||||
parsed_ip = ipaddress.ip_address(address)
|
|
||||||
if parsed_ip.is_loopback and allow_localhost:
|
|
||||||
return True
|
|
||||||
return not any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local,
|
|
||||||
parsed_ip.is_loopback, parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private,
|
|
||||||
parsed_ip.is_reserved,
|
|
||||||
CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
|
|
||||||
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
|
|
||||||
except ipaddress.AddressValueError:
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
class PeerManager:
|
class PeerManager:
|
||||||
@@ -48,12 +30,12 @@ class PeerManager:
|
|||||||
self._rpc_failures: typing.Dict[
|
self._rpc_failures: typing.Dict[
|
||||||
typing.Tuple[str, int], typing.Tuple[typing.Optional[float], typing.Optional[float]]
|
typing.Tuple[str, int], typing.Tuple[typing.Optional[float], typing.Optional[float]]
|
||||||
] = {}
|
] = {}
|
||||||
self._last_replied: typing.Dict[typing.Tuple[str, int], float] = {}
|
self._last_replied: typing.Dict[typing.Tuple[str, int], float] = LRUCache(2048)
|
||||||
self._last_sent: typing.Dict[typing.Tuple[str, int], float] = {}
|
self._last_sent: typing.Dict[typing.Tuple[str, int], float] = LRUCache(2048)
|
||||||
self._last_requested: typing.Dict[typing.Tuple[str, int], float] = {}
|
self._last_requested: typing.Dict[typing.Tuple[str, int], float] = LRUCache(2048)
|
||||||
self._node_id_mapping: typing.Dict[typing.Tuple[str, int], bytes] = {}
|
self._node_id_mapping: typing.Dict[typing.Tuple[str, int], bytes] = LRUCache(2048)
|
||||||
self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = {}
|
self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = LRUCache(2048)
|
||||||
self._node_tokens: typing.Dict[bytes, (float, bytes)] = {}
|
self._node_tokens: typing.Dict[bytes, (float, bytes)] = LRUCache(2048)
|
||||||
|
|
||||||
def reset(self):
|
def reset(self):
|
||||||
for statistic in (self._rpc_failures, self._last_replied, self._last_sent, self._last_requested):
|
for statistic in (self._rpc_failures, self._last_replied, self._last_sent, self._last_requested):
|
||||||
@@ -171,11 +153,11 @@ class KademliaPeer:
|
|||||||
def __post_init__(self):
|
def __post_init__(self):
|
||||||
if self._node_id is not None:
|
if self._node_id is not None:
|
||||||
if not len(self._node_id) == constants.HASH_LENGTH:
|
if not len(self._node_id) == constants.HASH_LENGTH:
|
||||||
raise ValueError("invalid node_id: {}".format(hexlify(self._node_id).decode()))
|
raise ValueError("invalid node_id: {}".format(self._node_id.hex()))
|
||||||
if self.udp_port is not None and not 1 <= self.udp_port <= 65535:
|
if self.udp_port is not None and not 1024 <= self.udp_port <= 65535:
|
||||||
raise ValueError("invalid udp port")
|
raise ValueError(f"invalid udp port: {self.address}:{self.udp_port}")
|
||||||
if self.tcp_port is not None and not 1 <= self.tcp_port <= 65535:
|
if self.tcp_port is not None and not 1024 <= self.tcp_port <= 65535:
|
||||||
raise ValueError("invalid tcp port")
|
raise ValueError(f"invalid tcp port: {self.address}:{self.tcp_port}")
|
||||||
if not is_valid_public_ipv4(self.address, self.allow_localhost):
|
if not is_valid_public_ipv4(self.address, self.allow_localhost):
|
||||||
raise ValueError(f"invalid ip address: '{self.address}'")
|
raise ValueError(f"invalid ip address: '{self.address}'")
|
||||||
|
|
||||||
|
|||||||
@@ -16,6 +16,9 @@ class DictDataStore:
|
|||||||
self._peer_manager = peer_manager
|
self._peer_manager = peer_manager
|
||||||
self.completed_blobs: typing.Set[str] = set()
|
self.completed_blobs: typing.Set[str] = set()
|
||||||
|
|
||||||
|
def __len__(self):
|
||||||
|
return self._data_store.__len__()
|
||||||
|
|
||||||
def removed_expired_peers(self):
|
def removed_expired_peers(self):
|
||||||
now = self.loop.time()
|
now = self.loop.time()
|
||||||
keys = list(self._data_store.keys())
|
keys = list(self._data_store.keys())
|
||||||
|
|||||||
@@ -1,5 +1,4 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
from binascii import hexlify
|
|
||||||
from itertools import chain
|
from itertools import chain
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
import typing
|
import typing
|
||||||
@@ -198,7 +197,7 @@ class IterativeFinder:
|
|||||||
added += 1
|
added += 1
|
||||||
log.debug("running %d probes", len(self.running_probes))
|
log.debug("running %d probes", len(self.running_probes))
|
||||||
if not added and not self.running_probes:
|
if not added and not self.running_probes:
|
||||||
log.debug("search for %s exhausted", hexlify(self.key)[:8])
|
log.debug("search for %s exhausted", self.key.hex()[:8])
|
||||||
self.search_exhausted()
|
self.search_exhausted()
|
||||||
|
|
||||||
def _schedule_probe(self, peer: 'KademliaPeer'):
|
def _schedule_probe(self, peer: 'KademliaPeer'):
|
||||||
@@ -271,7 +270,7 @@ class IterativeNodeFinder(IterativeFinder):
|
|||||||
self.yielded_peers: typing.Set['KademliaPeer'] = set()
|
self.yielded_peers: typing.Set['KademliaPeer'] = set()
|
||||||
|
|
||||||
async def send_probe(self, peer: 'KademliaPeer') -> FindNodeResponse:
|
async def send_probe(self, peer: 'KademliaPeer') -> FindNodeResponse:
|
||||||
log.debug("probing %s:%d %s", peer.address, peer.udp_port, hexlify(peer.node_id)[:8] if peer.node_id else '')
|
log.debug("probing %s:%d %s", peer.address, peer.udp_port, peer.node_id.hex()[:8] if peer.node_id else '')
|
||||||
response = await self.protocol.get_rpc_peer(peer).find_node(self.key)
|
response = await self.protocol.get_rpc_peer(peer).find_node(self.key)
|
||||||
return FindNodeResponse(self.key, response)
|
return FindNodeResponse(self.key, response)
|
||||||
|
|
||||||
|
|||||||
@@ -4,12 +4,12 @@ import functools
|
|||||||
import hashlib
|
import hashlib
|
||||||
import asyncio
|
import asyncio
|
||||||
import typing
|
import typing
|
||||||
import binascii
|
|
||||||
import random
|
import random
|
||||||
from asyncio.protocols import DatagramProtocol
|
from asyncio.protocols import DatagramProtocol
|
||||||
from asyncio.transports import DatagramTransport
|
from asyncio.transports import DatagramTransport
|
||||||
|
|
||||||
from lbry.dht import constants
|
from lbry.dht import constants
|
||||||
|
from lbry.dht.serialization.bencoding import DecodeError
|
||||||
from lbry.dht.serialization.datagram import decode_datagram, ErrorDatagram, ResponseDatagram, RequestDatagram
|
from lbry.dht.serialization.datagram import decode_datagram, ErrorDatagram, ResponseDatagram, RequestDatagram
|
||||||
from lbry.dht.serialization.datagram import RESPONSE_TYPE, ERROR_TYPE, PAGE_KEY
|
from lbry.dht.serialization.datagram import RESPONSE_TYPE, ERROR_TYPE, PAGE_KEY
|
||||||
from lbry.dht.error import RemoteException, TransportNotConnected
|
from lbry.dht.error import RemoteException, TransportNotConnected
|
||||||
@@ -96,8 +96,7 @@ class KademliaRPC:
|
|||||||
if not rpc_contact.tcp_port or peer.compact_address_tcp() != rpc_contact.compact_address_tcp()
|
if not rpc_contact.tcp_port or peer.compact_address_tcp() != rpc_contact.compact_address_tcp()
|
||||||
]
|
]
|
||||||
# if we don't have k storing peers to return and we have this hash locally, include our contact information
|
# if we don't have k storing peers to return and we have this hash locally, include our contact information
|
||||||
if len(peers) < constants.K and binascii.hexlify(key).decode() in self.protocol.data_store.completed_blobs:
|
peers.append(self.compact_address())
|
||||||
peers.append(self.compact_address())
|
|
||||||
if not peers:
|
if not peers:
|
||||||
response[PAGE_KEY] = 0
|
response[PAGE_KEY] = 0
|
||||||
else:
|
else:
|
||||||
@@ -414,8 +413,8 @@ class KademliaProtocol(DatagramProtocol):
|
|||||||
self._wakeup_routing_task.clear()
|
self._wakeup_routing_task.clear()
|
||||||
|
|
||||||
def _handle_rpc(self, sender_contact: 'KademliaPeer', message: RequestDatagram):
|
def _handle_rpc(self, sender_contact: 'KademliaPeer', message: RequestDatagram):
|
||||||
assert sender_contact.node_id != self.node_id, (binascii.hexlify(sender_contact.node_id)[:8].decode(),
|
assert sender_contact.node_id != self.node_id, (sender_contact.node_id.hex()[:8],
|
||||||
binascii.hexlify(self.node_id)[:8].decode())
|
self.node_id.hex()[:8])
|
||||||
method = message.method
|
method = message.method
|
||||||
if method not in [b'ping', b'store', b'findNode', b'findValue']:
|
if method not in [b'ping', b'store', b'findNode', b'findValue']:
|
||||||
raise AttributeError('Invalid method: %s' % message.method.decode())
|
raise AttributeError('Invalid method: %s' % message.method.decode())
|
||||||
@@ -451,7 +450,11 @@ class KademliaProtocol(DatagramProtocol):
|
|||||||
try:
|
try:
|
||||||
peer = self.routing_table.get_peer(request_datagram.node_id)
|
peer = self.routing_table.get_peer(request_datagram.node_id)
|
||||||
except IndexError:
|
except IndexError:
|
||||||
peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
try:
|
||||||
|
peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||||
|
except ValueError as err:
|
||||||
|
log.warning("error replying to %s: %s", address[0], str(err))
|
||||||
|
return
|
||||||
try:
|
try:
|
||||||
self._handle_rpc(peer, request_datagram)
|
self._handle_rpc(peer, request_datagram)
|
||||||
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
|
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
|
||||||
@@ -551,12 +554,12 @@ class KademliaProtocol(DatagramProtocol):
|
|||||||
address[0], address[1], OLD_PROTOCOL_ERRORS[error_datagram.response]
|
address[0], address[1], OLD_PROTOCOL_ERRORS[error_datagram.response]
|
||||||
)
|
)
|
||||||
|
|
||||||
def datagram_received(self, datagram: bytes, address: typing.Tuple[str, int]) -> None: # pylint: disable=arguments-differ
|
def datagram_received(self, datagram: bytes, address: typing.Tuple[str, int]) -> None: # pylint: disable=arguments-renamed
|
||||||
try:
|
try:
|
||||||
message = decode_datagram(datagram)
|
message = decode_datagram(datagram)
|
||||||
except (ValueError, TypeError):
|
except (ValueError, TypeError, DecodeError):
|
||||||
self.peer_manager.report_failure(address[0], address[1])
|
self.peer_manager.report_failure(address[0], address[1])
|
||||||
log.warning("Couldn't decode dht datagram from %s: %s", address, binascii.hexlify(datagram).decode())
|
log.warning("Couldn't decode dht datagram from %s: %s", address, datagram.hex())
|
||||||
return
|
return
|
||||||
|
|
||||||
if isinstance(message, RequestDatagram):
|
if isinstance(message, RequestDatagram):
|
||||||
@@ -598,7 +601,7 @@ class KademliaProtocol(DatagramProtocol):
|
|||||||
if len(data) > constants.MSG_SIZE_LIMIT:
|
if len(data) > constants.MSG_SIZE_LIMIT:
|
||||||
log.warning("cannot send datagram larger than %i bytes (packet is %i bytes)",
|
log.warning("cannot send datagram larger than %i bytes (packet is %i bytes)",
|
||||||
constants.MSG_SIZE_LIMIT, len(data))
|
constants.MSG_SIZE_LIMIT, len(data))
|
||||||
log.debug("Packet is too large to send: %s", binascii.hexlify(data[:3500]).decode())
|
log.debug("Packet is too large to send: %s", data[:3500].hex())
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"cannot send datagram larger than {constants.MSG_SIZE_LIMIT} bytes (packet is {len(data)} bytes)"
|
f"cannot send datagram larger than {constants.MSG_SIZE_LIMIT} bytes (packet is {len(data)} bytes)"
|
||||||
)
|
)
|
||||||
@@ -658,13 +661,13 @@ class KademliaProtocol(DatagramProtocol):
|
|||||||
res = await self.get_rpc_peer(peer).store(hash_value)
|
res = await self.get_rpc_peer(peer).store(hash_value)
|
||||||
if res != b"OK":
|
if res != b"OK":
|
||||||
raise ValueError(res)
|
raise ValueError(res)
|
||||||
log.debug("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
log.debug("Stored %s to %s", hash_value.hex()[:8], peer)
|
||||||
return peer.node_id, True
|
return peer.node_id, True
|
||||||
|
|
||||||
try:
|
try:
|
||||||
return await __store()
|
return await __store()
|
||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
log.debug("Timeout while storing blob_hash %s at %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
log.debug("Timeout while storing blob_hash %s at %s", hash_value.hex()[:8], peer)
|
||||||
return peer.node_id, False
|
return peer.node_id, False
|
||||||
except ValueError as err:
|
except ValueError as err:
|
||||||
log.error("Unexpected response: %s", err)
|
log.error("Unexpected response: %s", err)
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ class KBucket:
|
|||||||
self.peers.append(peer)
|
self.peers.append(peer)
|
||||||
return True
|
return True
|
||||||
else:
|
else:
|
||||||
for i in range(len(self.peers)):
|
for i, _ in enumerate(self.peers):
|
||||||
local_peer = self.peers[i]
|
local_peer = self.peers[i]
|
||||||
if local_peer.node_id == peer.node_id:
|
if local_peer.node_id == peer.node_id:
|
||||||
self.peers.remove(local_peer)
|
self.peers.remove(local_peer)
|
||||||
|
|||||||
@@ -144,7 +144,7 @@ class ErrorDatagram(KademliaDatagramBase):
|
|||||||
self.response = response.decode()
|
self.response = response.decode()
|
||||||
|
|
||||||
|
|
||||||
def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]:
|
def _decode_datagram(datagram: bytes):
|
||||||
msg_types = {
|
msg_types = {
|
||||||
REQUEST_TYPE: RequestDatagram,
|
REQUEST_TYPE: RequestDatagram,
|
||||||
RESPONSE_TYPE: ResponseDatagram,
|
RESPONSE_TYPE: ResponseDatagram,
|
||||||
@@ -152,26 +152,36 @@ def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDa
|
|||||||
}
|
}
|
||||||
|
|
||||||
primitive: typing.Dict = bdecode(datagram)
|
primitive: typing.Dict = bdecode(datagram)
|
||||||
if primitive[0] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
|
|
||||||
datagram_type = primitive[0] # pylint: disable=unsubscriptable-object
|
converted = {
|
||||||
|
str(k).encode() if not isinstance(k, bytes) else k: v for k, v in primitive.items()
|
||||||
|
}
|
||||||
|
|
||||||
|
if converted[b'0'] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
|
||||||
|
datagram_type = converted[b'0'] # pylint: disable=unsubscriptable-object
|
||||||
else:
|
else:
|
||||||
raise ValueError("invalid datagram type")
|
raise ValueError("invalid datagram type")
|
||||||
datagram_class = msg_types[datagram_type]
|
datagram_class = msg_types[datagram_type]
|
||||||
decoded = {
|
decoded = {
|
||||||
k: primitive[i] # pylint: disable=unsubscriptable-object
|
k: converted[str(i).encode()] # pylint: disable=unsubscriptable-object
|
||||||
for i, k in enumerate(datagram_class.required_fields)
|
for i, k in enumerate(datagram_class.required_fields)
|
||||||
if i in primitive # pylint: disable=unsupported-membership-test
|
if str(i).encode() in converted # pylint: disable=unsupported-membership-test
|
||||||
}
|
}
|
||||||
for i, _ in enumerate(OPTIONAL_FIELDS):
|
for i, _ in enumerate(OPTIONAL_FIELDS):
|
||||||
if i + OPTIONAL_ARG_OFFSET in primitive:
|
if str(i + OPTIONAL_ARG_OFFSET).encode() in converted:
|
||||||
decoded[i + OPTIONAL_ARG_OFFSET] = primitive[i + OPTIONAL_ARG_OFFSET]
|
decoded[i + OPTIONAL_ARG_OFFSET] = converted[str(i + OPTIONAL_ARG_OFFSET).encode()]
|
||||||
|
return decoded, datagram_class
|
||||||
|
|
||||||
|
|
||||||
|
def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]:
|
||||||
|
decoded, datagram_class = _decode_datagram(datagram)
|
||||||
return datagram_class(**decoded)
|
return datagram_class(**decoded)
|
||||||
|
|
||||||
|
|
||||||
def make_compact_ip(address: str) -> bytearray:
|
def make_compact_ip(address: str) -> bytearray:
|
||||||
compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), address.split('.'), bytearray())
|
compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), address.split('.'), bytearray())
|
||||||
if len(compact_ip) != 4:
|
if len(compact_ip) != 4:
|
||||||
raise ValueError(f"invalid IPv4 length")
|
raise ValueError("invalid IPv4 length")
|
||||||
return compact_ip
|
return compact_ip
|
||||||
|
|
||||||
|
|
||||||
@@ -180,7 +190,7 @@ def make_compact_address(node_id: bytes, address: str, port: int) -> bytearray:
|
|||||||
if not 0 < port < 65536:
|
if not 0 < port < 65536:
|
||||||
raise ValueError(f'Invalid port: {port}')
|
raise ValueError(f'Invalid port: {port}')
|
||||||
if len(node_id) != constants.HASH_BITS // 8:
|
if len(node_id) != constants.HASH_BITS // 8:
|
||||||
raise ValueError(f"invalid node node_id length")
|
raise ValueError("invalid node node_id length")
|
||||||
return compact_ip + port.to_bytes(2, 'big') + node_id
|
return compact_ip + port.to_bytes(2, 'big') + node_id
|
||||||
|
|
||||||
|
|
||||||
@@ -191,5 +201,5 @@ def decode_compact_address(compact_address: bytes) -> typing.Tuple[bytes, str, i
|
|||||||
if not 0 < port < 65536:
|
if not 0 < port < 65536:
|
||||||
raise ValueError(f'Invalid port: {port}')
|
raise ValueError(f'Invalid port: {port}')
|
||||||
if len(node_id) != constants.HASH_BITS // 8:
|
if len(node_id) != constants.HASH_BITS // 8:
|
||||||
raise ValueError(f"invalid node node_id length")
|
raise ValueError("invalid node node_id length")
|
||||||
return node_id, address, port
|
return node_id, address, port
|
||||||
@@ -34,6 +34,7 @@ Code | Name | Message
|
|||||||
**11x** | InputValue(ValueError) | Invalid argument value provided to command.
|
**11x** | InputValue(ValueError) | Invalid argument value provided to command.
|
||||||
111 | GenericInputValue | The value '{value}' for argument '{argument}' is not valid.
|
111 | GenericInputValue | The value '{value}' for argument '{argument}' is not valid.
|
||||||
112 | InputValueIsNone | None or null is not valid value for argument '{argument}'.
|
112 | InputValueIsNone | None or null is not valid value for argument '{argument}'.
|
||||||
|
113 | ConflictingInputValue | Only '{first_argument}' or '{second_argument}' is allowed, not both.
|
||||||
**2xx** | Configuration | Configuration errors.
|
**2xx** | Configuration | Configuration errors.
|
||||||
201 | ConfigWrite | Cannot write configuration file '{path}'. -- When writing the default config fails on startup, such as due to permission issues.
|
201 | ConfigWrite | Cannot write configuration file '{path}'. -- When writing the default config fails on startup, such as due to permission issues.
|
||||||
202 | ConfigRead | Cannot find provided configuration file '{path}'. -- Can't open the config file user provided via command line args.
|
202 | ConfigRead | Cannot find provided configuration file '{path}'. -- Can't open the config file user provided via command line args.
|
||||||
@@ -53,13 +54,18 @@ Code | Name | Message
|
|||||||
407 | DataDownload | Failed to download blob. *generic*
|
407 | DataDownload | Failed to download blob. *generic*
|
||||||
410 | Resolve | Failed to resolve '{url}'.
|
410 | Resolve | Failed to resolve '{url}'.
|
||||||
411 | ResolveTimeout | Failed to resolve '{url}' within the timeout.
|
411 | ResolveTimeout | Failed to resolve '{url}' within the timeout.
|
||||||
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{claim_id(censor_hash)}'.
|
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{censor_id}'.
|
||||||
420 | KeyFeeAboveMaxAllowed | {message}
|
420 | KeyFeeAboveMaxAllowed | {message}
|
||||||
421 | InvalidPassword | Password is invalid.
|
421 | InvalidPassword | Password is invalid.
|
||||||
422 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version.
|
422 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version.
|
||||||
|
423 | TooManyClaimSearchParameters | {key} cant have more than {limit} items.
|
||||||
431 | ServerPaymentInvalidAddress | Invalid address from wallet server: '{address}' - skipping payment round.
|
431 | ServerPaymentInvalidAddress | Invalid address from wallet server: '{address}' - skipping payment round.
|
||||||
432 | ServerPaymentWalletLocked | Cannot spend funds with locked wallet, skipping payment round.
|
432 | ServerPaymentWalletLocked | Cannot spend funds with locked wallet, skipping payment round.
|
||||||
433 | ServerPaymentFeeAboveMaxAllowed | Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.
|
433 | ServerPaymentFeeAboveMaxAllowed | Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.
|
||||||
|
434 | WalletNotLoaded | Wallet {wallet_id} is not loaded.
|
||||||
|
435 | WalletAlreadyLoaded | Wallet {wallet_path} is already loaded.
|
||||||
|
436 | WalletNotFound | Wallet not found at {wallet_path}.
|
||||||
|
437 | WalletAlreadyExists | Wallet {wallet_path} already exists, use `wallet_add` to load it.
|
||||||
**5xx** | Blob | **Blobs**
|
**5xx** | Blob | **Blobs**
|
||||||
500 | BlobNotFound | Blob not found.
|
500 | BlobNotFound | Blob not found.
|
||||||
501 | BlobPermissionDenied | Permission denied to read blob.
|
501 | BlobPermissionDenied | Permission denied to read blob.
|
||||||
|
|||||||
+47
-3
@@ -76,6 +76,14 @@ class InputValueIsNoneError(InputValueError):
|
|||||||
super().__init__(f"None or null is not valid value for argument '{argument}'.")
|
super().__init__(f"None or null is not valid value for argument '{argument}'.")
|
||||||
|
|
||||||
|
|
||||||
|
class ConflictingInputValueError(InputValueError):
|
||||||
|
|
||||||
|
def __init__(self, first_argument, second_argument):
|
||||||
|
self.first_argument = first_argument
|
||||||
|
self.second_argument = second_argument
|
||||||
|
super().__init__(f"Only '{first_argument}' or '{second_argument}' is allowed, not both.")
|
||||||
|
|
||||||
|
|
||||||
class ConfigurationError(BaseError):
|
class ConfigurationError(BaseError):
|
||||||
"""
|
"""
|
||||||
Configuration errors.
|
Configuration errors.
|
||||||
@@ -215,10 +223,10 @@ class ResolveTimeoutError(WalletError):
|
|||||||
|
|
||||||
class ResolveCensoredError(WalletError):
|
class ResolveCensoredError(WalletError):
|
||||||
|
|
||||||
def __init__(self, url, censor_hash):
|
def __init__(self, url, censor_id):
|
||||||
self.url = url
|
self.url = url
|
||||||
self.censor_hash = censor_hash
|
self.censor_id = censor_id
|
||||||
super().__init__(f"Resolve of '{url}' was censored by channel with claim id '{claim_id(censor_hash)}'.")
|
super().__init__(f"Resolve of '{url}' was censored by channel with claim id '{censor_id}'.")
|
||||||
|
|
||||||
|
|
||||||
class KeyFeeAboveMaxAllowedError(WalletError):
|
class KeyFeeAboveMaxAllowedError(WalletError):
|
||||||
@@ -242,6 +250,14 @@ class IncompatibleWalletServerError(WalletError):
|
|||||||
super().__init__(f"'{server}:{port}' has an incompatibly old version.")
|
super().__init__(f"'{server}:{port}' has an incompatibly old version.")
|
||||||
|
|
||||||
|
|
||||||
|
class TooManyClaimSearchParametersError(WalletError):
|
||||||
|
|
||||||
|
def __init__(self, key, limit):
|
||||||
|
self.key = key
|
||||||
|
self.limit = limit
|
||||||
|
super().__init__(f"{key} cant have more than {limit} items.")
|
||||||
|
|
||||||
|
|
||||||
class ServerPaymentInvalidAddressError(WalletError):
|
class ServerPaymentInvalidAddressError(WalletError):
|
||||||
|
|
||||||
def __init__(self, address):
|
def __init__(self, address):
|
||||||
@@ -263,6 +279,34 @@ class ServerPaymentFeeAboveMaxAllowedError(WalletError):
|
|||||||
super().__init__(f"Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.")
|
super().__init__(f"Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.")
|
||||||
|
|
||||||
|
|
||||||
|
class WalletNotLoadedError(WalletError):
|
||||||
|
|
||||||
|
def __init__(self, wallet_id):
|
||||||
|
self.wallet_id = wallet_id
|
||||||
|
super().__init__(f"Wallet {wallet_id} is not loaded.")
|
||||||
|
|
||||||
|
|
||||||
|
class WalletAlreadyLoadedError(WalletError):
|
||||||
|
|
||||||
|
def __init__(self, wallet_path):
|
||||||
|
self.wallet_path = wallet_path
|
||||||
|
super().__init__(f"Wallet {wallet_path} is already loaded.")
|
||||||
|
|
||||||
|
|
||||||
|
class WalletNotFoundError(WalletError):
|
||||||
|
|
||||||
|
def __init__(self, wallet_path):
|
||||||
|
self.wallet_path = wallet_path
|
||||||
|
super().__init__(f"Wallet not found at {wallet_path}.")
|
||||||
|
|
||||||
|
|
||||||
|
class WalletAlreadyExistsError(WalletError):
|
||||||
|
|
||||||
|
def __init__(self, wallet_path):
|
||||||
|
self.wallet_path = wallet_path
|
||||||
|
super().__init__(f"Wallet {wallet_path} already exists, use `wallet_add` to load it.")
|
||||||
|
|
||||||
|
|
||||||
class BlobError(BaseError):
|
class BlobError(BaseError):
|
||||||
"""
|
"""
|
||||||
**Blobs**
|
**Blobs**
|
||||||
|
|||||||
@@ -63,7 +63,7 @@ class ErrorClass:
|
|||||||
@staticmethod
|
@staticmethod
|
||||||
def get_fields(args):
|
def get_fields(args):
|
||||||
if len(args) > 1:
|
if len(args) > 1:
|
||||||
return f''.join(f'\n{INDENT*2}self.{field} = {field}' for field in args[1:])
|
return ''.join(f'\n{INDENT*2}self.{field} = {field}' for field in args[1:])
|
||||||
return ''
|
return ''
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|||||||
+1
-10
@@ -3,7 +3,6 @@ import sys
|
|||||||
import shutil
|
import shutil
|
||||||
import signal
|
import signal
|
||||||
import pathlib
|
import pathlib
|
||||||
import platform
|
|
||||||
import json
|
import json
|
||||||
import asyncio
|
import asyncio
|
||||||
import argparse
|
import argparse
|
||||||
@@ -15,7 +14,6 @@ from aiohttp.web import GracefulExit
|
|||||||
from docopt import docopt
|
from docopt import docopt
|
||||||
|
|
||||||
from lbry import __version__ as lbrynet_version
|
from lbry import __version__ as lbrynet_version
|
||||||
from lbry.extras.daemon.loggly_handler import get_loggly_handler
|
|
||||||
from lbry.extras.daemon.daemon import Daemon
|
from lbry.extras.daemon.daemon import Daemon
|
||||||
from lbry.conf import Config, CLIConfig
|
from lbry.conf import Config, CLIConfig
|
||||||
|
|
||||||
@@ -103,7 +101,7 @@ class ArgumentParser(argparse.ArgumentParser):
|
|||||||
self._optionals.title = 'Options'
|
self._optionals.title = 'Options'
|
||||||
if group_name is None:
|
if group_name is None:
|
||||||
self.epilog = (
|
self.epilog = (
|
||||||
f"Run 'lbrynet COMMAND --help' for more information on a command or group."
|
"Run 'lbrynet COMMAND --help' for more information on a command or group."
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
self.epilog = (
|
self.epilog = (
|
||||||
@@ -256,15 +254,8 @@ def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config
|
|||||||
else:
|
else:
|
||||||
logger.getChild('lbry').setLevel(logging.DEBUG)
|
logger.getChild('lbry').setLevel(logging.DEBUG)
|
||||||
|
|
||||||
loggly_handler = get_loggly_handler(conf)
|
|
||||||
loggly_handler.setLevel(logging.ERROR)
|
|
||||||
logger.getChild('lbry').addHandler(loggly_handler)
|
|
||||||
|
|
||||||
|
|
||||||
def run_daemon(args: argparse.Namespace, conf: Config):
|
def run_daemon(args: argparse.Namespace, conf: Config):
|
||||||
if sys.version_info < (3, 8) and platform.system() == "Windows":
|
|
||||||
# TODO: remove after we move to requiring Python 3.8
|
|
||||||
asyncio.set_event_loop(asyncio.ProactorEventLoop())
|
|
||||||
loop = asyncio.get_event_loop()
|
loop = asyncio.get_event_loop()
|
||||||
if args.verbose is not None:
|
if args.verbose is not None:
|
||||||
loop.set_debug(True)
|
loop.set_debug(True)
|
||||||
|
|||||||
@@ -66,7 +66,7 @@ def _download_properties(conf: Config, external_ip: str, resolve_duration: float
|
|||||||
"node_rpc_timeout": conf.node_rpc_timeout,
|
"node_rpc_timeout": conf.node_rpc_timeout,
|
||||||
"peer_connect_timeout": conf.peer_connect_timeout,
|
"peer_connect_timeout": conf.peer_connect_timeout,
|
||||||
"blob_download_timeout": conf.blob_download_timeout,
|
"blob_download_timeout": conf.blob_download_timeout,
|
||||||
"use_fixed_peers": len(conf.reflector_servers) > 0,
|
"use_fixed_peers": len(conf.fixed_peers) > 0,
|
||||||
"fixed_peer_delay": fixed_peer_delay,
|
"fixed_peer_delay": fixed_peer_delay,
|
||||||
"added_fixed_peers": added_fixed_peers,
|
"added_fixed_peers": added_fixed_peers,
|
||||||
"active_peer_count": active_peer_count,
|
"active_peer_count": active_peer_count,
|
||||||
@@ -132,7 +132,7 @@ class AnalyticsManager:
|
|||||||
async def run(self):
|
async def run(self):
|
||||||
while True:
|
while True:
|
||||||
if self.enabled:
|
if self.enabled:
|
||||||
self.external_ip = await utils.get_external_ip()
|
self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||||
await self._send_heartbeat()
|
await self._send_heartbeat()
|
||||||
await asyncio.sleep(1800)
|
await asyncio.sleep(1800)
|
||||||
|
|
||||||
|
|||||||
@@ -1,66 +0,0 @@
|
|||||||
import logging
|
|
||||||
import time
|
|
||||||
import hashlib
|
|
||||||
import binascii
|
|
||||||
|
|
||||||
import ecdsa
|
|
||||||
from lbry import utils
|
|
||||||
from lbry.crypto.hash import sha256
|
|
||||||
from lbry.wallet.transaction import Output
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
def get_encoded_signature(signature):
|
|
||||||
signature = signature.encode() if isinstance(signature, str) else signature
|
|
||||||
r = int(signature[:int(len(signature) / 2)], 16)
|
|
||||||
s = int(signature[int(len(signature) / 2):], 16)
|
|
||||||
return ecdsa.util.sigencode_der(r, s, len(signature) * 4)
|
|
||||||
|
|
||||||
|
|
||||||
def cid2hash(claim_id: str) -> bytes:
|
|
||||||
return binascii.unhexlify(claim_id.encode())[::-1]
|
|
||||||
|
|
||||||
|
|
||||||
def is_comment_signed_by_channel(comment: dict, channel: Output, abandon=False):
|
|
||||||
if isinstance(channel, Output):
|
|
||||||
try:
|
|
||||||
signing_field = comment['comment_id'] if abandon else comment['comment']
|
|
||||||
pieces = [
|
|
||||||
comment['signing_ts'].encode(),
|
|
||||||
cid2hash(comment['channel_id']),
|
|
||||||
signing_field.encode()
|
|
||||||
]
|
|
||||||
return Output.is_signature_valid(
|
|
||||||
get_encoded_signature(comment['signature']),
|
|
||||||
sha256(b''.join(pieces)),
|
|
||||||
channel.claim.channel.public_key_bytes
|
|
||||||
)
|
|
||||||
except KeyError:
|
|
||||||
pass
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def sign_comment(comment: dict, channel: Output, abandon=False):
|
|
||||||
timestamp = str(int(time.time()))
|
|
||||||
signing_field = comment['comment_id'] if abandon else comment['comment']
|
|
||||||
pieces = [timestamp.encode(), channel.claim_hash, signing_field.encode()]
|
|
||||||
digest = sha256(b''.join(pieces))
|
|
||||||
signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
|
|
||||||
comment.update({
|
|
||||||
'signature': binascii.hexlify(signature).decode(),
|
|
||||||
'signing_ts': timestamp
|
|
||||||
})
|
|
||||||
|
|
||||||
|
|
||||||
async def jsonrpc_post(url: str, method: str, params: dict = None, **kwargs) -> any:
|
|
||||||
params = params or {}
|
|
||||||
params.update(kwargs)
|
|
||||||
json_body = {'jsonrpc': '2.0', 'id': None, 'method': method, 'params': params}
|
|
||||||
async with utils.aiohttp_request('POST', url, json=json_body) as response:
|
|
||||||
try:
|
|
||||||
result = await response.json()
|
|
||||||
return result['result'] if 'result' in result else result
|
|
||||||
except Exception as cte:
|
|
||||||
log.exception('Unable to decode response from server: %s', cte)
|
|
||||||
return await response.text()
|
|
||||||
@@ -158,11 +158,14 @@ class ComponentManager:
|
|||||||
for component in self.components
|
for component in self.components
|
||||||
}
|
}
|
||||||
|
|
||||||
def get_component(self, component_name):
|
def get_actual_component(self, component_name):
|
||||||
for component in self.components:
|
for component in self.components:
|
||||||
if component.component_name == component_name:
|
if component.component_name == component_name:
|
||||||
return component.component
|
return component
|
||||||
raise NameError(component_name)
|
raise NameError(component_name)
|
||||||
|
|
||||||
|
def get_component(self, component_name):
|
||||||
|
return self.get_actual_component(component_name).component
|
||||||
|
|
||||||
def has_component(self, component_name):
|
def has_component(self, component_name):
|
||||||
return any(component for component in self.components if component_name == component.component_name)
|
return any(component for component in self.components if component_name == component.component_name)
|
||||||
@@ -15,13 +15,20 @@ from lbry.dht.node import Node
|
|||||||
from lbry.dht.peer import is_valid_public_ipv4
|
from lbry.dht.peer import is_valid_public_ipv4
|
||||||
from lbry.dht.blob_announcer import BlobAnnouncer
|
from lbry.dht.blob_announcer import BlobAnnouncer
|
||||||
from lbry.blob.blob_manager import BlobManager
|
from lbry.blob.blob_manager import BlobManager
|
||||||
|
from lbry.blob.disk_space_manager import DiskSpaceManager
|
||||||
from lbry.blob_exchange.server import BlobServer
|
from lbry.blob_exchange.server import BlobServer
|
||||||
from lbry.stream.stream_manager import StreamManager
|
from lbry.stream.stream_manager import StreamManager
|
||||||
|
from lbry.file.file_manager import FileManager
|
||||||
from lbry.extras.daemon.component import Component
|
from lbry.extras.daemon.component import Component
|
||||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||||
from lbry.extras.daemon.storage import SQLiteStorage
|
from lbry.extras.daemon.storage import SQLiteStorage
|
||||||
|
from lbry.torrent.torrent_manager import TorrentManager
|
||||||
from lbry.wallet import WalletManager
|
from lbry.wallet import WalletManager
|
||||||
from lbry.wallet.usage_payment import WalletServerPayer
|
from lbry.wallet.usage_payment import WalletServerPayer
|
||||||
|
try:
|
||||||
|
from lbry.torrent.session import TorrentSession
|
||||||
|
except ImportError:
|
||||||
|
TorrentSession = None
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -33,10 +40,12 @@ WALLET_COMPONENT = "wallet"
|
|||||||
WALLET_SERVER_PAYMENTS_COMPONENT = "wallet_server_payments"
|
WALLET_SERVER_PAYMENTS_COMPONENT = "wallet_server_payments"
|
||||||
DHT_COMPONENT = "dht"
|
DHT_COMPONENT = "dht"
|
||||||
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
|
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
|
||||||
STREAM_MANAGER_COMPONENT = "stream_manager"
|
FILE_MANAGER_COMPONENT = "file_manager"
|
||||||
|
DISK_SPACE_COMPONENT = "disk_space"
|
||||||
PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
|
PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
|
||||||
UPNP_COMPONENT = "upnp"
|
UPNP_COMPONENT = "upnp"
|
||||||
EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager"
|
EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager"
|
||||||
|
LIBTORRENT_COMPONENT = "libtorrent_component"
|
||||||
|
|
||||||
|
|
||||||
class DatabaseComponent(Component):
|
class DatabaseComponent(Component):
|
||||||
@@ -52,7 +61,7 @@ class DatabaseComponent(Component):
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_current_db_revision():
|
def get_current_db_revision():
|
||||||
return 14
|
return 15
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def revision_filename(self):
|
def revision_filename(self):
|
||||||
@@ -112,15 +121,17 @@ class WalletComponent(Component):
|
|||||||
async def get_status(self):
|
async def get_status(self):
|
||||||
if self.wallet_manager is None:
|
if self.wallet_manager is None:
|
||||||
return
|
return
|
||||||
session_pool = self.wallet_manager.ledger.network.session_pool
|
is_connected = self.wallet_manager.ledger.network.is_connected
|
||||||
sessions = session_pool.sessions
|
sessions = []
|
||||||
connected = None
|
connected = None
|
||||||
if self.wallet_manager.ledger.network.client:
|
if is_connected:
|
||||||
addr_and_port = self.wallet_manager.ledger.network.client.server_address_and_port
|
addr, port = self.wallet_manager.ledger.network.client.server
|
||||||
if addr_and_port:
|
connected = f"{addr}:{port}"
|
||||||
connected = f"{addr_and_port[0]}:{addr_and_port[1]}"
|
sessions.append(self.wallet_manager.ledger.network.client)
|
||||||
|
|
||||||
result = {
|
result = {
|
||||||
'connected': connected,
|
'connected': connected,
|
||||||
|
'connected_features': self.wallet_manager.ledger.network.server_features,
|
||||||
'servers': [
|
'servers': [
|
||||||
{
|
{
|
||||||
'host': session.server[0],
|
'host': session.server[0],
|
||||||
@@ -129,8 +140,8 @@ class WalletComponent(Component):
|
|||||||
'availability': session.available,
|
'availability': session.available,
|
||||||
} for session in sessions
|
} for session in sessions
|
||||||
],
|
],
|
||||||
'known_servers': len(sessions),
|
'known_servers': len(self.wallet_manager.ledger.network.known_hubs),
|
||||||
'available_servers': len(list(session_pool.available_sessions))
|
'available_servers': 1 if is_connected else 0
|
||||||
}
|
}
|
||||||
|
|
||||||
if self.wallet_manager.ledger.network.remote_height:
|
if self.wallet_manager.ledger.network.remote_height:
|
||||||
@@ -142,7 +153,7 @@ class WalletComponent(Component):
|
|||||||
progress = min(max(math.ceil(float(download_height) / float(target_height) * 100), 0), 100)
|
progress = min(max(math.ceil(float(download_height) / float(target_height) * 100), 0), 100)
|
||||||
else:
|
else:
|
||||||
progress = 100
|
progress = 100
|
||||||
best_hash = self.wallet_manager.get_best_blockhash()
|
best_hash = await self.wallet_manager.get_best_blockhash()
|
||||||
result.update({
|
result.update({
|
||||||
'headers_synchronization_progress': progress,
|
'headers_synchronization_progress': progress,
|
||||||
'blocks': max(local_height, 0),
|
'blocks': max(local_height, 0),
|
||||||
@@ -266,7 +277,7 @@ class DHTComponent(Component):
|
|||||||
external_ip = upnp_component.external_ip
|
external_ip = upnp_component.external_ip
|
||||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||||
if not external_ip:
|
if not external_ip:
|
||||||
external_ip = await utils.get_external_ip()
|
external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||||
if not external_ip:
|
if not external_ip:
|
||||||
log.warning("failed to get external ip")
|
log.warning("failed to get external ip")
|
||||||
|
|
||||||
@@ -318,23 +329,23 @@ class HashAnnouncerComponent(Component):
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
class StreamManagerComponent(Component):
|
class FileManagerComponent(Component):
|
||||||
component_name = STREAM_MANAGER_COMPONENT
|
component_name = FILE_MANAGER_COMPONENT
|
||||||
depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT]
|
depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT]
|
||||||
|
|
||||||
def __init__(self, component_manager):
|
def __init__(self, component_manager):
|
||||||
super().__init__(component_manager)
|
super().__init__(component_manager)
|
||||||
self.stream_manager: typing.Optional[StreamManager] = None
|
self.file_manager: typing.Optional[FileManager] = None
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def component(self) -> typing.Optional[StreamManager]:
|
def component(self) -> typing.Optional[FileManager]:
|
||||||
return self.stream_manager
|
return self.file_manager
|
||||||
|
|
||||||
async def get_status(self):
|
async def get_status(self):
|
||||||
if not self.stream_manager:
|
if not self.file_manager:
|
||||||
return
|
return
|
||||||
return {
|
return {
|
||||||
'managed_files': len(self.stream_manager.streams),
|
'managed_files': len(self.file_manager.get_filtered()),
|
||||||
}
|
}
|
||||||
|
|
||||||
async def start(self):
|
async def start(self):
|
||||||
@@ -343,16 +354,85 @@ class StreamManagerComponent(Component):
|
|||||||
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
||||||
node = self.component_manager.get_component(DHT_COMPONENT) \
|
node = self.component_manager.get_component(DHT_COMPONENT) \
|
||||||
if self.component_manager.has_component(DHT_COMPONENT) else None
|
if self.component_manager.has_component(DHT_COMPONENT) else None
|
||||||
|
try:
|
||||||
|
torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT) if TorrentSession else None
|
||||||
|
except NameError:
|
||||||
|
torrent = None
|
||||||
log.info('Starting the file manager')
|
log.info('Starting the file manager')
|
||||||
loop = asyncio.get_event_loop()
|
loop = asyncio.get_event_loop()
|
||||||
self.stream_manager = StreamManager(
|
self.file_manager = FileManager(
|
||||||
loop, self.conf, blob_manager, wallet, storage, node, self.component_manager.analytics_manager
|
loop, self.conf, wallet, storage, self.component_manager.analytics_manager
|
||||||
)
|
)
|
||||||
await self.stream_manager.start()
|
self.file_manager.source_managers['stream'] = StreamManager(
|
||||||
|
loop, self.conf, blob_manager, wallet, storage, node,
|
||||||
|
)
|
||||||
|
if TorrentSession and LIBTORRENT_COMPONENT not in self.conf.components_to_skip:
|
||||||
|
self.file_manager.source_managers['torrent'] = TorrentManager(
|
||||||
|
loop, self.conf, torrent, storage, self.component_manager.analytics_manager
|
||||||
|
)
|
||||||
|
await self.file_manager.start()
|
||||||
log.info('Done setting up file manager')
|
log.info('Done setting up file manager')
|
||||||
|
|
||||||
async def stop(self):
|
async def stop(self):
|
||||||
self.stream_manager.stop()
|
self.file_manager.stop()
|
||||||
|
|
||||||
|
|
||||||
|
class DiskSpaceComponent(Component):
|
||||||
|
component_name = DISK_SPACE_COMPONENT
|
||||||
|
depends_on = [DATABASE_COMPONENT, BLOB_COMPONENT]
|
||||||
|
|
||||||
|
def __init__(self, component_manager):
|
||||||
|
super().__init__(component_manager)
|
||||||
|
self.disk_space_manager: typing.Optional[DiskSpaceManager] = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def component(self) -> typing.Optional[DiskSpaceManager]:
|
||||||
|
return self.disk_space_manager
|
||||||
|
|
||||||
|
async def get_status(self):
|
||||||
|
if self.disk_space_manager:
|
||||||
|
return {
|
||||||
|
'space_used': str(await self.disk_space_manager.get_space_used_mb()),
|
||||||
|
'running': self.disk_space_manager.running,
|
||||||
|
}
|
||||||
|
return {'space_used': '0', 'running': False}
|
||||||
|
|
||||||
|
async def start(self):
|
||||||
|
db = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||||
|
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||||
|
self.disk_space_manager = DiskSpaceManager(self.conf, db, blob_manager)
|
||||||
|
await self.disk_space_manager.start()
|
||||||
|
|
||||||
|
async def stop(self):
|
||||||
|
await self.disk_space_manager.stop()
|
||||||
|
|
||||||
|
|
||||||
|
class TorrentComponent(Component):
|
||||||
|
component_name = LIBTORRENT_COMPONENT
|
||||||
|
|
||||||
|
def __init__(self, component_manager):
|
||||||
|
super().__init__(component_manager)
|
||||||
|
self.torrent_session = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def component(self) -> typing.Optional[TorrentSession]:
|
||||||
|
return self.torrent_session
|
||||||
|
|
||||||
|
async def get_status(self):
|
||||||
|
if not self.torrent_session:
|
||||||
|
return
|
||||||
|
return {
|
||||||
|
'running': True, # TODO: what to return here?
|
||||||
|
}
|
||||||
|
|
||||||
|
async def start(self):
|
||||||
|
if TorrentSession:
|
||||||
|
self.torrent_session = TorrentSession(asyncio.get_event_loop(), None)
|
||||||
|
await self.torrent_session.bind() # TODO: specify host/port
|
||||||
|
|
||||||
|
async def stop(self):
|
||||||
|
if self.torrent_session:
|
||||||
|
await self.torrent_session.pause()
|
||||||
|
|
||||||
|
|
||||||
class PeerProtocolServerComponent(Component):
|
class PeerProtocolServerComponent(Component):
|
||||||
@@ -428,11 +508,15 @@ class UPnPComponent(Component):
|
|||||||
pass
|
pass
|
||||||
if external_ip and not is_valid_public_ipv4(external_ip):
|
if external_ip and not is_valid_public_ipv4(external_ip):
|
||||||
log.warning("UPnP returned a private/reserved ip - %s, checking lbry.com fallback", external_ip)
|
log.warning("UPnP returned a private/reserved ip - %s, checking lbry.com fallback", external_ip)
|
||||||
external_ip = await utils.get_external_ip()
|
external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||||
if self.external_ip and self.external_ip != external_ip:
|
if self.external_ip and self.external_ip != external_ip:
|
||||||
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
|
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
|
||||||
if external_ip:
|
if external_ip:
|
||||||
self.external_ip = external_ip
|
self.external_ip = external_ip
|
||||||
|
dht_component = self.component_manager.get_component(DHT_COMPONENT)
|
||||||
|
if dht_component:
|
||||||
|
dht_node = dht_component.component
|
||||||
|
dht_node.protocol.external_ip = external_ip
|
||||||
# assert self.external_ip is not None # TODO: handle going/starting offline
|
# assert self.external_ip is not None # TODO: handle going/starting offline
|
||||||
|
|
||||||
if not self.upnp_redirects and self.upnp: # setup missing redirects
|
if not self.upnp_redirects and self.upnp: # setup missing redirects
|
||||||
@@ -486,13 +570,15 @@ class UPnPComponent(Component):
|
|||||||
async def start(self):
|
async def start(self):
|
||||||
log.info("detecting external ip")
|
log.info("detecting external ip")
|
||||||
if not self.use_upnp:
|
if not self.use_upnp:
|
||||||
self.external_ip = await utils.get_external_ip()
|
self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||||
return
|
return
|
||||||
success = False
|
success = False
|
||||||
await self._maintain_redirects()
|
await self._maintain_redirects()
|
||||||
if self.upnp:
|
if self.upnp:
|
||||||
if not self.upnp_redirects and not all([x in self.component_manager.skip_components for x in
|
if not self.upnp_redirects and not all(
|
||||||
(DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT)]):
|
x in self.component_manager.skip_components
|
||||||
|
for x in (DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT)
|
||||||
|
):
|
||||||
log.error("failed to setup upnp")
|
log.error("failed to setup upnp")
|
||||||
else:
|
else:
|
||||||
success = True
|
success = True
|
||||||
@@ -501,9 +587,9 @@ class UPnPComponent(Component):
|
|||||||
else:
|
else:
|
||||||
log.error("failed to setup upnp")
|
log.error("failed to setup upnp")
|
||||||
if not self.external_ip:
|
if not self.external_ip:
|
||||||
self.external_ip = await utils.get_external_ip()
|
self.external_ip, probed_url = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||||
if self.external_ip:
|
if self.external_ip:
|
||||||
log.info("detected external ip using lbry.com fallback")
|
log.info("detected external ip using %s fallback", probed_url)
|
||||||
if self.component_manager.analytics_manager:
|
if self.component_manager.analytics_manager:
|
||||||
self.component_manager.loop.create_task(
|
self.component_manager.loop.create_task(
|
||||||
self.component_manager.analytics_manager.send_upnp_setup_success_fail(
|
self.component_manager.analytics_manager.send_upnp_setup_success_fail(
|
||||||
|
|||||||
+1020
-530
File diff suppressed because it is too large.
Load diff
@@ -2,9 +2,10 @@ import json
|
|||||||
import time
|
import time
|
||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
|
from statistics import median
|
||||||
from decimal import Decimal
|
from decimal import Decimal
|
||||||
from typing import Optional, Iterable, Type
|
from typing import Optional, Iterable, Type
|
||||||
from aiohttp.client_exceptions import ContentTypeError
|
from aiohttp.client_exceptions import ContentTypeError, ClientConnectionError
|
||||||
from lbry.error import InvalidExchangeRateResponseError, CurrencyConversionError
|
from lbry.error import InvalidExchangeRateResponseError, CurrencyConversionError
|
||||||
from lbry.utils import aiohttp_request
|
from lbry.utils import aiohttp_request
|
||||||
from lbry.wallet.dewies import lbc_to_dewies
|
from lbry.wallet.dewies import lbc_to_dewies
|
||||||
@@ -58,9 +59,12 @@ class MarketFeed:
|
|||||||
raise NotImplementedError()
|
raise NotImplementedError()
|
||||||
|
|
||||||
async def get_response(self):
|
async def get_response(self):
|
||||||
async with aiohttp_request('get', self.url, params=self.params, timeout=self.request_timeout) as response:
|
async with aiohttp_request(
|
||||||
|
'get', self.url, params=self.params,
|
||||||
|
timeout=self.request_timeout, headers={"User-Agent": "lbrynet"}
|
||||||
|
) as response:
|
||||||
try:
|
try:
|
||||||
self._last_response = await response.json()
|
self._last_response = await response.json(content_type=None)
|
||||||
except ContentTypeError as e:
|
except ContentTypeError as e:
|
||||||
self._last_response = {}
|
self._last_response = {}
|
||||||
log.warning("Could not parse exchange rate response from %s: %s", self.name, e.message)
|
log.warning("Could not parse exchange rate response from %s: %s", self.name, e.message)
|
||||||
@@ -75,18 +79,23 @@ class MarketFeed:
|
|||||||
log.debug("Saving rate update %f for %s from %s", rate, self.market, self.name)
|
log.debug("Saving rate update %f for %s from %s", rate, self.market, self.name)
|
||||||
self.rate = ExchangeRate(self.market, rate, int(time.time()))
|
self.rate = ExchangeRate(self.market, rate, int(time.time()))
|
||||||
self.last_check = time.time()
|
self.last_check = time.time()
|
||||||
self.event.set()
|
|
||||||
return self.rate
|
return self.rate
|
||||||
except asyncio.CancelledError:
|
except asyncio.CancelledError:
|
||||||
raise
|
raise
|
||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
log.warning("Timed out fetching exchange rate from %s.", self.name)
|
log.warning("Timed out fetching exchange rate from %s.", self.name)
|
||||||
except json.JSONDecodeError as e:
|
except json.JSONDecodeError as e:
|
||||||
log.warning("Could not parse exchange rate response from %s: %s", self.name, e.doc)
|
msg = e.doc if '<html>' not in e.doc else 'unexpected content type.'
|
||||||
|
log.warning("Could not parse exchange rate response from %s: %s", self.name, msg)
|
||||||
|
log.debug(e.doc)
|
||||||
except InvalidExchangeRateResponseError as e:
|
except InvalidExchangeRateResponseError as e:
|
||||||
log.warning(str(e))
|
log.warning(str(e))
|
||||||
|
except ClientConnectionError as e:
|
||||||
|
log.warning("Error trying to connect to exchange rate %s: %s", self.name, str(e))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.exception("Exchange rate error (%s from %s):", self.market, self.name)
|
log.exception("Exchange rate error (%s from %s):", self.market, self.name)
|
||||||
|
finally:
|
||||||
|
self.event.set()
|
||||||
|
|
||||||
async def keep_updated(self):
|
async def keep_updated(self):
|
||||||
while True:
|
while True:
|
||||||
@@ -104,70 +113,92 @@ class MarketFeed:
|
|||||||
self.event.clear()
|
self.event.clear()
|
||||||
|
|
||||||
|
|
||||||
class BittrexFeed(MarketFeed):
|
class BaseBittrexFeed(MarketFeed):
|
||||||
name = "Bittrex"
|
name = "Bittrex"
|
||||||
market = "BTCLBC"
|
market = None
|
||||||
url = "https://bittrex.com/api/v1.1/public/getmarkethistory"
|
url = None
|
||||||
params = {'market': 'BTC-LBC', 'count': 50}
|
|
||||||
fee = 0.0025
|
fee = 0.0025
|
||||||
|
|
||||||
|
def get_rate_from_response(self, json_response):
|
||||||
|
if 'lastTradeRate' not in json_response:
|
||||||
|
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||||
|
return 1.0 / float(json_response['lastTradeRate'])
|
||||||
|
|
||||||
|
|
||||||
|
class BittrexBTCFeed(BaseBittrexFeed):
|
||||||
|
market = "BTCLBC"
|
||||||
|
url = "https://api.bittrex.com/v3/markets/LBC-BTC/ticker"
|
||||||
|
|
||||||
|
|
||||||
|
class BittrexUSDFeed(BaseBittrexFeed):
|
||||||
|
market = "USDLBC"
|
||||||
|
url = "https://api.bittrex.com/v3/markets/LBC-USD/ticker"
|
||||||
|
|
||||||
|
|
||||||
|
class BaseCoinExFeed(MarketFeed):
|
||||||
|
name = "CoinEx"
|
||||||
|
market = None
|
||||||
|
url = None
|
||||||
|
|
||||||
|
def get_rate_from_response(self, json_response):
|
||||||
|
if 'data' not in json_response or \
|
||||||
|
'ticker' not in json_response['data'] or \
|
||||||
|
'last' not in json_response['data']['ticker']:
|
||||||
|
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||||
|
return 1.0 / float(json_response['data']['ticker']['last'])
|
||||||
|
|
||||||
|
|
||||||
|
class CoinExBTCFeed(BaseCoinExFeed):
|
||||||
|
market = "BTCLBC"
|
||||||
|
url = "https://api.coinex.com/v1/market/ticker?market=LBCBTC"
|
||||||
|
|
||||||
|
|
||||||
|
class CoinExUSDFeed(BaseCoinExFeed):
|
||||||
|
market = "USDLBC"
|
||||||
|
url = "https://api.coinex.com/v1/market/ticker?market=LBCUSDT"
|
||||||
|
|
||||||
|
|
||||||
|
class BaseHotbitFeed(MarketFeed):
|
||||||
|
name = "hotbit"
|
||||||
|
market = None
|
||||||
|
url = "https://api.hotbit.io/api/v1/market.last"
|
||||||
|
|
||||||
def get_rate_from_response(self, json_response):
|
def get_rate_from_response(self, json_response):
|
||||||
if 'result' not in json_response:
|
if 'result' not in json_response:
|
||||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||||
trades = json_response['result']
|
return 1.0 / float(json_response['result'])
|
||||||
if len(trades) == 0:
|
|
||||||
raise InvalidExchangeRateResponseError(self.name, 'trades not found')
|
|
||||||
totals = sum([i['Total'] for i in trades])
|
|
||||||
qtys = sum([i['Quantity'] for i in trades])
|
|
||||||
if totals <= 0 or qtys <= 0:
|
|
||||||
raise InvalidExchangeRateResponseError(self.name, 'quantities were not positive')
|
|
||||||
vwap = totals / qtys
|
|
||||||
return float(1.0 / vwap)
|
|
||||||
|
|
||||||
|
|
||||||
class LBRYFeed(MarketFeed):
|
class HotbitBTCFeed(BaseHotbitFeed):
|
||||||
name = "lbry.com"
|
|
||||||
market = "BTCLBC"
|
market = "BTCLBC"
|
||||||
url = "https://api.lbry.com/lbc/exchange_rate"
|
params = {"market": "LBC/BTC"}
|
||||||
|
|
||||||
def get_rate_from_response(self, json_response):
|
|
||||||
if 'data' not in json_response:
|
|
||||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
|
||||||
return 1.0 / json_response['data']['lbc_btc']
|
|
||||||
|
|
||||||
|
|
||||||
class LBRYBTCFeed(LBRYFeed):
|
class HotbitUSDFeed(BaseHotbitFeed):
|
||||||
market = "USDBTC"
|
market = "USDLBC"
|
||||||
|
params = {"market": "LBC/USDT"}
|
||||||
def get_rate_from_response(self, json_response):
|
|
||||||
if 'data' not in json_response:
|
|
||||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
|
||||||
return 1.0 / json_response['data']['btc_usd']
|
|
||||||
|
|
||||||
|
|
||||||
class CryptonatorFeed(MarketFeed):
|
class UPbitBTCFeed(MarketFeed):
|
||||||
name = "cryptonator.com"
|
name = "UPbit"
|
||||||
market = "BTCLBC"
|
market = "BTCLBC"
|
||||||
url = "https://api.cryptonator.com/api/ticker/btc-lbc"
|
url = "https://api.upbit.com/v1/ticker"
|
||||||
|
params = {"markets": "BTC-LBC"}
|
||||||
|
|
||||||
def get_rate_from_response(self, json_response):
|
def get_rate_from_response(self, json_response):
|
||||||
if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \
|
if "error" in json_response or len(json_response) != 1 or 'trade_price' not in json_response[0]:
|
||||||
'success' not in json_response or json_response['success'] is not True:
|
|
||||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||||
return float(json_response['ticker']['price'])
|
return 1.0 / float(json_response[0]['trade_price'])
|
||||||
|
|
||||||
|
|
||||||
class CryptonatorBTCFeed(CryptonatorFeed):
|
|
||||||
market = "USDBTC"
|
|
||||||
url = "https://api.cryptonator.com/api/ticker/usd-btc"
|
|
||||||
|
|
||||||
|
|
||||||
FEEDS: Iterable[Type[MarketFeed]] = (
|
FEEDS: Iterable[Type[MarketFeed]] = (
|
||||||
LBRYFeed,
|
BittrexBTCFeed,
|
||||||
LBRYBTCFeed,
|
BittrexUSDFeed,
|
||||||
BittrexFeed,
|
CoinExBTCFeed,
|
||||||
# CryptonatorFeed,
|
CoinExUSDFeed,
|
||||||
# CryptonatorBTCFeed,
|
HotbitBTCFeed,
|
||||||
|
HotbitUSDFeed,
|
||||||
|
UPbitBTCFeed,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -191,20 +222,23 @@ class ExchangeRateManager:
|
|||||||
source.stop()
|
source.stop()
|
||||||
|
|
||||||
def convert_currency(self, from_currency, to_currency, amount):
|
def convert_currency(self, from_currency, to_currency, amount):
|
||||||
rates = [market.rate for market in self.market_feeds]
|
log.debug(
|
||||||
log.debug("Converting %f %s to %s, rates: %s", amount, from_currency, to_currency, rates)
|
"Converting %f %s to %s, rates: %s",
|
||||||
|
amount, from_currency, to_currency,
|
||||||
|
[market.rate for market in self.market_feeds]
|
||||||
|
)
|
||||||
if from_currency == to_currency:
|
if from_currency == to_currency:
|
||||||
return round(amount, 8)
|
return round(amount, 8)
|
||||||
|
|
||||||
|
rates = []
|
||||||
for market in self.market_feeds:
|
for market in self.market_feeds:
|
||||||
if (market.has_rate and market.is_online and
|
if (market.has_rate and market.is_online and
|
||||||
market.rate.currency_pair == (from_currency, to_currency)):
|
market.rate.currency_pair == (from_currency, to_currency)):
|
||||||
return round(amount * Decimal(market.rate.spot), 8)
|
rates.append(market.rate.spot)
|
||||||
for market in self.market_feeds:
|
|
||||||
if (market.has_rate and market.is_online and
|
if rates:
|
||||||
market.rate.currency_pair[0] == from_currency):
|
return round(amount * Decimal(median(rates)), 8)
|
||||||
return round(self.convert_currency(
|
|
||||||
market.rate.currency_pair[1], to_currency, amount * Decimal(market.rate.spot)), 8)
|
|
||||||
raise CurrencyConversionError(
|
raise CurrencyConversionError(
|
||||||
f'Unable to convert {amount} from {from_currency} to {to_currency}')
|
f'Unable to convert {amount} from {from_currency} to {to_currency}')
|
||||||
|
|
||||||
|
|||||||
@@ -7,6 +7,8 @@ from json import JSONEncoder
|
|||||||
from google.protobuf.message import DecodeError
|
from google.protobuf.message import DecodeError
|
||||||
|
|
||||||
from lbry.schema.claim import Claim
|
from lbry.schema.claim import Claim
|
||||||
|
from lbry.schema.support import Support
|
||||||
|
from lbry.torrent.torrent_manager import TorrentSource
|
||||||
from lbry.wallet import Wallet, Ledger, Account, Transaction, Output
|
from lbry.wallet import Wallet, Ledger, Account, Transaction, Output
|
||||||
from lbry.wallet.bip32 import PubKey
|
from lbry.wallet.bip32 import PubKey
|
||||||
from lbry.wallet.dewies import dewies_to_lbc
|
from lbry.wallet.dewies import dewies_to_lbc
|
||||||
@@ -25,6 +27,8 @@ def encode_txo_doc():
|
|||||||
'address': "address of who can spend the txo",
|
'address': "address of who can spend the txo",
|
||||||
'confirmations': "number of confirmed blocks",
|
'confirmations': "number of confirmed blocks",
|
||||||
'is_change': "payment to change address, only available when it can be determined",
|
'is_change': "payment to change address, only available when it can be determined",
|
||||||
|
'is_received': "true if txo was sent from external account to this account",
|
||||||
|
'is_spent': "true if txo is spent",
|
||||||
'is_mine': "payment to one of your accounts, only available when it can be determined",
|
'is_mine': "payment to one of your accounts, only available when it can be determined",
|
||||||
'type': "one of 'claim', 'support' or 'purchase'",
|
'type': "one of 'claim', 'support' or 'purchase'",
|
||||||
'name': "when type is 'claim' or 'support', this is the claim name",
|
'name': "when type is 'claim' or 'support', this is the claim name",
|
||||||
@@ -106,7 +110,9 @@ def encode_file_doc():
|
|||||||
'metadata': '(dict) None if claim is not found else the claim metadata',
|
'metadata': '(dict) None if claim is not found else the claim metadata',
|
||||||
'channel_claim_id': '(str) None if claim is not found or not signed',
|
'channel_claim_id': '(str) None if claim is not found or not signed',
|
||||||
'channel_name': '(str) None if claim is not found or not signed',
|
'channel_name': '(str) None if claim is not found or not signed',
|
||||||
'claim_name': '(str) None if claim is not found else the claim name'
|
'claim_name': '(str) None if claim is not found else the claim name',
|
||||||
|
'reflector_progress': '(int) reflector upload progress, 0 to 100',
|
||||||
|
'uploading_to_reflector': '(bool) set to True when currently uploading to reflector'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -117,12 +123,12 @@ class JSONResponseEncoder(JSONEncoder):
|
|||||||
self.ledger = ledger
|
self.ledger = ledger
|
||||||
self.include_protobuf = include_protobuf
|
self.include_protobuf = include_protobuf
|
||||||
|
|
||||||
def default(self, obj): # pylint: disable=method-hidden,arguments-differ,too-many-return-statements
|
def default(self, obj): # pylint: disable=method-hidden,arguments-renamed,too-many-return-statements
|
||||||
if isinstance(obj, Account):
|
if isinstance(obj, Account):
|
||||||
return self.encode_account(obj)
|
return self.encode_account(obj)
|
||||||
if isinstance(obj, Wallet):
|
if isinstance(obj, Wallet):
|
||||||
return self.encode_wallet(obj)
|
return self.encode_wallet(obj)
|
||||||
if isinstance(obj, ManagedStream):
|
if isinstance(obj, (ManagedStream, TorrentSource)):
|
||||||
return self.encode_file(obj)
|
return self.encode_file(obj)
|
||||||
if isinstance(obj, Transaction):
|
if isinstance(obj, Transaction):
|
||||||
return self.encode_transaction(obj)
|
return self.encode_transaction(obj)
|
||||||
@@ -130,6 +136,8 @@ class JSONResponseEncoder(JSONEncoder):
|
|||||||
return self.encode_output(obj)
|
return self.encode_output(obj)
|
||||||
if isinstance(obj, Claim):
|
if isinstance(obj, Claim):
|
||||||
return self.encode_claim(obj)
|
return self.encode_claim(obj)
|
||||||
|
if isinstance(obj, Support):
|
||||||
|
return obj.to_dict()
|
||||||
if isinstance(obj, PubKey):
|
if isinstance(obj, PubKey):
|
||||||
return obj.extended_key_string()
|
return obj.extended_key_string()
|
||||||
if isinstance(obj, datetime):
|
if isinstance(obj, datetime):
|
||||||
@@ -164,12 +172,22 @@ class JSONResponseEncoder(JSONEncoder):
|
|||||||
'amount': dewies_to_lbc(txo.amount),
|
'amount': dewies_to_lbc(txo.amount),
|
||||||
'address': txo.get_address(self.ledger) if txo.has_address else None,
|
'address': txo.get_address(self.ledger) if txo.has_address else None,
|
||||||
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
|
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
|
||||||
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None
|
'timestamp': self.ledger.headers.estimated_timestamp(tx_height)
|
||||||
}
|
}
|
||||||
if txo.is_change is not None:
|
if txo.is_spent is not None:
|
||||||
output['is_change'] = txo.is_change
|
output['is_spent'] = txo.is_spent
|
||||||
if txo.is_my_account is not None:
|
if txo.is_my_output is not None:
|
||||||
output['is_mine'] = txo.is_my_account
|
output['is_my_output'] = txo.is_my_output
|
||||||
|
if txo.is_my_input is not None:
|
||||||
|
output['is_my_input'] = txo.is_my_input
|
||||||
|
if txo.sent_supports is not None:
|
||||||
|
output['sent_supports'] = dewies_to_lbc(txo.sent_supports)
|
||||||
|
if txo.sent_tips is not None:
|
||||||
|
output['sent_tips'] = dewies_to_lbc(txo.sent_tips)
|
||||||
|
if txo.received_tips is not None:
|
||||||
|
output['received_tips'] = dewies_to_lbc(txo.received_tips)
|
||||||
|
if txo.is_internal_transfer is not None:
|
||||||
|
output['is_internal_transfer'] = txo.is_internal_transfer
|
||||||
|
|
||||||
if txo.script.is_claim_name:
|
if txo.script.is_claim_name:
|
||||||
output['type'] = 'claim'
|
output['type'] = 'claim'
|
||||||
@@ -205,25 +223,26 @@ class JSONResponseEncoder(JSONEncoder):
|
|||||||
output['claims'] = [self.encode_output(o) for o in txo.claims]
|
output['claims'] = [self.encode_output(o) for o in txo.claims]
|
||||||
if txo.reposted_claim is not None:
|
if txo.reposted_claim is not None:
|
||||||
output['reposted_claim'] = self.encode_output(txo.reposted_claim)
|
output['reposted_claim'] = self.encode_output(txo.reposted_claim)
|
||||||
if txo.script.is_claim_name or txo.script.is_update_claim:
|
if txo.script.is_claim_name or txo.script.is_update_claim or txo.script.is_support_claim_data:
|
||||||
try:
|
try:
|
||||||
output['value'] = txo.claim
|
output['value'] = txo.signable
|
||||||
|
if self.include_protobuf:
|
||||||
|
output['protobuf'] = hexlify(txo.signable.to_bytes())
|
||||||
|
if txo.purchase_receipt is not None:
|
||||||
|
output['purchase_receipt'] = self.encode_output(txo.purchase_receipt)
|
||||||
|
if txo.script.is_claim_name or txo.script.is_update_claim:
|
||||||
output['value_type'] = txo.claim.claim_type
|
output['value_type'] = txo.claim.claim_type
|
||||||
if self.include_protobuf:
|
|
||||||
output['protobuf'] = hexlify(txo.claim.to_bytes())
|
|
||||||
if txo.purchase_receipt is not None:
|
|
||||||
output['purchase_receipt'] = self.encode_output(txo.purchase_receipt)
|
|
||||||
if txo.claim.is_channel:
|
if txo.claim.is_channel:
|
||||||
output['has_signing_key'] = txo.has_private_key
|
output['has_signing_key'] = txo.has_private_key
|
||||||
if check_signature and txo.claim.is_signed:
|
if check_signature and txo.signable.is_signed:
|
||||||
if txo.channel is not None:
|
if txo.channel is not None:
|
||||||
output['signing_channel'] = self.encode_output(txo.channel)
|
output['signing_channel'] = self.encode_output(txo.channel)
|
||||||
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
|
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
|
||||||
else:
|
else:
|
||||||
output['signing_channel'] = {'channel_id': txo.claim.signing_channel_id}
|
output['signing_channel'] = {'channel_id': txo.signable.signing_channel_id}
|
||||||
output['is_channel_signature_valid'] = False
|
output['is_channel_signature_valid'] = False
|
||||||
except DecodeError:
|
except DecodeError:
|
||||||
pass
|
pass
|
||||||
return output
|
return output
|
||||||
|
|
||||||
def encode_claim_meta(self, meta):
|
def encode_claim_meta(self, meta):
|
||||||
@@ -232,7 +251,7 @@ class JSONResponseEncoder(JSONEncoder):
|
|||||||
if isinstance(value, int):
|
if isinstance(value, int):
|
||||||
meta[key] = dewies_to_lbc(value)
|
meta[key] = dewies_to_lbc(value)
|
||||||
if 0 < meta.get('creation_height', 0) <= self.ledger.headers.height:
|
if 0 < meta.get('creation_height', 0) <= self.ledger.headers.height:
|
||||||
meta['creation_timestamp'] = self.ledger.headers[meta['creation_height']]['timestamp']
|
meta['creation_timestamp'] = self.ledger.headers.estimated_timestamp(meta['creation_height'])
|
||||||
return meta
|
return meta
|
||||||
|
|
||||||
def encode_input(self, txi):
|
def encode_input(self, txi):
|
||||||
@@ -259,26 +278,32 @@ class JSONResponseEncoder(JSONEncoder):
|
|||||||
output_exists = managed_stream.output_file_exists
|
output_exists = managed_stream.output_file_exists
|
||||||
tx_height = managed_stream.stream_claim_info.height
|
tx_height = managed_stream.stream_claim_info.height
|
||||||
best_height = self.ledger.headers.height
|
best_height = self.ledger.headers.height
|
||||||
return {
|
is_stream = hasattr(managed_stream, 'stream_hash')
|
||||||
'streaming_url': managed_stream.stream_url,
|
if is_stream:
|
||||||
|
total_bytes_lower_bound = managed_stream.descriptor.lower_bound_decrypted_length()
|
||||||
|
total_bytes = managed_stream.descriptor.upper_bound_decrypted_length()
|
||||||
|
else:
|
||||||
|
total_bytes_lower_bound = total_bytes = managed_stream.torrent_length
|
||||||
|
result = {
|
||||||
|
'streaming_url': None,
|
||||||
'completed': managed_stream.completed,
|
'completed': managed_stream.completed,
|
||||||
'file_name': managed_stream.file_name if output_exists else None,
|
'file_name': None,
|
||||||
'download_directory': managed_stream.download_directory if output_exists else None,
|
'download_directory': None,
|
||||||
'download_path': managed_stream.full_path if output_exists else None,
|
'download_path': None,
|
||||||
'points_paid': 0.0,
|
'points_paid': 0.0,
|
||||||
'stopped': not managed_stream.running,
|
'stopped': not managed_stream.running,
|
||||||
'stream_hash': managed_stream.stream_hash,
|
'stream_hash': None,
|
||||||
'stream_name': managed_stream.descriptor.stream_name,
|
'stream_name': None,
|
||||||
'suggested_file_name': managed_stream.descriptor.suggested_file_name,
|
'suggested_file_name': None,
|
||||||
'sd_hash': managed_stream.descriptor.sd_hash,
|
'sd_hash': None,
|
||||||
'mime_type': managed_stream.mime_type,
|
'mime_type': None,
|
||||||
'key': managed_stream.descriptor.key,
|
'key': None,
|
||||||
'total_bytes_lower_bound': managed_stream.descriptor.lower_bound_decrypted_length(),
|
'total_bytes_lower_bound': total_bytes_lower_bound,
|
||||||
'total_bytes': managed_stream.descriptor.upper_bound_decrypted_length(),
|
'total_bytes': total_bytes,
|
||||||
'written_bytes': managed_stream.written_bytes,
|
'written_bytes': managed_stream.written_bytes,
|
||||||
'blobs_completed': managed_stream.blobs_completed,
|
'blobs_completed': None,
|
||||||
'blobs_in_stream': managed_stream.blobs_in_stream,
|
'blobs_in_stream': None,
|
||||||
'blobs_remaining': managed_stream.blobs_remaining,
|
'blobs_remaining': None,
|
||||||
'status': managed_stream.status,
|
'status': managed_stream.status,
|
||||||
'claim_id': managed_stream.claim_id,
|
'claim_id': managed_stream.claim_id,
|
||||||
'txid': managed_stream.txid,
|
'txid': managed_stream.txid,
|
||||||
@@ -294,9 +319,38 @@ class JSONResponseEncoder(JSONEncoder):
|
|||||||
'added_on': managed_stream.added_on,
|
'added_on': managed_stream.added_on,
|
||||||
'height': tx_height,
|
'height': tx_height,
|
||||||
'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height,
|
'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height,
|
||||||
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None,
|
'timestamp': self.ledger.headers.estimated_timestamp(tx_height),
|
||||||
'is_fully_reflected': managed_stream.is_fully_reflected
|
'is_fully_reflected': False,
|
||||||
|
'reflector_progress': False,
|
||||||
|
'uploading_to_reflector': False
|
||||||
}
|
}
|
||||||
|
if is_stream:
|
||||||
|
result.update({
|
||||||
|
'streaming_url': managed_stream.stream_url,
|
||||||
|
'stream_hash': managed_stream.stream_hash,
|
||||||
|
'stream_name': managed_stream.descriptor.stream_name,
|
||||||
|
'suggested_file_name': managed_stream.descriptor.suggested_file_name,
|
||||||
|
'sd_hash': managed_stream.descriptor.sd_hash,
|
||||||
|
'mime_type': managed_stream.mime_type,
|
||||||
|
'key': managed_stream.descriptor.key,
|
||||||
|
'blobs_completed': managed_stream.blobs_completed,
|
||||||
|
'blobs_in_stream': managed_stream.blobs_in_stream,
|
||||||
|
'blobs_remaining': managed_stream.blobs_remaining,
|
||||||
|
'is_fully_reflected': managed_stream.is_fully_reflected,
|
||||||
|
'reflector_progress': managed_stream.reflector_progress,
|
||||||
|
'uploading_to_reflector': managed_stream.uploading_to_reflector
|
||||||
|
})
|
||||||
|
else:
|
||||||
|
result.update({
|
||||||
|
'streaming_url': f'file://{managed_stream.full_path}',
|
||||||
|
})
|
||||||
|
if output_exists:
|
||||||
|
result.update({
|
||||||
|
'file_name': managed_stream.file_name,
|
||||||
|
'download_directory': managed_stream.download_directory,
|
||||||
|
'download_path': managed_stream.full_path,
|
||||||
|
})
|
||||||
|
return result
|
||||||
|
|
||||||
def encode_claim(self, claim):
|
def encode_claim(self, claim):
|
||||||
encoded = getattr(claim, claim.claim_type).to_dict()
|
encoded = getattr(claim, claim.claim_type).to_dict()
|
||||||
|
|||||||
@@ -1,95 +0,0 @@
|
|||||||
import asyncio
|
|
||||||
import json
|
|
||||||
import logging.handlers
|
|
||||||
import traceback
|
|
||||||
|
|
||||||
import typing
|
|
||||||
from aiohttp.client_exceptions import ClientError
|
|
||||||
import aiohttp
|
|
||||||
from lbry import utils, __version__
|
|
||||||
if typing.TYPE_CHECKING:
|
|
||||||
from lbry.conf import Config
|
|
||||||
|
|
||||||
LOGGLY_TOKEN = 'BQEzZmMzLJHgAGxkBF00LGD0YGuyATVgAmqxAQEuAQZ2BQH4'
|
|
||||||
|
|
||||||
|
|
||||||
class JsonFormatter(logging.Formatter):
|
|
||||||
"""Format log records using json serialization"""
|
|
||||||
|
|
||||||
def __init__(self, **kwargs):
|
|
||||||
super().__init__()
|
|
||||||
self.attributes = kwargs
|
|
||||||
|
|
||||||
def format(self, record):
|
|
||||||
data = {
|
|
||||||
'loggerName': record.name,
|
|
||||||
'asciTime': self.formatTime(record),
|
|
||||||
'fileName': record.filename,
|
|
||||||
'functionName': record.funcName,
|
|
||||||
'levelNo': record.levelno,
|
|
||||||
'lineNo': record.lineno,
|
|
||||||
'levelName': record.levelname,
|
|
||||||
'message': record.getMessage(),
|
|
||||||
}
|
|
||||||
data.update(self.attributes)
|
|
||||||
if record.exc_info:
|
|
||||||
data['exc_info'] = self.formatException(record.exc_info)
|
|
||||||
return json.dumps(data)
|
|
||||||
|
|
||||||
|
|
||||||
class HTTPSLogglyHandler(logging.Handler):
|
|
||||||
def __init__(self, loggly_token: str, config: 'Config'):
|
|
||||||
super().__init__()
|
|
||||||
self.cookies = {}
|
|
||||||
self.url = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}".format(
|
|
||||||
token=utils.deobfuscate(loggly_token), tag='lbrynet-' + __version__
|
|
||||||
)
|
|
||||||
self._loop = asyncio.get_event_loop()
|
|
||||||
self._session = aiohttp.ClientSession()
|
|
||||||
self._config = config
|
|
||||||
|
|
||||||
@property
|
|
||||||
def enabled(self):
|
|
||||||
return self._config.share_usage_data
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def get_full_message(record):
|
|
||||||
if record.exc_info:
|
|
||||||
return '\n'.join(traceback.format_exception(*record.exc_info))
|
|
||||||
else:
|
|
||||||
return record.getMessage()
|
|
||||||
|
|
||||||
async def _emit(self, record, retry=True):
|
|
||||||
data = self.format(record).encode()
|
|
||||||
try:
|
|
||||||
async with self._session.post(self.url, data=data,
|
|
||||||
cookies=self.cookies) as response:
|
|
||||||
self.cookies.update(response.cookies)
|
|
||||||
except ClientError:
|
|
||||||
if self._loop.is_running() and retry and self.enabled:
|
|
||||||
await self._session.close()
|
|
||||||
self._session = aiohttp.ClientSession()
|
|
||||||
return await self._emit(record, retry=False)
|
|
||||||
|
|
||||||
def emit(self, record):
|
|
||||||
if not self.enabled:
|
|
||||||
return
|
|
||||||
try:
|
|
||||||
asyncio.ensure_future(self._emit(record), loop=self._loop)
|
|
||||||
except RuntimeError: # TODO: use a second loop
|
|
||||||
print(f"\nfailed to send traceback to loggly, please file an issue with the following traceback:\n"
|
|
||||||
f"{self.format(record)}")
|
|
||||||
|
|
||||||
def close(self):
|
|
||||||
super().close()
|
|
||||||
try:
|
|
||||||
loop = asyncio.get_event_loop()
|
|
||||||
loop.run_until_complete(self._session.close())
|
|
||||||
except RuntimeError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def get_loggly_handler(config):
|
|
||||||
handler = HTTPSLogglyHandler(LOGGLY_TOKEN, config=config)
|
|
||||||
handler.setFormatter(JsonFormatter())
|
|
||||||
return handler
|
|
||||||
@@ -35,6 +35,8 @@ def migrate_db(conf, start, end):
|
|||||||
from .migrate12to13 import do_migration
|
from .migrate12to13 import do_migration
|
||||||
elif current == 13:
|
elif current == 13:
|
||||||
from .migrate13to14 import do_migration
|
from .migrate13to14 import do_migration
|
||||||
|
elif current == 14:
|
||||||
|
from .migrate14to15 import do_migration
|
||||||
else:
|
else:
|
||||||
raise Exception(f"DB migration of version {current} to {current+1} is not available")
|
raise Exception(f"DB migration of version {current} to {current+1} is not available")
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
|
||||||
|
|
||||||
|
def do_migration(conf):
|
||||||
|
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||||
|
connection = sqlite3.connect(db_path)
|
||||||
|
cursor = connection.cursor()
|
||||||
|
|
||||||
|
cursor.executescript("""
|
||||||
|
alter table blob add column added_on integer not null default 0;
|
||||||
|
alter table blob add column is_mine integer not null default 1;
|
||||||
|
""")
|
||||||
|
|
||||||
|
connection.commit()
|
||||||
|
connection.close()
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
import logging
|
||||||
|
from aiohttp import web
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_request_allowed(request, conf):
|
||||||
|
if is_request_allowed(request, conf):
|
||||||
|
return
|
||||||
|
if conf.allowed_origin:
|
||||||
|
log.warning(
|
||||||
|
"API requests with Origin '%s' are not allowed, "
|
||||||
|
"configuration 'allowed_origin' limits requests to: '%s'",
|
||||||
|
request.headers.get('Origin'), conf.allowed_origin
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
log.warning(
|
||||||
|
"API requests with Origin '%s' are not allowed, "
|
||||||
|
"update configuration 'allowed_origin' to enable this origin.",
|
||||||
|
request.headers.get('Origin')
|
||||||
|
)
|
||||||
|
raise web.HTTPForbidden()
|
||||||
|
|
||||||
|
|
||||||
|
def is_request_allowed(request, conf) -> bool:
|
||||||
|
origin = request.headers.get('Origin')
|
||||||
|
return (
|
||||||
|
origin is None or
|
||||||
|
origin == conf.allowed_origin or
|
||||||
|
conf.allowed_origin == '*'
|
||||||
|
)
|
||||||
@@ -9,7 +9,7 @@ from typing import Optional
|
|||||||
from lbry.wallet import SQLiteMixin
|
from lbry.wallet import SQLiteMixin
|
||||||
from lbry.conf import Config
|
from lbry.conf import Config
|
||||||
from lbry.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
from lbry.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
||||||
from lbry.wallet.transaction import Transaction
|
from lbry.wallet.transaction import Transaction, Output
|
||||||
from lbry.schema.claim import Claim
|
from lbry.schema.claim import Claim
|
||||||
from lbry.dht.constants import DATA_EXPIRATION
|
from lbry.dht.constants import DATA_EXPIRATION
|
||||||
from lbry.blob.blob_info import BlobInfo
|
from lbry.blob.blob_info import BlobInfo
|
||||||
@@ -170,8 +170,8 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
|
|||||||
def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'):
|
def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'):
|
||||||
# add all blobs, except the last one, which is empty
|
# add all blobs, except the last one, which is empty
|
||||||
transaction.executemany(
|
transaction.executemany(
|
||||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||||
((blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
|
((blob.blob_hash, blob.length, 0, 0, "pending", 0, 0, blob.added_on, blob.is_mine)
|
||||||
for blob in (descriptor.blobs[:-1] if len(descriptor.blobs) > 1 else descriptor.blobs) + [sd_blob])
|
for blob in (descriptor.blobs[:-1] if len(descriptor.blobs) > 1 else descriptor.blobs) + [sd_blob])
|
||||||
).fetchall()
|
).fetchall()
|
||||||
# associate the blobs to the stream
|
# associate the blobs to the stream
|
||||||
@@ -242,7 +242,9 @@ class SQLiteStorage(SQLiteMixin):
|
|||||||
should_announce integer not null default 0,
|
should_announce integer not null default 0,
|
||||||
status text not null,
|
status text not null,
|
||||||
last_announced_time integer,
|
last_announced_time integer,
|
||||||
single_announce integer
|
single_announce integer,
|
||||||
|
added_on integer not null,
|
||||||
|
is_mine integer not null default 0
|
||||||
);
|
);
|
||||||
|
|
||||||
create table if not exists stream (
|
create table if not exists stream (
|
||||||
@@ -356,19 +358,19 @@ class SQLiteStorage(SQLiteMixin):
|
|||||||
|
|
||||||
# # # # # # # # # blob functions # # # # # # # # #
|
# # # # # # # # # blob functions # # # # # # # # #
|
||||||
|
|
||||||
async def add_blobs(self, *blob_hashes_and_lengths: typing.Tuple[str, int], finished=False):
|
async def add_blobs(self, *blob_hashes_and_lengths: typing.Tuple[str, int, int, int], finished=False):
|
||||||
def _add_blobs(transaction: sqlite3.Connection):
|
def _add_blobs(transaction: sqlite3.Connection):
|
||||||
transaction.executemany(
|
transaction.executemany(
|
||||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||||
(
|
(
|
||||||
(blob_hash, length, 0, 0, "pending" if not finished else "finished", 0, 0)
|
(blob_hash, length, 0, 0, "pending" if not finished else "finished", 0, 0, added_on, is_mine)
|
||||||
for blob_hash, length in blob_hashes_and_lengths
|
for blob_hash, length, added_on, is_mine in blob_hashes_and_lengths
|
||||||
)
|
)
|
||||||
).fetchall()
|
).fetchall()
|
||||||
if finished:
|
if finished:
|
||||||
transaction.executemany(
|
transaction.executemany(
|
||||||
"update blob set status='finished' where blob.blob_hash=?", (
|
"update blob set status='finished' where blob.blob_hash=?", (
|
||||||
(blob_hash, ) for blob_hash, _ in blob_hashes_and_lengths
|
(blob_hash, ) for blob_hash, _, _, _ in blob_hashes_and_lengths
|
||||||
)
|
)
|
||||||
).fetchall()
|
).fetchall()
|
||||||
return await self.db.run(_add_blobs)
|
return await self.db.run(_add_blobs)
|
||||||
@@ -435,6 +437,29 @@ class SQLiteStorage(SQLiteMixin):
|
|||||||
def get_all_blob_hashes(self):
|
def get_all_blob_hashes(self):
|
||||||
return self.run_and_return_list("select blob_hash from blob")
|
return self.run_and_return_list("select blob_hash from blob")
|
||||||
|
|
||||||
|
async def get_stored_blobs(self, is_mine: bool):
|
||||||
|
is_mine = 1 if is_mine else 0
|
||||||
|
return await self.db.execute_fetchall(
|
||||||
|
"select blob_hash, blob_length, added_on from blob where is_mine=? order by added_on asc",
|
||||||
|
(is_mine,)
|
||||||
|
)
|
||||||
|
|
||||||
|
async def get_stored_blob_disk_usage(self, is_mine: Optional[bool] = None):
|
||||||
|
if is_mine is None:
|
||||||
|
sql, args = "select coalesce(sum(blob_length), 0) from blob", ()
|
||||||
|
else:
|
||||||
|
is_mine = 1 if is_mine else 0
|
||||||
|
sql, args = "select coalesce(sum(blob_length), 0) from blob where is_mine=?", (is_mine,)
|
||||||
|
return (await self.db.execute_fetchone(sql, args))[0]
|
||||||
|
|
||||||
|
async def update_blob_ownership(self, sd_hash, is_mine: bool):
|
||||||
|
is_mine = 1 if is_mine else 0
|
||||||
|
await self.db.execute_fetchall(
|
||||||
|
"update blob set is_mine = ? where blob_hash in ("
|
||||||
|
" select blob_hash from blob natural join stream_blob natural join stream where sd_hash = ?"
|
||||||
|
") OR blob_hash = ?", (is_mine, sd_hash, sd_hash)
|
||||||
|
)
|
||||||
|
|
||||||
def sync_missing_blobs(self, blob_files: typing.Set[str]) -> typing.Awaitable[typing.Set[str]]:
|
def sync_missing_blobs(self, blob_files: typing.Set[str]) -> typing.Awaitable[typing.Set[str]]:
|
||||||
def _sync_blobs(transaction: sqlite3.Connection) -> typing.Set[str]:
|
def _sync_blobs(transaction: sqlite3.Connection) -> typing.Set[str]:
|
||||||
finished_blob_hashes = tuple(
|
finished_blob_hashes = tuple(
|
||||||
@@ -570,6 +595,10 @@ class SQLiteStorage(SQLiteMixin):
|
|||||||
log.debug("update file status %s -> %s", stream_hash, new_status)
|
log.debug("update file status %s -> %s", stream_hash, new_status)
|
||||||
return self.db.execute_fetchall("update file set status=? where stream_hash=?", (new_status, stream_hash))
|
return self.db.execute_fetchall("update file set status=? where stream_hash=?", (new_status, stream_hash))
|
||||||
|
|
||||||
|
def stop_all_files(self):
|
||||||
|
log.debug("stopping all files")
|
||||||
|
return self.db.execute_fetchall("update file set status=?", ("stopped",))
|
||||||
|
|
||||||
async def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: typing.Optional[str],
|
async def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: typing.Optional[str],
|
||||||
file_name: typing.Optional[str]):
|
file_name: typing.Optional[str]):
|
||||||
if not file_name or not download_dir:
|
if not file_name or not download_dir:
|
||||||
@@ -617,7 +646,7 @@ class SQLiteStorage(SQLiteMixin):
|
|||||||
).fetchall()
|
).fetchall()
|
||||||
download_dir = binascii.hexlify(self.conf.download_dir.encode()).decode()
|
download_dir = binascii.hexlify(self.conf.download_dir.encode()).decode()
|
||||||
transaction.executemany(
|
transaction.executemany(
|
||||||
f"update file set download_directory=? where stream_hash=?",
|
"update file set download_directory=? where stream_hash=?",
|
||||||
((download_dir, stream_hash) for stream_hash in stream_hashes)
|
((download_dir, stream_hash) for stream_hash in stream_hashes)
|
||||||
).fetchall()
|
).fetchall()
|
||||||
await self.db.run_with_foreign_keys_disabled(_recover)
|
await self.db.run_with_foreign_keys_disabled(_recover)
|
||||||
@@ -727,6 +756,19 @@ class SQLiteStorage(SQLiteMixin):
|
|||||||
if claim_id_to_supports:
|
if claim_id_to_supports:
|
||||||
await self.save_supports(claim_id_to_supports)
|
await self.save_supports(claim_id_to_supports)
|
||||||
|
|
||||||
|
def save_claim_from_output(self, ledger, *outputs: Output):
|
||||||
|
return self.save_claims([{
|
||||||
|
"claim_id": output.claim_id,
|
||||||
|
"name": output.claim_name,
|
||||||
|
"amount": dewies_to_lbc(output.amount),
|
||||||
|
"address": output.get_address(ledger),
|
||||||
|
"txid": output.tx_ref.id,
|
||||||
|
"nout": output.position,
|
||||||
|
"value": output.claim,
|
||||||
|
"height": output.tx_ref.height,
|
||||||
|
"claim_sequence": -1,
|
||||||
|
} for output in outputs])
|
||||||
|
|
||||||
def save_claims_for_resolve(self, claim_infos):
|
def save_claims_for_resolve(self, claim_infos):
|
||||||
to_save = {}
|
to_save = {}
|
||||||
for info in claim_infos:
|
for info in claim_infos:
|
||||||
@@ -740,7 +782,8 @@ class SQLiteStorage(SQLiteMixin):
|
|||||||
return self.save_claims(to_save.values())
|
return self.save_claims(to_save.values())
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _save_content_claim(transaction, claim_outpoint, stream_hash):
|
def _save_content_claim(transaction, claim_outpoint, stream_hash=None, bt_infohash=None):
|
||||||
|
assert stream_hash or bt_infohash
|
||||||
# get the claim id and serialized metadata
|
# get the claim id and serialized metadata
|
||||||
claim_info = transaction.execute(
|
claim_info = transaction.execute(
|
||||||
"select claim_id, serialized_metadata from claim where claim_outpoint=?", (claim_outpoint,)
|
"select claim_id, serialized_metadata from claim where claim_outpoint=?", (claim_outpoint,)
|
||||||
@@ -788,6 +831,19 @@ class SQLiteStorage(SQLiteMixin):
|
|||||||
if stream_hash in self.content_claim_callbacks:
|
if stream_hash in self.content_claim_callbacks:
|
||||||
await self.content_claim_callbacks[stream_hash]()
|
await self.content_claim_callbacks[stream_hash]()
|
||||||
|
|
||||||
|
async def save_torrent_content_claim(self, bt_infohash, claim_outpoint, length, name):
|
||||||
|
def _save_torrent(transaction):
|
||||||
|
transaction.execute(
|
||||||
|
"insert or replace into torrent values (?, NULL, ?, ?)", (bt_infohash, length, name)
|
||||||
|
).fetchall()
|
||||||
|
transaction.execute(
|
||||||
|
"insert or replace into content_claim values (NULL, ?, ?)", (bt_infohash, claim_outpoint)
|
||||||
|
).fetchall()
|
||||||
|
await self.db.run(_save_torrent)
|
||||||
|
# update corresponding ManagedEncryptedFileDownloader object
|
||||||
|
if bt_infohash in self.content_claim_callbacks:
|
||||||
|
await self.content_claim_callbacks[bt_infohash]()
|
||||||
|
|
||||||
async def get_content_claim(self, stream_hash: str, include_supports: typing.Optional[bool] = True) -> typing.Dict:
|
async def get_content_claim(self, stream_hash: str, include_supports: typing.Optional[bool] = True) -> typing.Dict:
|
||||||
claims = await self.db.run(get_claims_from_stream_hashes, [stream_hash])
|
claims = await self.db.run(get_claims_from_stream_hashes, [stream_hash])
|
||||||
claim = None
|
claim = None
|
||||||
@@ -799,6 +855,10 @@ class SQLiteStorage(SQLiteMixin):
|
|||||||
claim['effective_amount'] = calculate_effective_amount(claim['amount'], supports)
|
claim['effective_amount'] = calculate_effective_amount(claim['amount'], supports)
|
||||||
return claim
|
return claim
|
||||||
|
|
||||||
|
async def get_content_claim_for_torrent(self, bt_infohash):
|
||||||
|
claims = await self.db.run(get_claims_from_torrent_info_hashes, [bt_infohash])
|
||||||
|
return claims[bt_infohash].as_dict() if claims else None
|
||||||
|
|
||||||
# # # # # # # # # reflector functions # # # # # # # # #
|
# # # # # # # # # reflector functions # # # # # # # # #
|
||||||
|
|
||||||
def update_reflected_stream(self, sd_hash, reflector_address, success=True):
|
def update_reflected_stream(self, sd_hash, reflector_address, success=True):
|
||||||
@@ -830,6 +890,6 @@ class SQLiteStorage(SQLiteMixin):
|
|||||||
transaction.execute('delete from peer').fetchall()
|
transaction.execute('delete from peer').fetchall()
|
||||||
transaction.executemany(
|
transaction.executemany(
|
||||||
'insert into peer(node_id, address, udp_port, tcp_port) values (?, ?, ?, ?)',
|
'insert into peer(node_id, address, udp_port, tcp_port) values (?, ?, ?, ?)',
|
||||||
tuple([(binascii.hexlify(p.node_id), p.address, p.udp_port, p.tcp_port) for p in peers])
|
((binascii.hexlify(p.node_id), p.address, p.udp_port, p.tcp_port) for p in peers)
|
||||||
).fetchall()
|
).fetchall()
|
||||||
return await self.db.run(_save_kademlia_peers)
|
return await self.db.run(_save_kademlia_peers)
|
||||||
File renamed without changes.
@@ -0,0 +1,302 @@
|
|||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
import typing
|
||||||
|
from typing import Optional
|
||||||
|
from aiohttp.web import Request
|
||||||
|
from lbry.error import ResolveError, DownloadSDTimeoutError, InsufficientFundsError
|
||||||
|
from lbry.error import ResolveTimeoutError, DownloadDataTimeoutError, KeyFeeAboveMaxAllowedError
|
||||||
|
from lbry.stream.managed_stream import ManagedStream
|
||||||
|
from lbry.torrent.torrent_manager import TorrentSource
|
||||||
|
from lbry.utils import cache_concurrent
|
||||||
|
from lbry.schema.url import URL
|
||||||
|
from lbry.wallet.dewies import dewies_to_lbc
|
||||||
|
from lbry.file.source_manager import SourceManager
|
||||||
|
from lbry.file.source import ManagedDownloadSource
|
||||||
|
if typing.TYPE_CHECKING:
|
||||||
|
from lbry.conf import Config
|
||||||
|
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||||
|
from lbry.extras.daemon.storage import SQLiteStorage
|
||||||
|
from lbry.wallet import WalletManager, Output
|
||||||
|
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class FileManager:
|
||||||
|
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', wallet_manager: 'WalletManager',
|
||||||
|
storage: 'SQLiteStorage', analytics_manager: Optional['AnalyticsManager'] = None):
|
||||||
|
self.loop = loop
|
||||||
|
self.config = config
|
||||||
|
self.wallet_manager = wallet_manager
|
||||||
|
self.storage = storage
|
||||||
|
self.analytics_manager = analytics_manager
|
||||||
|
self.source_managers: typing.Dict[str, SourceManager] = {}
|
||||||
|
self.started = asyncio.Event()
|
||||||
|
|
||||||
|
@property
|
||||||
|
def streams(self):
|
||||||
|
return self.source_managers['stream']._sources
|
||||||
|
|
||||||
|
async def create_stream(self, file_path: str, key: Optional[bytes] = None, **kwargs) -> ManagedDownloadSource:
|
||||||
|
if 'stream' in self.source_managers:
|
||||||
|
return await self.source_managers['stream'].create(file_path, key, **kwargs)
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
async def start(self):
|
||||||
|
await asyncio.gather(*(source_manager.start() for source_manager in self.source_managers.values()))
|
||||||
|
for manager in self.source_managers.values():
|
||||||
|
await manager.started.wait()
|
||||||
|
self.started.set()
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
for manager in self.source_managers.values():
|
||||||
|
# fixme: pop or not?
|
||||||
|
manager.stop()
|
||||||
|
self.started.clear()
|
||||||
|
|
||||||
|
@cache_concurrent
|
||||||
|
async def download_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
|
||||||
|
timeout: Optional[float] = None, file_name: Optional[str] = None,
|
||||||
|
download_directory: Optional[str] = None,
|
||||||
|
save_file: Optional[bool] = None, resolve_timeout: float = 3.0,
|
||||||
|
wallet: Optional['Wallet'] = None) -> ManagedDownloadSource:
|
||||||
|
|
||||||
|
wallet = wallet or self.wallet_manager.default_wallet
|
||||||
|
timeout = timeout or self.config.download_timeout
|
||||||
|
start_time = self.loop.time()
|
||||||
|
resolved_time = None
|
||||||
|
stream = None
|
||||||
|
claim = None
|
||||||
|
error = None
|
||||||
|
outpoint = None
|
||||||
|
if save_file is None:
|
||||||
|
save_file = self.config.save_files
|
||||||
|
if file_name and not save_file:
|
||||||
|
save_file = True
|
||||||
|
if save_file:
|
||||||
|
download_directory = download_directory or self.config.download_dir
|
||||||
|
else:
|
||||||
|
download_directory = None
|
||||||
|
|
||||||
|
payment = None
|
||||||
|
try:
|
||||||
|
# resolve the claim
|
||||||
|
if not URL.parse(uri).has_stream:
|
||||||
|
raise ResolveError("cannot download a channel claim, specify a /path")
|
||||||
|
try:
|
||||||
|
resolved_result = await asyncio.wait_for(
|
||||||
|
self.wallet_manager.ledger.resolve(
|
||||||
|
wallet.accounts, [uri],
|
||||||
|
include_purchase_receipt=True,
|
||||||
|
include_is_my_output=True
|
||||||
|
), resolve_timeout
|
||||||
|
)
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
raise ResolveTimeoutError(uri)
|
||||||
|
except Exception as err:
|
||||||
|
if isinstance(err, asyncio.CancelledError):
|
||||||
|
raise
|
||||||
|
log.exception("Unexpected error resolving stream:")
|
||||||
|
raise ResolveError(f"Unexpected error resolving stream: {str(err)}")
|
||||||
|
if 'error' in resolved_result:
|
||||||
|
raise ResolveError(f"Unexpected error resolving uri for download: {resolved_result['error']}")
|
||||||
|
if not resolved_result or uri not in resolved_result:
|
||||||
|
raise ResolveError(f"Failed to resolve stream at '{uri}'")
|
||||||
|
txo = resolved_result[uri]
|
||||||
|
if isinstance(txo, dict):
|
||||||
|
raise ResolveError(f"Failed to resolve stream at '{uri}': {txo}")
|
||||||
|
claim = txo.claim
|
||||||
|
outpoint = f"{txo.tx_ref.id}:{txo.position}"
|
||||||
|
resolved_time = self.loop.time() - start_time
|
||||||
|
await self.storage.save_claim_from_output(self.wallet_manager.ledger, txo)
|
||||||
|
|
||||||
|
####################
|
||||||
|
# update or replace
|
||||||
|
####################
|
||||||
|
|
||||||
|
if claim.stream.source.bt_infohash:
|
||||||
|
source_manager = self.source_managers['torrent']
|
||||||
|
existing = source_manager.get_filtered(bt_infohash=claim.stream.source.bt_infohash)
|
||||||
|
elif claim.stream.source.sd_hash:
|
||||||
|
source_manager = self.source_managers['stream']
|
||||||
|
existing = source_manager.get_filtered(sd_hash=claim.stream.source.sd_hash)
|
||||||
|
else:
|
||||||
|
raise ResolveError(f"There is nothing to download at {uri} - Source is unknown or unset")
|
||||||
|
|
||||||
|
# resume or update an existing stream, if the stream changed: download it and delete the old one after
|
||||||
|
to_replace, updated_stream = None, None
|
||||||
|
if existing and existing[0].claim_id != txo.claim_id:
|
||||||
|
raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {txo.claim_id}")
|
||||||
|
if existing:
|
||||||
|
log.info("claim contains a metadata only update to a stream we have")
|
||||||
|
if claim.stream.source.bt_infohash:
|
||||||
|
await self.storage.save_torrent_content_claim(
|
||||||
|
existing[0].identifier, outpoint, existing[0].torrent_length, existing[0].torrent_name
|
||||||
|
)
|
||||||
|
claim_info = await self.storage.get_content_claim_for_torrent(existing[0].identifier)
|
||||||
|
existing[0].set_claim(claim_info, claim)
|
||||||
|
else:
|
||||||
|
await self.storage.save_content_claim(
|
||||||
|
existing[0].stream_hash, outpoint
|
||||||
|
)
|
||||||
|
await source_manager._update_content_claim(existing[0])
|
||||||
|
updated_stream = existing[0]
|
||||||
|
else:
|
||||||
|
existing_for_claim_id = self.get_filtered(claim_id=txo.claim_id)
|
||||||
|
if existing_for_claim_id:
|
||||||
|
log.info("claim contains an update to a stream we have, downloading it")
|
||||||
|
if save_file and existing_for_claim_id[0].output_file_exists:
|
||||||
|
save_file = False
|
||||||
|
if not claim.stream.source.bt_infohash:
|
||||||
|
existing_for_claim_id[0].downloader.node = source_manager.node
|
||||||
|
await existing_for_claim_id[0].start(timeout=timeout, save_now=save_file)
|
||||||
|
if not existing_for_claim_id[0].output_file_exists and (
|
||||||
|
save_file or file_name or download_directory):
|
||||||
|
await existing_for_claim_id[0].save_file(
|
||||||
|
file_name=file_name, download_directory=download_directory
|
||||||
|
)
|
||||||
|
to_replace = existing_for_claim_id[0]
|
||||||
|
|
||||||
|
# resume or update an existing stream, if the stream changed: download it and delete the old one after
|
||||||
|
if updated_stream:
|
||||||
|
log.info("already have stream for %s", uri)
|
||||||
|
if save_file and updated_stream.output_file_exists:
|
||||||
|
save_file = False
|
||||||
|
if not claim.stream.source.bt_infohash:
|
||||||
|
updated_stream.downloader.node = source_manager.node
|
||||||
|
await updated_stream.start(timeout=timeout, save_now=save_file)
|
||||||
|
if not updated_stream.output_file_exists and (save_file or file_name or download_directory):
|
||||||
|
await updated_stream.save_file(
|
||||||
|
file_name=file_name, download_directory=download_directory
|
||||||
|
)
|
||||||
|
return updated_stream
|
||||||
|
|
||||||
|
####################
|
||||||
|
# pay fee
|
||||||
|
####################
|
||||||
|
|
||||||
|
needs_purchasing = (
|
||||||
|
not to_replace and
|
||||||
|
not txo.is_my_output and
|
||||||
|
txo.has_price and
|
||||||
|
not txo.purchase_receipt
|
||||||
|
)
|
||||||
|
|
||||||
|
if needs_purchasing:
|
||||||
|
payment = await self.wallet_manager.create_purchase_transaction(
|
||||||
|
wallet.accounts, txo, exchange_rate_manager
|
||||||
|
)
|
||||||
|
|
||||||
|
####################
|
||||||
|
# make downloader and wait for start
|
||||||
|
####################
|
||||||
|
|
||||||
|
if not claim.stream.source.bt_infohash:
|
||||||
|
# fixme: this shouldnt be here
|
||||||
|
stream = ManagedStream(
|
||||||
|
self.loop, self.config, source_manager.blob_manager, claim.stream.source.sd_hash,
|
||||||
|
download_directory, file_name, ManagedStream.STATUS_RUNNING, content_fee=payment,
|
||||||
|
analytics_manager=self.analytics_manager
|
||||||
|
)
|
||||||
|
stream.downloader.node = source_manager.node
|
||||||
|
else:
|
||||||
|
stream = TorrentSource(
|
||||||
|
self.loop, self.config, self.storage, identifier=claim.stream.source.bt_infohash,
|
||||||
|
file_name=file_name, download_directory=download_directory or self.config.download_dir,
|
||||||
|
status=ManagedStream.STATUS_RUNNING,
|
||||||
|
analytics_manager=self.analytics_manager,
|
||||||
|
torrent_session=source_manager.torrent_session
|
||||||
|
)
|
||||||
|
log.info("starting download for %s", uri)
|
||||||
|
|
||||||
|
before_download = self.loop.time()
|
||||||
|
await stream.start(timeout, save_file)
|
||||||
|
|
||||||
|
####################
|
||||||
|
# success case: delete to_replace if applicable, broadcast fee payment
|
||||||
|
####################
|
||||||
|
|
||||||
|
if to_replace: # delete old stream now that the replacement has started downloading
|
||||||
|
await source_manager.delete(to_replace)
|
||||||
|
|
||||||
|
if payment is not None:
|
||||||
|
await self.wallet_manager.broadcast_or_release(payment)
|
||||||
|
payment = None # to avoid releasing in `finally` later
|
||||||
|
log.info("paid fee of %s for %s", dewies_to_lbc(stream.content_fee.outputs[0].amount), uri)
|
||||||
|
await self.storage.save_content_fee(stream.stream_hash, stream.content_fee)
|
||||||
|
|
||||||
|
source_manager.add(stream)
|
||||||
|
|
||||||
|
if not claim.stream.source.bt_infohash:
|
||||||
|
await self.storage.save_content_claim(stream.stream_hash, outpoint)
|
||||||
|
else:
|
||||||
|
await self.storage.save_torrent_content_claim(
|
||||||
|
stream.identifier, outpoint, stream.torrent_length, stream.torrent_name
|
||||||
|
)
|
||||||
|
claim_info = await self.storage.get_content_claim_for_torrent(stream.identifier)
|
||||||
|
stream.set_claim(claim_info, claim)
|
||||||
|
if save_file:
|
||||||
|
await asyncio.wait_for(stream.save_file(), timeout - (self.loop.time() - before_download),
|
||||||
|
loop=self.loop)
|
||||||
|
return stream
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
error = DownloadDataTimeoutError(stream.sd_hash)
|
||||||
|
raise error
|
||||||
|
except Exception as err: # forgive data timeout, don't delete stream
|
||||||
|
expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError,
|
||||||
|
KeyFeeAboveMaxAllowedError)
|
||||||
|
if isinstance(err, expected):
|
||||||
|
log.warning("Failed to download %s: %s", uri, str(err))
|
||||||
|
elif isinstance(err, asyncio.CancelledError):
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
log.exception("Unexpected error downloading stream:")
|
||||||
|
error = err
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
if payment is not None:
|
||||||
|
# payment is set to None after broadcasting, if we're here an exception probably happened
|
||||||
|
await self.wallet_manager.ledger.release_tx(payment)
|
||||||
|
if self.analytics_manager and claim and claim.stream.source.bt_infohash:
|
||||||
|
# TODO: analytics for torrents
|
||||||
|
pass
|
||||||
|
elif self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or
|
||||||
|
stream.downloader.time_to_first_bytes))):
|
||||||
|
server = self.wallet_manager.ledger.network.client.server
|
||||||
|
self.loop.create_task(
|
||||||
|
self.analytics_manager.send_time_to_first_bytes(
|
||||||
|
resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id,
|
||||||
|
uri, outpoint,
|
||||||
|
None if not stream else len(stream.downloader.blob_downloader.active_connections),
|
||||||
|
None if not stream else len(stream.downloader.blob_downloader.scores),
|
||||||
|
None if not stream else len(stream.downloader.blob_downloader.connection_failures),
|
||||||
|
False if not stream else stream.downloader.added_fixed_peers,
|
||||||
|
self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay,
|
||||||
|
None if not stream else stream.sd_hash,
|
||||||
|
None if not stream else stream.downloader.time_to_descriptor,
|
||||||
|
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
|
||||||
|
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
|
||||||
|
None if not stream else stream.downloader.time_to_first_bytes,
|
||||||
|
None if not error else error.__class__.__name__,
|
||||||
|
None if not error else str(error),
|
||||||
|
None if not server else f"{server[0]}:{server[1]}"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
async def stream_partial_content(self, request: Request, sd_hash: str):
|
||||||
|
return await self.source_managers['stream'].stream_partial_content(request, sd_hash)
|
||||||
|
|
||||||
|
def get_filtered(self, *args, **kwargs) -> typing.List[ManagedDownloadSource]:
|
||||||
|
"""
|
||||||
|
Get a list of filtered and sorted ManagedStream objects
|
||||||
|
|
||||||
|
:param sort_by: field to sort by
|
||||||
|
:param reverse: reverse sorting
|
||||||
|
:param comparison: comparison operator used for filtering
|
||||||
|
:param search_by: fields and values to filter by
|
||||||
|
"""
|
||||||
|
return sum((manager.get_filtered(*args, **kwargs) for manager in self.source_managers.values()), [])
|
||||||
|
|
||||||
|
async def delete(self, source: ManagedDownloadSource, delete_file=False):
|
||||||
|
for manager in self.source_managers.values():
|
||||||
|
await manager.delete(source, delete_file)
|
||||||
@@ -0,0 +1,161 @@
|
|||||||
|
import os
|
||||||
|
import asyncio
|
||||||
|
import typing
|
||||||
|
import logging
|
||||||
|
import binascii
|
||||||
|
from typing import Optional
|
||||||
|
from lbry.utils import generate_id
|
||||||
|
from lbry.extras.daemon.storage import StoredContentClaim
|
||||||
|
|
||||||
|
if typing.TYPE_CHECKING:
|
||||||
|
from lbry.conf import Config
|
||||||
|
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||||
|
from lbry.wallet.transaction import Transaction
|
||||||
|
from lbry.extras.daemon.storage import SQLiteStorage
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class ManagedDownloadSource:
|
||||||
|
STATUS_RUNNING = "running"
|
||||||
|
STATUS_STOPPED = "stopped"
|
||||||
|
STATUS_FINISHED = "finished"
|
||||||
|
|
||||||
|
SAVING_ID = 1
|
||||||
|
STREAMING_ID = 2
|
||||||
|
|
||||||
|
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage', identifier: str,
|
||||||
|
file_name: Optional[str] = None, download_directory: Optional[str] = None,
|
||||||
|
status: Optional[str] = STATUS_STOPPED, claim: Optional[StoredContentClaim] = None,
|
||||||
|
download_id: Optional[str] = None, rowid: Optional[int] = None,
|
||||||
|
content_fee: Optional['Transaction'] = None,
|
||||||
|
analytics_manager: Optional['AnalyticsManager'] = None,
|
||||||
|
added_on: Optional[int] = None):
|
||||||
|
self.loop = loop
|
||||||
|
self.storage = storage
|
||||||
|
self.config = config
|
||||||
|
self.identifier = identifier
|
||||||
|
self.download_directory = download_directory
|
||||||
|
self._file_name = file_name
|
||||||
|
self._status = status
|
||||||
|
self.stream_claim_info = claim
|
||||||
|
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
|
||||||
|
self.rowid = rowid
|
||||||
|
self.content_fee = content_fee
|
||||||
|
self.purchase_receipt = None
|
||||||
|
self._added_on = added_on
|
||||||
|
self.analytics_manager = analytics_manager
|
||||||
|
|
||||||
|
self.saving = asyncio.Event(loop=self.loop)
|
||||||
|
self.finished_writing = asyncio.Event(loop=self.loop)
|
||||||
|
self.started_writing = asyncio.Event(loop=self.loop)
|
||||||
|
self.finished_write_attempt = asyncio.Event(loop=self.loop)
|
||||||
|
|
||||||
|
# @classmethod
|
||||||
|
# async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', file_path: str,
|
||||||
|
# key: Optional[bytes] = None,
|
||||||
|
# iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedDownloadSource':
|
||||||
|
# raise NotImplementedError()
|
||||||
|
|
||||||
|
async def start(self, timeout: Optional[float] = None, save_now: Optional[bool] = False):
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
async def stop(self, finished: bool = False):
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
def stop_tasks(self):
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
def set_claim(self, claim_info: typing.Dict, claim: 'Claim'):
|
||||||
|
self.stream_claim_info = StoredContentClaim(
|
||||||
|
f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
|
||||||
|
claim_info['name'], claim_info['amount'], claim_info['height'],
|
||||||
|
binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'],
|
||||||
|
claim_info['claim_sequence'], claim_info.get('channel_name')
|
||||||
|
)
|
||||||
|
|
||||||
|
# async def update_content_claim(self, claim_info: Optional[typing.Dict] = None):
|
||||||
|
# if not claim_info:
|
||||||
|
# claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash)
|
||||||
|
# self.set_claim(claim_info, claim_info['value'])
|
||||||
|
|
||||||
|
@property
|
||||||
|
def file_name(self) -> Optional[str]:
|
||||||
|
return self._file_name
|
||||||
|
|
||||||
|
@property
|
||||||
|
def added_on(self) -> Optional[int]:
|
||||||
|
return self._added_on
|
||||||
|
|
||||||
|
@property
|
||||||
|
def status(self) -> str:
|
||||||
|
return self._status
|
||||||
|
|
||||||
|
@property
|
||||||
|
def completed(self):
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
# @property
|
||||||
|
# def stream_url(self):
|
||||||
|
# return f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash}
|
||||||
|
|
||||||
|
@property
|
||||||
|
def finished(self) -> bool:
|
||||||
|
return self.status == self.STATUS_FINISHED
|
||||||
|
|
||||||
|
@property
|
||||||
|
def running(self) -> bool:
|
||||||
|
return self.status == self.STATUS_RUNNING
|
||||||
|
|
||||||
|
@property
|
||||||
|
def claim_id(self) -> Optional[str]:
|
||||||
|
return None if not self.stream_claim_info else self.stream_claim_info.claim_id
|
||||||
|
|
||||||
|
@property
|
||||||
|
def txid(self) -> Optional[str]:
|
||||||
|
return None if not self.stream_claim_info else self.stream_claim_info.txid
|
||||||
|
|
||||||
|
@property
|
||||||
|
def nout(self) -> Optional[int]:
|
||||||
|
return None if not self.stream_claim_info else self.stream_claim_info.nout
|
||||||
|
|
||||||
|
@property
|
||||||
|
def outpoint(self) -> Optional[str]:
|
||||||
|
return None if not self.stream_claim_info else self.stream_claim_info.outpoint
|
||||||
|
|
||||||
|
@property
|
||||||
|
def claim_height(self) -> Optional[int]:
|
||||||
|
return None if not self.stream_claim_info else self.stream_claim_info.height
|
||||||
|
|
||||||
|
@property
|
||||||
|
def channel_claim_id(self) -> Optional[str]:
|
||||||
|
return None if not self.stream_claim_info else self.stream_claim_info.channel_claim_id
|
||||||
|
|
||||||
|
@property
|
||||||
|
def channel_name(self) -> Optional[str]:
|
||||||
|
return None if not self.stream_claim_info else self.stream_claim_info.channel_name
|
||||||
|
|
||||||
|
@property
|
||||||
|
def claim_name(self) -> Optional[str]:
|
||||||
|
return None if not self.stream_claim_info else self.stream_claim_info.claim_name
|
||||||
|
|
||||||
|
@property
|
||||||
|
def metadata(self) -> Optional[typing.Dict]:
|
||||||
|
return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict()
|
||||||
|
|
||||||
|
@property
|
||||||
|
def metadata_protobuf(self) -> bytes:
|
||||||
|
if self.stream_claim_info:
|
||||||
|
return binascii.hexlify(self.stream_claim_info.claim.to_bytes())
|
||||||
|
|
||||||
|
@property
|
||||||
|
def full_path(self) -> Optional[str]:
|
||||||
|
return os.path.join(self.download_directory, os.path.basename(self.file_name)) \
|
||||||
|
if self.file_name and self.download_directory else None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def output_file_exists(self):
|
||||||
|
return os.path.isfile(self.full_path) if self.full_path else False
|
||||||
@@ -0,0 +1,138 @@
|
|||||||
|
import os
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
import typing
|
||||||
|
from typing import Optional
|
||||||
|
from lbry.file.source import ManagedDownloadSource
|
||||||
|
if typing.TYPE_CHECKING:
|
||||||
|
from lbry.conf import Config
|
||||||
|
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||||
|
from lbry.extras.daemon.storage import SQLiteStorage
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
COMPARISON_OPERATORS = {
|
||||||
|
'eq': lambda a, b: a == b,
|
||||||
|
'ne': lambda a, b: a != b,
|
||||||
|
'g': lambda a, b: a > b,
|
||||||
|
'l': lambda a, b: a < b,
|
||||||
|
'ge': lambda a, b: a >= b,
|
||||||
|
'le': lambda a, b: a <= b,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class SourceManager:
|
||||||
|
filter_fields = {
|
||||||
|
'rowid',
|
||||||
|
'status',
|
||||||
|
'file_name',
|
||||||
|
'added_on',
|
||||||
|
'download_path',
|
||||||
|
'claim_name',
|
||||||
|
'claim_height',
|
||||||
|
'claim_id',
|
||||||
|
'outpoint',
|
||||||
|
'txid',
|
||||||
|
'nout',
|
||||||
|
'channel_claim_id',
|
||||||
|
'channel_name',
|
||||||
|
'completed'
|
||||||
|
}
|
||||||
|
|
||||||
|
set_filter_fields = {
|
||||||
|
"claim_ids": "claim_id",
|
||||||
|
"channel_claim_ids": "channel_claim_id",
|
||||||
|
"outpoints": "outpoint"
|
||||||
|
}
|
||||||
|
|
||||||
|
source_class = ManagedDownloadSource
|
||||||
|
|
||||||
|
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage',
|
||||||
|
analytics_manager: Optional['AnalyticsManager'] = None):
|
||||||
|
self.loop = loop
|
||||||
|
self.config = config
|
||||||
|
self.storage = storage
|
||||||
|
self.analytics_manager = analytics_manager
|
||||||
|
self._sources: typing.Dict[str, ManagedDownloadSource] = {}
|
||||||
|
self.started = asyncio.Event(loop=self.loop)
|
||||||
|
|
||||||
|
def add(self, source: ManagedDownloadSource):
|
||||||
|
self._sources[source.identifier] = source
|
||||||
|
|
||||||
|
def remove(self, source: ManagedDownloadSource):
|
||||||
|
if source.identifier not in self._sources:
|
||||||
|
return
|
||||||
|
self._sources.pop(source.identifier)
|
||||||
|
source.stop_tasks()
|
||||||
|
|
||||||
|
async def initialize_from_database(self):
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
async def start(self):
|
||||||
|
await self.initialize_from_database()
|
||||||
|
self.started.set()
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
while self._sources:
|
||||||
|
_, source = self._sources.popitem()
|
||||||
|
source.stop_tasks()
|
||||||
|
self.started.clear()
|
||||||
|
|
||||||
|
async def create(self, file_path: str, key: Optional[bytes] = None,
|
||||||
|
iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedDownloadSource:
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
|
||||||
|
self.remove(source)
|
||||||
|
if delete_file and source.output_file_exists:
|
||||||
|
os.remove(source.full_path)
|
||||||
|
|
||||||
|
def get_filtered(self, sort_by: Optional[str] = None, reverse: Optional[bool] = False,
|
||||||
|
comparison: Optional[str] = None, **search_by) -> typing.List[ManagedDownloadSource]:
|
||||||
|
"""
|
||||||
|
Get a list of filtered and sorted ManagedStream objects
|
||||||
|
|
||||||
|
:param sort_by: field to sort by
|
||||||
|
:param reverse: reverse sorting
|
||||||
|
:param comparison: comparison operator used for filtering
|
||||||
|
:param search_by: fields and values to filter by
|
||||||
|
"""
|
||||||
|
if sort_by and sort_by not in self.filter_fields:
|
||||||
|
raise ValueError(f"'{sort_by}' is not a valid field to sort by")
|
||||||
|
if comparison and comparison not in COMPARISON_OPERATORS:
|
||||||
|
raise ValueError(f"'{comparison}' is not a valid comparison")
|
||||||
|
if 'full_status' in search_by:
|
||||||
|
del search_by['full_status']
|
||||||
|
|
||||||
|
for search in search_by:
|
||||||
|
if search not in self.filter_fields:
|
||||||
|
raise ValueError(f"'{search}' is not a valid search operation")
|
||||||
|
|
||||||
|
compare_sets = {}
|
||||||
|
if isinstance(search_by.get('claim_id'), list):
|
||||||
|
compare_sets['claim_ids'] = search_by.pop('claim_id')
|
||||||
|
if isinstance(search_by.get('outpoint'), list):
|
||||||
|
compare_sets['outpoints'] = search_by.pop('outpoint')
|
||||||
|
if isinstance(search_by.get('channel_claim_id'), list):
|
||||||
|
compare_sets['channel_claim_ids'] = search_by.pop('channel_claim_id')
|
||||||
|
|
||||||
|
if search_by or compare_sets:
|
||||||
|
comparison = comparison or 'eq'
|
||||||
|
streams = []
|
||||||
|
for stream in self._sources.values():
|
||||||
|
if compare_sets and not all(
|
||||||
|
getattr(stream, self.set_filter_fields[set_search]) in val
|
||||||
|
for set_search, val in compare_sets.items()):
|
||||||
|
continue
|
||||||
|
if search_by and not all(
|
||||||
|
COMPARISON_OPERATORS[comparison](getattr(stream, search), val)
|
||||||
|
for search, val in search_by.items()):
|
||||||
|
continue
|
||||||
|
streams.append(stream)
|
||||||
|
else:
|
||||||
|
streams = list(self._sources.values())
|
||||||
|
if sort_by:
|
||||||
|
streams.sort(key=lambda s: getattr(s, sort_by) or "")
|
||||||
|
if reverse:
|
||||||
|
streams.reverse()
|
||||||
|
return streams
|
||||||
+188
-89
@@ -3,10 +3,14 @@ import json
|
|||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import pathlib
|
import pathlib
|
||||||
|
import platform
|
||||||
import re
|
import re
|
||||||
import shlex
|
import shlex
|
||||||
import shutil
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
from math import ceil
|
||||||
|
|
||||||
|
import lbry.utils
|
||||||
from lbry.conf import TranscodeConfig
|
from lbry.conf import TranscodeConfig
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
@@ -14,58 +18,104 @@ log = logging.getLogger(__name__)
|
|||||||
|
|
||||||
class VideoFileAnalyzer:
|
class VideoFileAnalyzer:
|
||||||
|
|
||||||
|
def _replace_or_pop_env(self, variable):
|
||||||
|
if variable + '_ORIG' in self._env_copy:
|
||||||
|
self._env_copy[variable] = self._env_copy[variable + '_ORIG']
|
||||||
|
else:
|
||||||
|
self._env_copy.pop(variable, None)
|
||||||
|
|
||||||
def __init__(self, conf: TranscodeConfig):
|
def __init__(self, conf: TranscodeConfig):
|
||||||
self._conf = conf
|
self._conf = conf
|
||||||
self._available_encoders = ""
|
self._available_encoders = ""
|
||||||
self._ffmpeg_installed = False
|
self._ffmpeg_installed = None
|
||||||
self._which = None
|
self._which_ffmpeg = None
|
||||||
|
self._which_ffprobe = None
|
||||||
|
self._env_copy = dict(os.environ)
|
||||||
self._checked_ffmpeg = False
|
self._checked_ffmpeg = False
|
||||||
|
if lbry.utils.is_running_from_bundle():
|
||||||
|
# handle the situation where PyInstaller overrides our runtime environment:
|
||||||
|
self._replace_or_pop_env('LD_LIBRARY_PATH')
|
||||||
|
|
||||||
async def _execute(self, command, arguments):
|
@staticmethod
|
||||||
args = shlex.split(arguments)
|
def _execute(command, environment):
|
||||||
process = await asyncio.create_subprocess_exec(self._conf.ffmpeg_folder + command, *args,
|
# log.debug("Executing: %s", command)
|
||||||
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
|
|
||||||
stdout, stderr = await process.communicate() # returns when the streams are closed
|
|
||||||
return stdout.decode(errors='replace') + stderr.decode(errors='replace'), process.returncode
|
|
||||||
|
|
||||||
async def _verify_executable(self, name):
|
|
||||||
try:
|
try:
|
||||||
version, code = await self._execute(name, "-version")
|
with subprocess.Popen(
|
||||||
|
shlex.split(command) if platform.system() != 'Windows' else command,
|
||||||
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=environment
|
||||||
|
) as process:
|
||||||
|
(stdout, stderr) = process.communicate() # blocks until the process exits
|
||||||
|
return stdout.decode(errors='replace') + stderr.decode(errors='replace'), process.returncode
|
||||||
|
except subprocess.SubprocessError as e:
|
||||||
|
return str(e), -1
|
||||||
|
|
||||||
|
# This create_subprocess_exec call is broken in Windows Python 3.7, but it's prettier than what's here.
|
||||||
|
# The recommended fix is switching to ProactorEventLoop, but that breaks UDP in Linux Python 3.7.
|
||||||
|
# We work around that issue here by using run_in_executor. Check it again in Python 3.8.
|
||||||
|
async def _execute_ffmpeg(self, arguments):
|
||||||
|
arguments = self._which_ffmpeg + " " + arguments
|
||||||
|
return await asyncio.get_event_loop().run_in_executor(None, self._execute, arguments, self._env_copy)
|
||||||
|
|
||||||
|
async def _execute_ffprobe(self, arguments):
|
||||||
|
arguments = self._which_ffprobe + " " + arguments
|
||||||
|
return await asyncio.get_event_loop().run_in_executor(None, self._execute, arguments, self._env_copy)
|
||||||
|
|
||||||
|
async def _verify_executables(self):
|
||||||
|
try:
|
||||||
|
await self._execute_ffprobe("-version")
|
||||||
|
version, code = await self._execute_ffmpeg("-version")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.warning("Unable to run %s, but it was requested. Message: %s", name, str(e))
|
|
||||||
code = -1
|
code = -1
|
||||||
version = ""
|
version = str(e)
|
||||||
if code != 0 or not version.startswith(name):
|
if code != 0 or not version.startswith("ffmpeg"):
|
||||||
raise FileNotFoundError(f"Unable to locate or run {name}. Please install FFmpeg "
|
log.warning("Unable to run ffmpeg, but it was requested. Code: %d; Message: %s", code, version)
|
||||||
f"and ensure that it is callable via PATH or conf.ffmpeg_folder")
|
raise FileNotFoundError("Unable to locate or run ffmpeg or ffprobe. Please install FFmpeg "
|
||||||
|
"and ensure that it is callable via PATH or conf.ffmpeg_path")
|
||||||
|
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which_ffmpeg)
|
||||||
return version
|
return version
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _which_ffmpeg_and_ffmprobe(path):
|
||||||
|
return shutil.which("ffmpeg", path=path), shutil.which("ffprobe", path=path)
|
||||||
|
|
||||||
async def _verify_ffmpeg_installed(self):
|
async def _verify_ffmpeg_installed(self):
|
||||||
if self._ffmpeg_installed:
|
if self._ffmpeg_installed:
|
||||||
return
|
return
|
||||||
await self._verify_executable("ffprobe")
|
self._ffmpeg_installed = False
|
||||||
version = await self._verify_executable("ffmpeg")
|
path = self._conf.ffmpeg_path
|
||||||
self._which = shutil.which(f"{self._conf.ffmpeg_folder}ffmpeg")
|
if hasattr(self._conf, "data_dir"):
|
||||||
|
path += os.path.pathsep + os.path.join(getattr(self._conf, "data_dir"), "ffmpeg", "bin")
|
||||||
|
path += os.path.pathsep + self._env_copy.get("PATH", "")
|
||||||
|
self._which_ffmpeg, self._which_ffprobe = await asyncio.get_running_loop().run_in_executor(
|
||||||
|
None, self._which_ffmpeg_and_ffmprobe, path
|
||||||
|
)
|
||||||
|
if not self._which_ffmpeg:
|
||||||
|
log.warning("Unable to locate ffmpeg executable. Path: %s", path)
|
||||||
|
raise FileNotFoundError(f"Unable to locate ffmpeg executable. Path: {path}")
|
||||||
|
if not self._which_ffprobe:
|
||||||
|
log.warning("Unable to locate ffprobe executable. Path: %s", path)
|
||||||
|
raise FileNotFoundError(f"Unable to locate ffprobe executable. Path: {path}")
|
||||||
|
if os.path.dirname(self._which_ffmpeg) != os.path.dirname(self._which_ffprobe):
|
||||||
|
log.warning("ffmpeg and ffprobe are in different folders!")
|
||||||
|
|
||||||
|
await self._verify_executables()
|
||||||
self._ffmpeg_installed = True
|
self._ffmpeg_installed = True
|
||||||
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which)
|
|
||||||
|
|
||||||
async def status(self, reset=False, recheck=False):
|
async def status(self, reset=False, recheck=False):
|
||||||
if reset:
|
if reset:
|
||||||
self._available_encoders = ""
|
self._available_encoders = ""
|
||||||
self._ffmpeg_installed = False
|
self._ffmpeg_installed = None
|
||||||
self._which = None
|
|
||||||
if self._checked_ffmpeg and not recheck:
|
if self._checked_ffmpeg and not recheck:
|
||||||
installed = self._ffmpeg_installed
|
pass
|
||||||
else:
|
elif self._ffmpeg_installed is None:
|
||||||
installed = True
|
|
||||||
try:
|
try:
|
||||||
await self._verify_ffmpeg_installed()
|
await self._verify_ffmpeg_installed()
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
installed = False
|
pass
|
||||||
self._checked_ffmpeg = True
|
self._checked_ffmpeg = True
|
||||||
return {
|
return {
|
||||||
"available": installed,
|
"available": self._ffmpeg_installed,
|
||||||
"which": self._which,
|
"which": self._which_ffmpeg,
|
||||||
"analyze_audio_volume": int(self._conf.volume_analysis_time) > 0
|
"analyze_audio_volume": int(self._conf.volume_analysis_time) > 0
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -73,9 +123,24 @@ class VideoFileAnalyzer:
|
|||||||
def _verify_container(scan_data: json):
|
def _verify_container(scan_data: json):
|
||||||
container = scan_data["format"]["format_name"]
|
container = scan_data["format"]["format_name"]
|
||||||
log.debug(" Detected container is %s", container)
|
log.debug(" Detected container is %s", container)
|
||||||
if not {"webm", "mp4", "3gp", "ogg"}.intersection(container.split(",")):
|
splits = container.split(",")
|
||||||
|
if not {"webm", "mp4", "3gp", "ogg"}.intersection(splits):
|
||||||
return "Container format is not in the approved list of WebM, MP4. " \
|
return "Container format is not in the approved list of WebM, MP4. " \
|
||||||
f"Actual: {container} [{scan_data['format']['format_long_name']}]"
|
f"Actual: {container} [{scan_data['format']['format_long_name']}]"
|
||||||
|
|
||||||
|
if "matroska" in splits:
|
||||||
|
for stream in scan_data["streams"]:
|
||||||
|
if stream["codec_type"] == "video":
|
||||||
|
codec = stream["codec_name"]
|
||||||
|
if not {"vp8", "vp9", "av1"}.intersection(codec.split(",")):
|
||||||
|
return "WebM format requires VP8/9 or AV1 video. " \
|
||||||
|
f"Actual: {codec} [{stream['codec_long_name']}]"
|
||||||
|
elif stream["codec_type"] == "audio":
|
||||||
|
codec = stream["codec_name"]
|
||||||
|
if not {"vorbis", "opus"}.intersection(codec.split(",")):
|
||||||
|
return "WebM format requires Vorbis or Opus audio. " \
|
||||||
|
f"Actual: {codec} [{stream['codec_long_name']}]"
|
||||||
|
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -95,24 +160,21 @@ class VideoFileAnalyzer:
|
|||||||
|
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
@staticmethod
|
def _verify_bitrate(self, scan_data: json, file_path):
|
||||||
def _verify_bitrate(scan_data: json):
|
bit_rate_max = float(self._conf.video_bitrate_maximum)
|
||||||
if "bit_rate" not in scan_data["format"]:
|
if bit_rate_max <= 0:
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
bit_rate = float(scan_data["format"]["bit_rate"])
|
if "bit_rate" in scan_data["format"]:
|
||||||
log.debug(" Detected bitrate is %s Mbps", str(bit_rate / 1000000.0))
|
bit_rate = float(scan_data["format"]["bit_rate"])
|
||||||
pixels = -1.0
|
else:
|
||||||
for stream in scan_data["streams"]:
|
bit_rate = os.stat(file_path).st_size / float(scan_data["format"]["duration"])
|
||||||
if stream["codec_type"] == "video":
|
log.debug(" Detected bitrate is %s Mbps. Allowed max: %s Mbps",
|
||||||
pieces = stream["r_frame_rate"].split('/', 1)
|
str(bit_rate / 1000000.0), str(bit_rate_max / 1000000.0))
|
||||||
frame_rate = float(pieces[0]) if len(pieces) == 1 \
|
|
||||||
else float(pieces[0]) / float(pieces[1])
|
|
||||||
pixels = max(pixels, float(stream["height"]) * float(stream["width"]) * frame_rate)
|
|
||||||
|
|
||||||
if pixels > 0.0 and pixels / bit_rate < 3.0:
|
if bit_rate > bit_rate_max:
|
||||||
return "Bits per second is excessive for this data; this may impact web streaming performance. " \
|
return "The bit rate is above the configured maximum. Actual: " \
|
||||||
f"Actual: {str(bit_rate / 1000000.0)} Mbps"
|
f"{bit_rate / 1000000.0} Mbps; Allowed max: {bit_rate_max / 1000000.0} Mbps"
|
||||||
|
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
@@ -121,7 +183,7 @@ class VideoFileAnalyzer:
|
|||||||
if {"webm", "ogg"}.intersection(container.split(",")):
|
if {"webm", "ogg"}.intersection(container.split(",")):
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
result, _ = await self._execute("ffprobe", f'-v debug "{video_file}"')
|
result, _ = await self._execute_ffprobe(f'-v debug "{video_file}"')
|
||||||
match = re.search(r"Before avformat_find_stream_info.+?\s+seeks:(\d+)\s+", result)
|
match = re.search(r"Before avformat_find_stream_info.+?\s+seeks:(\d+)\s+", result)
|
||||||
if match and int(match.group(1)) != 0:
|
if match and int(match.group(1)) != 0:
|
||||||
return "Video stream descriptors are not at the start of the file (the faststart flag was not used)."
|
return "Video stream descriptors are not at the start of the file (the faststart flag was not used)."
|
||||||
@@ -137,6 +199,8 @@ class VideoFileAnalyzer:
|
|||||||
if not {"aac", "mp3", "flac", "vorbis", "opus"}.intersection(codec.split(",")):
|
if not {"aac", "mp3", "flac", "vorbis", "opus"}.intersection(codec.split(",")):
|
||||||
return "Audio codec is not in the approved list of AAC, FLAC, MP3, Vorbis, and Opus. " \
|
return "Audio codec is not in the approved list of AAC, FLAC, MP3, Vorbis, and Opus. " \
|
||||||
f"Actual: {codec} [{stream['codec_long_name']}]"
|
f"Actual: {codec} [{stream['codec_long_name']}]"
|
||||||
|
if int(stream['sample_rate']) > 48000:
|
||||||
|
return "Sample rate out of range"
|
||||||
|
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
@@ -149,8 +213,8 @@ class VideoFileAnalyzer:
|
|||||||
if not validate_volume:
|
if not validate_volume:
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
result, _ = await self._execute("ffmpeg", f'-i "{video_file}" -t {seconds} '
|
result, _ = await self._execute_ffmpeg(f'-i "{video_file}" -t {seconds} '
|
||||||
f'-af volumedetect -vn -sn -dn -f null "{os.devnull}"')
|
f'-af volumedetect -vn -sn -dn -f null "{os.devnull}"')
|
||||||
try:
|
try:
|
||||||
mean_volume = float(re.search(r"mean_volume:\s+([-+]?\d*\.\d+|\d+)", result).group(1))
|
mean_volume = float(re.search(r"mean_volume:\s+([-+]?\d*\.\d+|\d+)", result).group(1))
|
||||||
max_volume = float(re.search(r"max_volume:\s+([-+]?\d*\.\d+|\d+)", result).group(1))
|
max_volume = float(re.search(r"max_volume:\s+([-+]?\d*\.\d+|\d+)", result).group(1))
|
||||||
@@ -176,13 +240,16 @@ class VideoFileAnalyzer:
|
|||||||
# https://developers.google.com/media/vp9/settings/vod/
|
# https://developers.google.com/media/vp9/settings/vod/
|
||||||
return int(-0.011 * height + 40)
|
return int(-0.011 * height + 40)
|
||||||
|
|
||||||
|
def _get_video_scaler(self):
|
||||||
|
return self._conf.video_scaler
|
||||||
|
|
||||||
async def _get_video_encoder(self, scan_data):
|
async def _get_video_encoder(self, scan_data):
|
||||||
# use what the user said if it's there:
|
# use what the user said if it's there:
|
||||||
# if it's not there, use h264 if we can because it's way faster than the others
|
# if it's not there, use h264 if we can because it's way faster than the others
|
||||||
# if we don't have h264 use vp9; it's fairly compatible even though it's slow
|
# if we don't have h264 use vp9; it's fairly compatible even though it's slow
|
||||||
|
|
||||||
if not self._available_encoders:
|
if not self._available_encoders:
|
||||||
self._available_encoders, _ = await self._execute("ffmpeg", "-encoders -v quiet")
|
self._available_encoders, _ = await self._execute_ffmpeg("-encoders -v quiet")
|
||||||
|
|
||||||
encoder = self._conf.video_encoder.split(" ", 1)[0]
|
encoder = self._conf.video_encoder.split(" ", 1)[0]
|
||||||
if re.search(fr"^\s*V..... {encoder} ", self._available_encoders, re.MULTILINE):
|
if re.search(fr"^\s*V..... {encoder} ", self._available_encoders, re.MULTILINE):
|
||||||
@@ -216,7 +283,7 @@ class VideoFileAnalyzer:
|
|||||||
|
|
||||||
wants_opus = extension != "mp4"
|
wants_opus = extension != "mp4"
|
||||||
if not self._available_encoders:
|
if not self._available_encoders:
|
||||||
self._available_encoders, _ = await self._execute("ffmpeg", "-encoders -v quiet")
|
self._available_encoders, _ = await self._execute_ffmpeg("-encoders -v quiet")
|
||||||
|
|
||||||
encoder = self._conf.audio_encoder.split(" ", 1)[0]
|
encoder = self._conf.audio_encoder.split(" ", 1)[0]
|
||||||
if wants_opus and 'opus' in encoder:
|
if wants_opus and 'opus' in encoder:
|
||||||
@@ -239,9 +306,6 @@ class VideoFileAnalyzer:
|
|||||||
|
|
||||||
raise Exception(f"The audio encoder is not available. Requested: {encoder or 'aac'}")
|
raise Exception(f"The audio encoder is not available. Requested: {encoder or 'aac'}")
|
||||||
|
|
||||||
async def _get_volume_filter(self):
|
|
||||||
return self._conf.volume_filter if self._conf.volume_filter else "-af loudnorm"
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _get_best_container_extension(scan_data, video_encoder):
|
def _get_best_container_extension(scan_data, video_encoder):
|
||||||
# the container is chosen by the video format
|
# the container is chosen by the video format
|
||||||
@@ -249,61 +313,93 @@ class VideoFileAnalyzer:
|
|||||||
# if we are vp8/vp9/av1 we want webm
|
# if we are vp8/vp9/av1 we want webm
|
||||||
# use mp4 for anything else
|
# use mp4 for anything else
|
||||||
|
|
||||||
if not video_encoder: # not re-encoding video
|
if video_encoder: # not re-encoding video
|
||||||
for stream in scan_data["streams"]:
|
if "theora" in video_encoder:
|
||||||
if stream["codec_type"] != "video":
|
return "ogv"
|
||||||
continue
|
if re.search(r"vp[89x]|av1", video_encoder.split(" ", 1)[0]):
|
||||||
codec = stream["codec_name"].split(",")
|
return "webm"
|
||||||
if "theora" in codec:
|
return "mp4"
|
||||||
return "ogg"
|
|
||||||
if {"vp8", "vp9", "av1"}.intersection(codec):
|
for stream in scan_data["streams"]:
|
||||||
return "webm"
|
if stream["codec_type"] != "video":
|
||||||
|
continue
|
||||||
|
codec = stream["codec_name"].split(",")
|
||||||
|
if "theora" in codec:
|
||||||
|
return "ogv"
|
||||||
|
if {"vp8", "vp9", "av1"}.intersection(codec):
|
||||||
|
return "webm"
|
||||||
|
|
||||||
if "theora" in video_encoder:
|
|
||||||
return "ogg"
|
|
||||||
elif re.search(r"vp[89x]|av1", video_encoder.split(" ", 1)[0]):
|
|
||||||
return "webm"
|
|
||||||
return "mp4"
|
return "mp4"
|
||||||
|
|
||||||
async def _get_scan_data(self, validate, file_path):
|
async def _get_scan_data(self, validate, file_path):
|
||||||
result, _ = await self._execute("ffprobe",
|
arguments = f'-v quiet -print_format json -show_format -show_streams "{file_path}"'
|
||||||
f'-v quiet -print_format json -show_format -show_streams "{file_path}"')
|
result, _ = await self._execute_ffprobe(arguments)
|
||||||
try:
|
try:
|
||||||
scan_data = json.loads(result)
|
scan_data = json.loads(result)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.debug("Failure in JSON parsing ffprobe results. Message: %s", str(e))
|
log.debug("Failure in JSON parsing ffprobe results. Message: %s", str(e))
|
||||||
if validate:
|
raise ValueError(f'Absent or unreadable video file: {file_path}')
|
||||||
raise Exception(f'Invalid video file: {file_path}')
|
|
||||||
log.info("Unable to optimize %s . FFmpeg output was unreadable.", file_path)
|
|
||||||
return
|
|
||||||
|
|
||||||
if "format" not in scan_data:
|
if "format" not in scan_data or "duration" not in scan_data["format"]:
|
||||||
if validate:
|
log.debug("Format data is missing from ffprobe results for: %s", file_path)
|
||||||
raise FileNotFoundError(f'Unexpected or absent video file contents at: {file_path}')
|
raise ValueError(f'Media file does not appear to contain video content: {file_path}')
|
||||||
log.info("Unable to optimize %s . FFmpeg output is missing the format section.", file_path)
|
|
||||||
return
|
if float(scan_data["format"]["duration"]) < 0.1:
|
||||||
|
log.debug("Media file appears to be an image: %s", file_path)
|
||||||
|
raise ValueError(f'Assuming image file at: {file_path}')
|
||||||
|
|
||||||
return scan_data
|
return scan_data
|
||||||
|
|
||||||
async def verify_or_repair(self, validate, repair, file_path):
|
@staticmethod
|
||||||
|
def _build_spec(scan_data):
|
||||||
|
assert scan_data
|
||||||
|
|
||||||
|
duration = ceil(float(scan_data["format"]["duration"])) # existence verified when scan_data made
|
||||||
|
width = -1
|
||||||
|
height = -1
|
||||||
|
for stream in scan_data["streams"]:
|
||||||
|
if stream["codec_type"] != "video":
|
||||||
|
continue
|
||||||
|
width = max(width, int(stream["width"]))
|
||||||
|
height = max(height, int(stream["height"]))
|
||||||
|
|
||||||
|
log.debug(" Detected duration: %d sec. with resolution: %d x %d", duration, width, height)
|
||||||
|
|
||||||
|
spec = {"duration": duration}
|
||||||
|
if height >= 0:
|
||||||
|
spec["height"] = height
|
||||||
|
if width >= 0:
|
||||||
|
spec["width"] = width
|
||||||
|
return spec
|
||||||
|
|
||||||
|
async def verify_or_repair(self, validate, repair, file_path, ignore_non_video=False):
|
||||||
if not validate and not repair:
|
if not validate and not repair:
|
||||||
return file_path
|
return file_path, {}
|
||||||
|
|
||||||
|
if ignore_non_video and not file_path:
|
||||||
|
return file_path, {}
|
||||||
|
|
||||||
await self._verify_ffmpeg_installed()
|
await self._verify_ffmpeg_installed()
|
||||||
scan_data = await self._get_scan_data(validate, file_path)
|
try:
|
||||||
|
scan_data = await self._get_scan_data(validate, file_path)
|
||||||
|
except ValueError:
|
||||||
|
if ignore_non_video:
|
||||||
|
return file_path, {}
|
||||||
|
raise
|
||||||
|
|
||||||
fast_start_msg = await self._verify_fast_start(scan_data, file_path)
|
fast_start_msg = await self._verify_fast_start(scan_data, file_path)
|
||||||
log.debug("Analyzing %s:", file_path)
|
log.debug("Analyzing %s:", file_path)
|
||||||
|
spec = self._build_spec(scan_data)
|
||||||
log.debug(" Detected faststart is %s", "false" if fast_start_msg else "true")
|
log.debug(" Detected faststart is %s", "false" if fast_start_msg else "true")
|
||||||
container_msg = self._verify_container(scan_data)
|
container_msg = self._verify_container(scan_data)
|
||||||
bitrate_msg = self._verify_bitrate(scan_data)
|
bitrate_msg = self._verify_bitrate(scan_data, file_path)
|
||||||
video_msg = self._verify_video_encoding(scan_data)
|
video_msg = self._verify_video_encoding(scan_data)
|
||||||
audio_msg = self._verify_audio_encoding(scan_data)
|
audio_msg = self._verify_audio_encoding(scan_data)
|
||||||
volume_msg = await self._verify_audio_volume(self._conf.volume_analysis_time, file_path)
|
volume_msg = await self._verify_audio_volume(self._conf.volume_analysis_time, file_path)
|
||||||
messages = [container_msg, bitrate_msg, fast_start_msg, video_msg, audio_msg, volume_msg]
|
messages = [container_msg, bitrate_msg, fast_start_msg, video_msg, audio_msg, volume_msg]
|
||||||
|
|
||||||
if not any(messages):
|
if not any(messages):
|
||||||
return file_path
|
return file_path, spec
|
||||||
|
|
||||||
if not repair:
|
if not repair:
|
||||||
errors = ["Streamability verification failed:"]
|
errors = ["Streamability verification failed:"]
|
||||||
@@ -313,7 +409,7 @@ class VideoFileAnalyzer:
|
|||||||
# the plan for transcoding:
|
# the plan for transcoding:
|
||||||
# we have to re-encode the video if it is in a nonstandard format
|
# we have to re-encode the video if it is in a nonstandard format
|
||||||
# we also re-encode if we are h264 but not yuv420p (both errors caught in video_msg)
|
# we also re-encode if we are h264 but not yuv420p (both errors caught in video_msg)
|
||||||
# we also re-encode if our bitrate is too high
|
# we also re-encode if our bitrate or sample rate is too high
|
||||||
|
|
||||||
try:
|
try:
|
||||||
transcode_command = [f'-i "{file_path}" -y -c:s copy -c:d copy -c:v']
|
transcode_command = [f'-i "{file_path}" -y -c:s copy -c:d copy -c:v']
|
||||||
@@ -322,29 +418,32 @@ class VideoFileAnalyzer:
|
|||||||
if video_msg or bitrate_msg:
|
if video_msg or bitrate_msg:
|
||||||
video_encoder = await self._get_video_encoder(scan_data)
|
video_encoder = await self._get_video_encoder(scan_data)
|
||||||
transcode_command.append(video_encoder)
|
transcode_command.append(video_encoder)
|
||||||
|
# could do the scaling only if bitrate_msg, but if we're going to the effort to re-encode anyway...
|
||||||
|
transcode_command.append(self._get_video_scaler())
|
||||||
else:
|
else:
|
||||||
transcode_command.append("copy")
|
transcode_command.append("copy")
|
||||||
|
|
||||||
transcode_command.append("-movflags +faststart -c:a")
|
transcode_command.append("-movflags +faststart -c:a")
|
||||||
path = pathlib.Path(file_path)
|
|
||||||
extension = self._get_best_container_extension(scan_data, video_encoder)
|
extension = self._get_best_container_extension(scan_data, video_encoder)
|
||||||
|
|
||||||
if audio_msg or volume_msg:
|
if audio_msg or volume_msg:
|
||||||
audio_encoder = await self._get_audio_encoder(extension)
|
audio_encoder = await self._get_audio_encoder(extension)
|
||||||
transcode_command.append(audio_encoder)
|
transcode_command.append(audio_encoder)
|
||||||
if volume_msg:
|
if volume_msg and self._conf.volume_filter:
|
||||||
volume_filter = await self._get_volume_filter()
|
transcode_command.append(self._conf.volume_filter)
|
||||||
transcode_command.append(volume_filter)
|
if audio_msg == "Sample rate out of range":
|
||||||
|
transcode_command.append(" -ar 48000 ")
|
||||||
else:
|
else:
|
||||||
transcode_command.append("copy")
|
transcode_command.append("copy")
|
||||||
|
|
||||||
# TODO: put it in a temp folder and delete it after we upload?
|
# TODO: put it in a temp folder and delete it after we upload?
|
||||||
|
path = pathlib.Path(file_path)
|
||||||
output = path.parent / f"{path.stem}_fixed.{extension}"
|
output = path.parent / f"{path.stem}_fixed.{extension}"
|
||||||
transcode_command.append(f'"{output}"')
|
transcode_command.append(f'"{output}"')
|
||||||
|
|
||||||
ffmpeg_command = " ".join(transcode_command)
|
ffmpeg_command = " ".join(transcode_command)
|
||||||
log.info("Proceeding on transcode via: ffmpeg %s", ffmpeg_command)
|
log.info("Proceeding on transcode via: ffmpeg %s", ffmpeg_command)
|
||||||
result, code = await self._execute("ffmpeg", ffmpeg_command)
|
result, code = await self._execute_ffmpeg(ffmpeg_command)
|
||||||
if code != 0:
|
if code != 0:
|
||||||
raise Exception(f"Failure to complete the transcode command. Output: {result}")
|
raise Exception(f"Failure to complete the transcode command. Output: {result}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -352,6 +451,6 @@ class VideoFileAnalyzer:
|
|||||||
raise
|
raise
|
||||||
log.info("Unable to transcode %s . Message: %s", file_path, str(e))
|
log.info("Unable to transcode %s . Message: %s", file_path, str(e))
|
||||||
# TODO: delete partial output file here if it exists?
|
# TODO: delete partial output file here if it exists?
|
||||||
return file_path
|
return file_path, spec
|
||||||
|
|
||||||
return str(output)
|
return str(output), spec
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
import time
|
||||||
|
import logging
|
||||||
|
import asyncio
|
||||||
|
import asyncio.tasks
|
||||||
|
from aiohttp import web
|
||||||
|
from prometheus_client import generate_latest as prom_generate_latest
|
||||||
|
from prometheus_client import Counter, Histogram, Gauge
|
||||||
|
|
||||||
|
|
||||||
|
PROBES_IN_FLIGHT = Counter("probes_in_flight", "Number of loop probes in flight", namespace='asyncio')
|
||||||
|
PROBES_FINISHED = Counter("probes_finished", "Number of finished loop probes", namespace='asyncio')
|
||||||
|
PROBE_TIMES = Histogram("probe_times", "Loop probe times", namespace='asyncio')
|
||||||
|
TASK_COUNT = Gauge("running_tasks", "Number of running tasks", namespace='asyncio')
|
||||||
|
|
||||||
|
|
||||||
|
def get_loop_metrics(delay=1):
|
||||||
|
loop = asyncio.get_event_loop()
|
||||||
|
|
||||||
|
def callback(started):
|
||||||
|
PROBE_TIMES.observe(time.perf_counter() - started - delay)
|
||||||
|
PROBES_FINISHED.inc()
|
||||||
|
|
||||||
|
async def monitor_loop_responsiveness():
|
||||||
|
while True:
|
||||||
|
now = time.perf_counter()
|
||||||
|
loop.call_later(delay, callback, now)
|
||||||
|
PROBES_IN_FLIGHT.inc()
|
||||||
|
TASK_COUNT.set(len(asyncio.tasks._all_tasks))
|
||||||
|
await asyncio.sleep(delay)
|
||||||
|
|
||||||
|
return loop.create_task(monitor_loop_responsiveness())
|
||||||
|
|
||||||
|
|
||||||
|
class PrometheusServer:
|
||||||
|
def __init__(self, logger=None):
|
||||||
|
self.runner = None
|
||||||
|
self.logger = logger or logging.getLogger(__name__)
|
||||||
|
self._monitor_loop_task = None
|
||||||
|
|
||||||
|
async def start(self, interface: str, port: int):
|
||||||
|
self.logger.info("start prometheus metrics")
|
||||||
|
prom_app = web.Application()
|
||||||
|
prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
|
||||||
|
self.runner = web.AppRunner(prom_app)
|
||||||
|
await self.runner.setup()
|
||||||
|
|
||||||
|
metrics_site = web.TCPSite(self.runner, interface, port, shutdown_timeout=.5)
|
||||||
|
await metrics_site.start()
|
||||||
|
self.logger.info(
|
||||||
|
'prometheus metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2]
|
||||||
|
)
|
||||||
|
self._monitor_loop_task = get_loop_metrics()
|
||||||
|
|
||||||
|
async def handle_metrics_get_request(self, request: web.Request):
|
||||||
|
try:
|
||||||
|
return web.Response(
|
||||||
|
text=prom_generate_latest().decode(),
|
||||||
|
content_type='text/plain; version=0.0.4'
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
self.logger.exception('could not generate prometheus data')
|
||||||
|
raise
|
||||||
|
|
||||||
|
async def stop(self):
|
||||||
|
if self._monitor_loop_task and not self._monitor_loop_task.done():
|
||||||
|
self._monitor_loop_task.cancel()
|
||||||
|
self._monitor_loop_task = None
|
||||||
|
await self.runner.cleanup()
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
Schema
|
||||||
|
=====
|
||||||
|
|
||||||
|
Those files are generated from the [types repo](https://github.com/lbryio/types). If you are modifying/adding a new type, make sure it is cloned in the same root folder as the SDK repo, like:
|
||||||
|
|
||||||
|
```
|
||||||
|
repos/
|
||||||
|
- lbry-sdk/
|
||||||
|
- types/
|
||||||
|
```
|
||||||
|
|
||||||
|
Then, [download protoc 3.2.0](https://github.com/protocolbuffers/protobuf/releases/tag/v3.2.0), add it to your PATH. On linux it is:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd ~/.local/bin
|
||||||
|
wget https://github.com/protocolbuffers/protobuf/releases/download/v3.2.0/protoc-3.2.0-linux-x86_64.zip
|
||||||
|
unzip protoc-3.2.0-linux-x86_64.zip bin/protoc -d..
|
||||||
|
```
|
||||||
|
|
||||||
|
Finally, `make` should update everything in place.
|
||||||
|
|
||||||
|
|
||||||
|
### Why protoc 3.2.0?
|
||||||
|
Different/newer versions will generate larger diffs and we need to make sure they are good. In theory, we can just update to latest and it will all work, but it is a good practice to check blockchain data and retro compatibility before bumping versions (if you do, please update this section!).
|
||||||
+13
-5
@@ -32,6 +32,17 @@ def calculate_sha384_file_hash(file_path):
|
|||||||
return sha384.digest()
|
return sha384.digest()
|
||||||
|
|
||||||
|
|
||||||
|
def country_int_to_str(country: int) -> str:
|
||||||
|
r = LocationMessage.Country.Name(country)
|
||||||
|
return r[1:] if r.startswith('R') else r
|
||||||
|
|
||||||
|
|
||||||
|
def country_str_to_int(country: str) -> int:
|
||||||
|
if len(country) == 3:
|
||||||
|
country = 'R' + country
|
||||||
|
return LocationMessage.Country.Value(country)
|
||||||
|
|
||||||
|
|
||||||
class Dimmensional(Metadata):
|
class Dimmensional(Metadata):
|
||||||
|
|
||||||
__slots__ = ()
|
__slots__ = ()
|
||||||
@@ -423,14 +434,11 @@ class Language(Metadata):
|
|||||||
@property
|
@property
|
||||||
def region(self) -> str:
|
def region(self) -> str:
|
||||||
if self.message.region:
|
if self.message.region:
|
||||||
r = LocationMessage.Country.Name(self.message.region)
|
return country_int_to_str(self.message.region)
|
||||||
return r[1:] if r.startswith('R') else r
|
|
||||||
|
|
||||||
@region.setter
|
@region.setter
|
||||||
def region(self, region: str):
|
def region(self, region: str):
|
||||||
if len(region) == 3:
|
self.message.region = country_str_to_int(region)
|
||||||
region = 'R'+region
|
|
||||||
self.message.region = LocationMessage.Country.Value(region)
|
|
||||||
|
|
||||||
|
|
||||||
class LanguageList(BaseMessageList[Language]):
|
class LanguageList(BaseMessageList[Language]):
|
||||||
|
|||||||
@@ -30,14 +30,10 @@ class Claim(Signable):
|
|||||||
COLLECTION = 'collection'
|
COLLECTION = 'collection'
|
||||||
REPOST = 'repost'
|
REPOST = 'repost'
|
||||||
|
|
||||||
__slots__ = 'version',
|
__slots__ = ()
|
||||||
|
|
||||||
message_class = ClaimMessage
|
message_class = ClaimMessage
|
||||||
|
|
||||||
def __init__(self, message=None):
|
|
||||||
super().__init__(message)
|
|
||||||
self.version = 2
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def claim_type(self) -> str:
|
def claim_type(self) -> str:
|
||||||
return self.message.WhichOneof('type')
|
return self.message.WhichOneof('type')
|
||||||
@@ -253,7 +249,7 @@ class Stream(BaseClaim):
|
|||||||
if stream_type in ('image', 'video', 'audio'):
|
if stream_type in ('image', 'video', 'audio'):
|
||||||
media = getattr(self, stream_type)
|
media = getattr(self, stream_type)
|
||||||
media_args = {'file_metadata': None}
|
media_args = {'file_metadata': None}
|
||||||
if file_path is not None:
|
if file_path is not None and not all((duration, width, height)):
|
||||||
try:
|
try:
|
||||||
media_args['file_metadata'] = binary_file_metadata(binary_file_parser(file_path))
|
media_args['file_metadata'] = binary_file_metadata(binary_file_parser(file_path))
|
||||||
except:
|
except:
|
||||||
@@ -307,6 +303,10 @@ class Stream(BaseClaim):
|
|||||||
def has_fee(self) -> bool:
|
def has_fee(self) -> bool:
|
||||||
return self.message.HasField('fee')
|
return self.message.HasField('fee')
|
||||||
|
|
||||||
|
@property
|
||||||
|
def has_source(self) -> bool:
|
||||||
|
return self.message.HasField('source')
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def source(self) -> Source:
|
def source(self) -> Source:
|
||||||
return Source(self.message.source)
|
return Source(self.message.source)
|
||||||
|
|||||||
@@ -46,8 +46,8 @@ types_map = {
|
|||||||
'.ksh': ('text/plain', 'document'),
|
'.ksh': ('text/plain', 'document'),
|
||||||
'.latex': ('application/x-latex', 'binary'),
|
'.latex': ('application/x-latex', 'binary'),
|
||||||
'.m1v': ('video/mpeg', 'video'),
|
'.m1v': ('video/mpeg', 'video'),
|
||||||
'.m3u': ('application/vnd.apple.mpegurl', 'audio'),
|
'.m3u': ('application/x-mpegurl', 'audio'),
|
||||||
'.m3u8': ('application/vnd.apple.mpegurl', 'audio'),
|
'.m3u8': ('application/x-mpegurl', 'video'),
|
||||||
'.man': ('application/x-troff-man', 'document'),
|
'.man': ('application/x-troff-man', 'document'),
|
||||||
'.markdown': ('text/markdown', 'document'),
|
'.markdown': ('text/markdown', 'document'),
|
||||||
'.md': ('text/markdown', 'document'),
|
'.md': ('text/markdown', 'document'),
|
||||||
@@ -61,10 +61,12 @@ types_map = {
|
|||||||
'.mp3': ('audio/mpeg', 'audio'),
|
'.mp3': ('audio/mpeg', 'audio'),
|
||||||
'.mp4': ('video/mp4', 'video'),
|
'.mp4': ('video/mp4', 'video'),
|
||||||
'.mpa': ('video/mpeg', 'video'),
|
'.mpa': ('video/mpeg', 'video'),
|
||||||
|
'.mpd': ('application/dash+xml', 'video'),
|
||||||
'.mpe': ('video/mpeg', 'video'),
|
'.mpe': ('video/mpeg', 'video'),
|
||||||
'.mpeg': ('video/mpeg', 'video'),
|
'.mpeg': ('video/mpeg', 'video'),
|
||||||
'.mpg': ('video/mpeg', 'video'),
|
'.mpg': ('video/mpeg', 'video'),
|
||||||
'.ms': ('application/x-troff-ms', 'binary'),
|
'.ms': ('application/x-troff-ms', 'binary'),
|
||||||
|
'.m4s': ('video/iso.segment', 'binary'),
|
||||||
'.nc': ('application/x-netcdf', 'binary'),
|
'.nc': ('application/x-netcdf', 'binary'),
|
||||||
'.nws': ('message/rfc822', 'document'),
|
'.nws': ('message/rfc822', 'document'),
|
||||||
'.o': ('application/octet-stream', 'binary'),
|
'.o': ('application/octet-stream', 'binary'),
|
||||||
@@ -118,10 +120,12 @@ types_map = {
|
|||||||
'.tif': ('image/tiff', 'image'),
|
'.tif': ('image/tiff', 'image'),
|
||||||
'.tiff': ('image/tiff', 'image'),
|
'.tiff': ('image/tiff', 'image'),
|
||||||
'.tr': ('application/x-troff', 'binary'),
|
'.tr': ('application/x-troff', 'binary'),
|
||||||
|
'.ts': ('video/mp2t', 'video'),
|
||||||
'.tsv': ('text/tab-separated-values', 'document'),
|
'.tsv': ('text/tab-separated-values', 'document'),
|
||||||
'.txt': ('text/plain', 'document'),
|
'.txt': ('text/plain', 'document'),
|
||||||
'.ustar': ('application/x-ustar', 'binary'),
|
'.ustar': ('application/x-ustar', 'binary'),
|
||||||
'.vcf': ('text/x-vcard', 'document'),
|
'.vcf': ('text/x-vcard', 'document'),
|
||||||
|
'.vtt': ('text/vtt', 'document'),
|
||||||
'.wav': ('audio/x-wav', 'audio'),
|
'.wav': ('audio/x-wav', 'audio'),
|
||||||
'.webm': ('video/webm', 'video'),
|
'.webm': ('video/webm', 'video'),
|
||||||
'.wiz': ('application/msword', 'document'),
|
'.wiz': ('application/msword', 'document'),
|
||||||
@@ -141,6 +145,7 @@ types_map = {
|
|||||||
'.cbz': ('application/vnd.comicbook+zip', 'document'),
|
'.cbz': ('application/vnd.comicbook+zip', 'document'),
|
||||||
'.flac': ('audio/flac', 'audio'),
|
'.flac': ('audio/flac', 'audio'),
|
||||||
'.lbry': ('application/x-ext-lbry', 'document'),
|
'.lbry': ('application/x-ext-lbry', 'document'),
|
||||||
|
'.m4a': ('audio/mp4', 'audio'),
|
||||||
'.m4v': ('video/m4v', 'video'),
|
'.m4v': ('video/m4v', 'video'),
|
||||||
'.mid': ('audio/midi', 'audio'),
|
'.mid': ('audio/midi', 'audio'),
|
||||||
'.midi': ('audio/midi', 'audio'),
|
'.midi': ('audio/midi', 'audio'),
|
||||||
@@ -148,6 +153,7 @@ types_map = {
|
|||||||
'.mobi': ('application/x-mobipocket-ebook', 'document'),
|
'.mobi': ('application/x-mobipocket-ebook', 'document'),
|
||||||
'.oga': ('audio/ogg', 'audio'),
|
'.oga': ('audio/ogg', 'audio'),
|
||||||
'.ogv': ('video/ogg', 'video'),
|
'.ogv': ('video/ogg', 'video'),
|
||||||
|
'.ogg': ('video/ogg', 'video'),
|
||||||
'.pct': ('image/pict', 'image'),
|
'.pct': ('image/pict', 'image'),
|
||||||
'.pic': ('image/pict', 'image'),
|
'.pic': ('image/pict', 'image'),
|
||||||
'.pict': ('image/pict', 'image'),
|
'.pict': ('image/pict', 'image'),
|
||||||
|
|||||||
+47
-36
@@ -13,48 +13,45 @@ NOT_FOUND = ErrorMessage.Code.Name(ErrorMessage.NOT_FOUND)
|
|||||||
BLOCKED = ErrorMessage.Code.Name(ErrorMessage.BLOCKED)
|
BLOCKED = ErrorMessage.Code.Name(ErrorMessage.BLOCKED)
|
||||||
|
|
||||||
|
|
||||||
def set_reference(reference, claim_hash, rows):
|
def set_reference(reference, txo_row):
|
||||||
if claim_hash:
|
if txo_row:
|
||||||
for txo in rows:
|
reference.tx_hash = txo_row['txo_hash'][:32]
|
||||||
if claim_hash == txo['claim_hash']:
|
reference.nout = struct.unpack('<I', txo_row['txo_hash'][32:])[0]
|
||||||
reference.tx_hash = txo['txo_hash'][:32]
|
reference.height = txo_row['height']
|
||||||
reference.nout = struct.unpack('<I', txo['txo_hash'][32:])[0]
|
|
||||||
reference.height = txo['height']
|
|
||||||
return
|
|
||||||
|
|
||||||
|
|
||||||
class Censor:
|
class Censor:
|
||||||
|
|
||||||
__slots__ = 'streams', 'channels', 'censored', 'total'
|
NOT_CENSORED = 0
|
||||||
|
SEARCH = 1
|
||||||
|
RESOLVE = 2
|
||||||
|
|
||||||
def __init__(self, streams: dict = None, channels: dict = None):
|
__slots__ = 'censor_type', 'censored'
|
||||||
self.streams = streams or {}
|
|
||||||
self.channels = channels or {}
|
def __init__(self, censor_type):
|
||||||
|
self.censor_type = censor_type
|
||||||
self.censored = {}
|
self.censored = {}
|
||||||
self.total = 0
|
|
||||||
|
def is_censored(self, row):
|
||||||
|
return (row.get('censor_type') or self.NOT_CENSORED) >= self.censor_type
|
||||||
|
|
||||||
|
def apply(self, rows):
|
||||||
|
return [row for row in rows if not self.censor(row)]
|
||||||
|
|
||||||
def censor(self, row) -> bool:
|
def censor(self, row) -> bool:
|
||||||
was_censored = False
|
if self.is_censored(row):
|
||||||
for claim_hash, lookup in (
|
censoring_channel_hash = bytes.fromhex(row['censoring_channel_id'])[::-1]
|
||||||
(row['claim_hash'], self.streams),
|
self.censored.setdefault(censoring_channel_hash, set())
|
||||||
(row['claim_hash'], self.channels),
|
self.censored[censoring_channel_hash].add(row['tx_hash'])
|
||||||
(row['channel_hash'], self.channels)):
|
return True
|
||||||
censoring_channel_hash = lookup.get(claim_hash)
|
return False
|
||||||
if censoring_channel_hash:
|
|
||||||
was_censored = True
|
|
||||||
self.censored.setdefault(censoring_channel_hash, 0)
|
|
||||||
self.censored[censoring_channel_hash] += 1
|
|
||||||
break
|
|
||||||
if was_censored:
|
|
||||||
self.total += 1
|
|
||||||
return was_censored
|
|
||||||
|
|
||||||
def to_message(self, outputs: OutputsMessage, extra_txo_rows):
|
def to_message(self, outputs: OutputsMessage, extra_txo_rows: dict):
|
||||||
outputs.blocked_total = self.total
|
|
||||||
for censoring_channel_hash, count in self.censored.items():
|
for censoring_channel_hash, count in self.censored.items():
|
||||||
blocked = outputs.blocked.add()
|
blocked = outputs.blocked.add()
|
||||||
blocked.count = count
|
blocked.count = len(count)
|
||||||
set_reference(blocked.channel, censoring_channel_hash, extra_txo_rows)
|
set_reference(blocked.channel, extra_txo_rows.get(censoring_channel_hash))
|
||||||
|
outputs.blocked_total += len(count)
|
||||||
|
|
||||||
|
|
||||||
class Outputs:
|
class Outputs:
|
||||||
@@ -153,12 +150,26 @@ class Outputs:
|
|||||||
outputs.blocked, outputs.blocked_total
|
outputs.blocked, outputs.blocked_total
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_grpc(cls, outputs: OutputsMessage) -> 'Outputs':
|
||||||
|
txs = set()
|
||||||
|
for txo_message in chain(outputs.txos, outputs.extra_txos):
|
||||||
|
if txo_message.WhichOneof('meta') == 'error':
|
||||||
|
continue
|
||||||
|
txs.add((hexlify(txo_message.tx_hash[::-1]).decode(), txo_message.height))
|
||||||
|
return cls(
|
||||||
|
outputs.txos, outputs.extra_txos, txs,
|
||||||
|
outputs.offset, outputs.total,
|
||||||
|
outputs.blocked, outputs.blocked_total
|
||||||
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def to_base64(cls, txo_rows, extra_txo_rows, offset=0, total=None, blocked=None) -> str:
|
def to_base64(cls, txo_rows, extra_txo_rows, offset=0, total=None, blocked=None) -> str:
|
||||||
return base64.b64encode(cls.to_bytes(txo_rows, extra_txo_rows, offset, total, blocked)).decode()
|
return base64.b64encode(cls.to_bytes(txo_rows, extra_txo_rows, offset, total, blocked)).decode()
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def to_bytes(cls, txo_rows, extra_txo_rows, offset=0, total=None, blocked: Censor = None) -> bytes:
|
def to_bytes(cls, txo_rows, extra_txo_rows, offset=0, total=None, blocked: Censor = None) -> bytes:
|
||||||
|
extra_txo_rows = {row['claim_hash']: row for row in extra_txo_rows}
|
||||||
page = OutputsMessage()
|
page = OutputsMessage()
|
||||||
page.offset = offset
|
page.offset = offset
|
||||||
if total is not None:
|
if total is not None:
|
||||||
@@ -167,12 +178,12 @@ class Outputs:
|
|||||||
blocked.to_message(page, extra_txo_rows)
|
blocked.to_message(page, extra_txo_rows)
|
||||||
for row in txo_rows:
|
for row in txo_rows:
|
||||||
cls.row_to_message(row, page.txos.add(), extra_txo_rows)
|
cls.row_to_message(row, page.txos.add(), extra_txo_rows)
|
||||||
for row in extra_txo_rows:
|
for row in extra_txo_rows.values():
|
||||||
cls.row_to_message(row, page.extra_txos.add(), extra_txo_rows)
|
cls.row_to_message(row, page.extra_txos.add(), extra_txo_rows)
|
||||||
return page.SerializeToString()
|
return page.SerializeToString()
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def row_to_message(cls, txo, txo_message, extra_txo_rows):
|
def row_to_message(cls, txo, txo_message, extra_row_dict: dict):
|
||||||
if isinstance(txo, Exception):
|
if isinstance(txo, Exception):
|
||||||
txo_message.error.text = txo.args[0]
|
txo_message.error.text = txo.args[0]
|
||||||
if isinstance(txo, ValueError):
|
if isinstance(txo, ValueError):
|
||||||
@@ -181,7 +192,7 @@ class Outputs:
|
|||||||
txo_message.error.code = ErrorMessage.NOT_FOUND
|
txo_message.error.code = ErrorMessage.NOT_FOUND
|
||||||
elif isinstance(txo, ResolveCensoredError):
|
elif isinstance(txo, ResolveCensoredError):
|
||||||
txo_message.error.code = ErrorMessage.BLOCKED
|
txo_message.error.code = ErrorMessage.BLOCKED
|
||||||
set_reference(txo_message.error.blocked.channel, txo.censor_hash, extra_txo_rows)
|
set_reference(txo_message.error.blocked.channel, extra_row_dict.get(bytes.fromhex(txo.censor_id)[::-1]))
|
||||||
return
|
return
|
||||||
txo_message.tx_hash = txo['txo_hash'][:32]
|
txo_message.tx_hash = txo['txo_hash'][:32]
|
||||||
txo_message.nout, = struct.unpack('<I', txo['txo_hash'][32:])
|
txo_message.nout, = struct.unpack('<I', txo['txo_hash'][32:])
|
||||||
@@ -204,5 +215,5 @@ class Outputs:
|
|||||||
txo_message.claim.trending_mixed = txo['trending_mixed']
|
txo_message.claim.trending_mixed = txo['trending_mixed']
|
||||||
txo_message.claim.trending_local = txo['trending_local']
|
txo_message.claim.trending_local = txo['trending_local']
|
||||||
txo_message.claim.trending_global = txo['trending_global']
|
txo_message.claim.trending_global = txo['trending_global']
|
||||||
set_reference(txo_message.claim.channel, txo['channel_hash'], extra_txo_rows)
|
set_reference(txo_message.claim.channel, extra_row_dict.get(txo['channel_hash']))
|
||||||
set_reference(txo_message.claim.repost, txo['reposted_claim_hash'], extra_txo_rows)
|
set_reference(txo_message.claim.repost, extra_row_dict.get(txo['reposted_claim_hash']))
|
||||||
+18
-1
@@ -1,6 +1,23 @@
|
|||||||
from lbry.schema.base import Signable
|
from lbry.schema.base import Signable
|
||||||
|
from lbry.schema.types.v2.support_pb2 import Support as SupportMessage
|
||||||
|
|
||||||
|
|
||||||
class Support(Signable):
|
class Support(Signable):
|
||||||
__slots__ = ()
|
__slots__ = ()
|
||||||
message_class = None # TODO: add support protobufs
|
message_class = SupportMessage
|
||||||
|
|
||||||
|
@property
|
||||||
|
def emoji(self) -> str:
|
||||||
|
return self.message.emoji
|
||||||
|
|
||||||
|
@emoji.setter
|
||||||
|
def emoji(self, emoji: str):
|
||||||
|
self.message.emoji = emoji
|
||||||
|
|
||||||
|
@property
|
||||||
|
def comment(self) -> str:
|
||||||
|
return self.message.comment
|
||||||
|
|
||||||
|
@comment.setter
|
||||||
|
def comment(self, comment: str):
|
||||||
|
self.message.comment = comment
|
||||||
@@ -0,0 +1,740 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||||
|
# source: hub.proto
|
||||||
|
"""Generated protocol buffer code."""
|
||||||
|
from google.protobuf import descriptor as _descriptor
|
||||||
|
from google.protobuf import message as _message
|
||||||
|
from google.protobuf import reflection as _reflection
|
||||||
|
from google.protobuf import symbol_database as _symbol_database
|
||||||
|
# @@protoc_insertion_point(imports)
|
||||||
|
|
||||||
|
_sym_db = _symbol_database.Default()
|
||||||
|
|
||||||
|
|
||||||
|
import lbry.schema.types.v2.result_pb2 as result__pb2
|
||||||
|
|
||||||
|
|
||||||
|
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||||
|
name='hub.proto',
|
||||||
|
package='pb',
|
||||||
|
syntax='proto3',
|
||||||
|
serialized_options=b'Z$github.com/lbryio/hub/protobuf/go/pb',
|
||||||
|
create_key=_descriptor._internal_create_key,
|
||||||
|
serialized_pb=b'\n\thub.proto\x12\x02pb\x1a\x0cresult.proto\"0\n\x0fInvertibleField\x12\x0e\n\x06invert\x18\x01 \x01(\x08\x12\r\n\x05value\x18\x02 \x03(\t\"\x1a\n\tBoolValue\x12\r\n\x05value\x18\x01 \x01(\x08\"\x1c\n\x0bUInt32Value\x12\r\n\x05value\x18\x01 \x01(\r\"j\n\nRangeField\x12\x1d\n\x02op\x18\x01 \x01(\x0e\x32\x11.pb.RangeField.Op\x12\r\n\x05value\x18\x02 \x03(\t\".\n\x02Op\x12\x06\n\x02\x45Q\x10\x00\x12\x07\n\x03LTE\x10\x01\x12\x07\n\x03GTE\x10\x02\x12\x06\n\x02LT\x10\x03\x12\x06\n\x02GT\x10\x04\"\x9c\r\n\rSearchRequest\x12%\n\x08\x63laim_id\x18\x01 \x01(\x0b\x32\x13.pb.InvertibleField\x12\'\n\nchannel_id\x18\x02 \x01(\x0b\x32\x13.pb.InvertibleField\x12\x0c\n\x04text\x18\x03 \x01(\t\x12\r\n\x05limit\x18\x04 \x01(\r\x12\x10\n\x08order_by\x18\x05 \x03(\t\x12\x0e\n\x06offset\x18\x06 \x01(\r\x12\x16\n\x0eis_controlling\x18\x07 \x01(\x08\x12\x1d\n\x15last_take_over_height\x18\x08 \x01(\t\x12\x12\n\nclaim_name\x18\t \x01(\t\x12\x17\n\x0fnormalized_name\x18\n \x01(\t\x12#\n\x0btx_position\x18\x0b \x01(\x0b\x32\x0e.pb.RangeField\x12\x1e\n\x06\x61mount\x18\x0c \x01(\x0b\x32\x0e.pb.RangeField\x12!\n\ttimestamp\x18\r \x01(\x0b\x32\x0e.pb.RangeField\x12*\n\x12\x63reation_timestamp\x18\x0e \x01(\x0b\x32\x0e.pb.RangeField\x12\x1e\n\x06height\x18\x0f \x01(\x0b\x32\x0e.pb.RangeField\x12\'\n\x0f\x63reation_height\x18\x10 \x01(\x0b\x32\x0e.pb.RangeField\x12)\n\x11\x61\x63tivation_height\x18\x11 \x01(\x0b\x32\x0e.pb.RangeField\x12)\n\x11\x65xpiration_height\x18\x12 \x01(\x0b\x32\x0e.pb.RangeField\x12$\n\x0crelease_time\x18\x13 \x01(\x0b\x32\x0e.pb.RangeField\x12\x11\n\tshort_url\x18\x14 \x01(\t\x12\x15\n\rcanonical_url\x18\x15 \x01(\t\x12\r\n\x05title\x18\x16 \x01(\t\x12\x0e\n\x06\x61uthor\x18\x17 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x18 \x01(\t\x12\x12\n\nclaim_type\x18\x19 \x03(\t\x12$\n\x0crepost_count\x18\x1a \x01(\x0b\x32\x0e.pb.RangeField\x12\x13\n\x0bstream_type\x18\x1b \x03(\t\x12\x12\n\nmedia_type\x18\x1c \x03(\t\x12\"\n\nfee_amount\x18\x1d \x01(\x0b\x32\x0e.pb.RangeField\x12\x14\n\x0c\x66\x65\x65_currency\x18\x1e \x01(\t\x12 \n\x08\x64uration\x18\x1f \x01(\x0b\x32\x0e.pb.RangeField\x12\x19\n\x11reposted_claim_id\x18 \x01(\t\x12#\n\x0b\x63\x65nsor_type\x18! \x01(\x0b\x32\x0e.pb.RangeField\x12\x19\n\x11\x63laims_in_channel\x18\" \x01(\t\x12$\n\x0c\x63hannel_join\x18# \x01(\x0b\x32\x0e.pb.RangeField\x12)\n\x12is_signature_valid\x18$ \x01(\x0b\x32\r.pb.BoolValue\x12(\n\x10\x65\x66\x66\x65\x63tive_amount\x18% \x01(\x0b\x32\x0e.pb.RangeField\x12&\n\x0esupport_amount\x18& \x01(\x0b\x32\x0e.pb.RangeField\x12&\n\x0etrending_group\x18\' \x01(\x0b\x32\x0e.pb.RangeField\x12&\n\x0etrending_mixed\x18( \x01(\x0b\x32\x0e.pb.RangeField\x12&\n\x0etrending_local\x18) \x01(\x0b\x32\x0e.pb.RangeField\x12\'\n\x0ftrending_global\x18* \x01(\x0b\x32\x0e.pb.RangeField\x12\r\n\x05tx_id\x18+ \x01(\t\x12 \n\x07tx_nout\x18, \x01(\x0b\x32\x0f.pb.UInt32Value\x12\x11\n\tsignature\x18- \x01(\t\x12\x18\n\x10signature_digest\x18. \x01(\t\x12\x18\n\x10public_key_bytes\x18/ \x01(\t\x12\x15\n\rpublic_key_id\x18\x30 \x01(\t\x12\x10\n\x08\x61ny_tags\x18\x31 \x03(\t\x12\x10\n\x08\x61ll_tags\x18\x32 \x03(\t\x12\x10\n\x08not_tags\x18\x33 \x03(\t\x12\x1d\n\x15has_channel_signature\x18\x34 \x01(\x08\x12!\n\nhas_source\x18\x35 \x01(\x0b\x32\r.pb.BoolValue\x12 \n\x18limit_claims_per_channel\x18\x36 \x01(\r\x12\x15\n\rany_languages\x18\x37 \x03(\t\x12\x15\n\rall_languages\x18\x38 \x03(\t\x12\x19\n\x11remove_duplicates\x18\x39 \x01(\x08\x12\x11\n\tno_totals\x18: \x01(\x08\x32\x31\n\x03Hub\x12*\n\x06Search\x12\x11.pb.SearchRequest\x1a\x0b.pb.Outputs\"\x00\x42&Z$github.com/lbryio/hub/protobuf/go/pbb\x06proto3'
|
||||||
|
,
|
||||||
|
dependencies=[result__pb2.DESCRIPTOR,])
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
_RANGEFIELD_OP = _descriptor.EnumDescriptor(
|
||||||
|
name='Op',
|
||||||
|
full_name='pb.RangeField.Op',
|
||||||
|
filename=None,
|
||||||
|
file=DESCRIPTOR,
|
||||||
|
create_key=_descriptor._internal_create_key,
|
||||||
|
values=[
|
||||||
|
_descriptor.EnumValueDescriptor(
|
||||||
|
name='EQ', index=0, number=0,
|
||||||
|
serialized_options=None,
|
||||||
|
type=None,
|
||||||
|
create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.EnumValueDescriptor(
|
||||||
|
name='LTE', index=1, number=1,
|
||||||
|
serialized_options=None,
|
||||||
|
type=None,
|
||||||
|
create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.EnumValueDescriptor(
|
||||||
|
name='GTE', index=2, number=2,
|
||||||
|
serialized_options=None,
|
||||||
|
type=None,
|
||||||
|
create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.EnumValueDescriptor(
|
||||||
|
name='LT', index=3, number=3,
|
||||||
|
serialized_options=None,
|
||||||
|
type=None,
|
||||||
|
create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.EnumValueDescriptor(
|
||||||
|
name='GT', index=4, number=4,
|
||||||
|
serialized_options=None,
|
||||||
|
type=None,
|
||||||
|
create_key=_descriptor._internal_create_key),
|
||||||
|
],
|
||||||
|
containing_type=None,
|
||||||
|
serialized_options=None,
|
||||||
|
serialized_start=199,
|
||||||
|
serialized_end=245,
|
||||||
|
)
|
||||||
|
_sym_db.RegisterEnumDescriptor(_RANGEFIELD_OP)
|
||||||
|
|
||||||
|
|
||||||
|
_INVERTIBLEFIELD = _descriptor.Descriptor(
|
||||||
|
name='InvertibleField',
|
||||||
|
full_name='pb.InvertibleField',
|
||||||
|
filename=None,
|
||||||
|
file=DESCRIPTOR,
|
||||||
|
containing_type=None,
|
||||||
|
create_key=_descriptor._internal_create_key,
|
||||||
|
fields=[
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='invert', full_name='pb.InvertibleField.invert', index=0,
|
||||||
|
number=1, type=8, cpp_type=7, label=1,
|
||||||
|
has_default_value=False, default_value=False,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='value', full_name='pb.InvertibleField.value', index=1,
|
||||||
|
number=2, type=9, cpp_type=9, label=3,
|
||||||
|
has_default_value=False, default_value=[],
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
],
|
||||||
|
extensions=[
|
||||||
|
],
|
||||||
|
nested_types=[],
|
||||||
|
enum_types=[
|
||||||
|
],
|
||||||
|
serialized_options=None,
|
||||||
|
is_extendable=False,
|
||||||
|
syntax='proto3',
|
||||||
|
extension_ranges=[],
|
||||||
|
oneofs=[
|
||||||
|
],
|
||||||
|
serialized_start=31,
|
||||||
|
serialized_end=79,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
_BOOLVALUE = _descriptor.Descriptor(
|
||||||
|
name='BoolValue',
|
||||||
|
full_name='pb.BoolValue',
|
||||||
|
filename=None,
|
||||||
|
file=DESCRIPTOR,
|
||||||
|
containing_type=None,
|
||||||
|
create_key=_descriptor._internal_create_key,
|
||||||
|
fields=[
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='value', full_name='pb.BoolValue.value', index=0,
|
||||||
|
number=1, type=8, cpp_type=7, label=1,
|
||||||
|
has_default_value=False, default_value=False,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
],
|
||||||
|
extensions=[
|
||||||
|
],
|
||||||
|
nested_types=[],
|
||||||
|
enum_types=[
|
||||||
|
],
|
||||||
|
serialized_options=None,
|
||||||
|
is_extendable=False,
|
||||||
|
syntax='proto3',
|
||||||
|
extension_ranges=[],
|
||||||
|
oneofs=[
|
||||||
|
],
|
||||||
|
serialized_start=81,
|
||||||
|
serialized_end=107,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
_UINT32VALUE = _descriptor.Descriptor(
|
||||||
|
name='UInt32Value',
|
||||||
|
full_name='pb.UInt32Value',
|
||||||
|
filename=None,
|
||||||
|
file=DESCRIPTOR,
|
||||||
|
containing_type=None,
|
||||||
|
create_key=_descriptor._internal_create_key,
|
||||||
|
fields=[
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='value', full_name='pb.UInt32Value.value', index=0,
|
||||||
|
number=1, type=13, cpp_type=3, label=1,
|
||||||
|
has_default_value=False, default_value=0,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
],
|
||||||
|
extensions=[
|
||||||
|
],
|
||||||
|
nested_types=[],
|
||||||
|
enum_types=[
|
||||||
|
],
|
||||||
|
serialized_options=None,
|
||||||
|
is_extendable=False,
|
||||||
|
syntax='proto3',
|
||||||
|
extension_ranges=[],
|
||||||
|
oneofs=[
|
||||||
|
],
|
||||||
|
serialized_start=109,
|
||||||
|
serialized_end=137,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
_RANGEFIELD = _descriptor.Descriptor(
|
||||||
|
name='RangeField',
|
||||||
|
full_name='pb.RangeField',
|
||||||
|
filename=None,
|
||||||
|
file=DESCRIPTOR,
|
||||||
|
containing_type=None,
|
||||||
|
create_key=_descriptor._internal_create_key,
|
||||||
|
fields=[
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='op', full_name='pb.RangeField.op', index=0,
|
||||||
|
number=1, type=14, cpp_type=8, label=1,
|
||||||
|
has_default_value=False, default_value=0,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='value', full_name='pb.RangeField.value', index=1,
|
||||||
|
number=2, type=9, cpp_type=9, label=3,
|
||||||
|
has_default_value=False, default_value=[],
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
],
|
||||||
|
extensions=[
|
||||||
|
],
|
||||||
|
nested_types=[],
|
||||||
|
enum_types=[
|
||||||
|
_RANGEFIELD_OP,
|
||||||
|
],
|
||||||
|
serialized_options=None,
|
||||||
|
is_extendable=False,
|
||||||
|
syntax='proto3',
|
||||||
|
extension_ranges=[],
|
||||||
|
oneofs=[
|
||||||
|
],
|
||||||
|
serialized_start=139,
|
||||||
|
serialized_end=245,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
_SEARCHREQUEST = _descriptor.Descriptor(
|
||||||
|
name='SearchRequest',
|
||||||
|
full_name='pb.SearchRequest',
|
||||||
|
filename=None,
|
||||||
|
file=DESCRIPTOR,
|
||||||
|
containing_type=None,
|
||||||
|
create_key=_descriptor._internal_create_key,
|
||||||
|
fields=[
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='claim_id', full_name='pb.SearchRequest.claim_id', index=0,
|
||||||
|
number=1, type=11, cpp_type=10, label=1,
|
||||||
|
has_default_value=False, default_value=None,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='channel_id', full_name='pb.SearchRequest.channel_id', index=1,
|
||||||
|
number=2, type=11, cpp_type=10, label=1,
|
||||||
|
has_default_value=False, default_value=None,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='text', full_name='pb.SearchRequest.text', index=2,
|
||||||
|
number=3, type=9, cpp_type=9, label=1,
|
||||||
|
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='limit', full_name='pb.SearchRequest.limit', index=3,
|
||||||
|
number=4, type=13, cpp_type=3, label=1,
|
||||||
|
has_default_value=False, default_value=0,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='order_by', full_name='pb.SearchRequest.order_by', index=4,
|
||||||
|
number=5, type=9, cpp_type=9, label=3,
|
||||||
|
has_default_value=False, default_value=[],
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='offset', full_name='pb.SearchRequest.offset', index=5,
|
||||||
|
number=6, type=13, cpp_type=3, label=1,
|
||||||
|
has_default_value=False, default_value=0,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='is_controlling', full_name='pb.SearchRequest.is_controlling', index=6,
|
||||||
|
number=7, type=8, cpp_type=7, label=1,
|
||||||
|
has_default_value=False, default_value=False,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='last_take_over_height', full_name='pb.SearchRequest.last_take_over_height', index=7,
|
||||||
|
number=8, type=9, cpp_type=9, label=1,
|
||||||
|
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='claim_name', full_name='pb.SearchRequest.claim_name', index=8,
|
||||||
|
number=9, type=9, cpp_type=9, label=1,
|
||||||
|
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='normalized_name', full_name='pb.SearchRequest.normalized_name', index=9,
|
||||||
|
number=10, type=9, cpp_type=9, label=1,
|
||||||
|
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='tx_position', full_name='pb.SearchRequest.tx_position', index=10,
|
||||||
|
number=11, type=11, cpp_type=10, label=1,
|
||||||
|
has_default_value=False, default_value=None,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='amount', full_name='pb.SearchRequest.amount', index=11,
|
||||||
|
number=12, type=11, cpp_type=10, label=1,
|
||||||
|
has_default_value=False, default_value=None,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='timestamp', full_name='pb.SearchRequest.timestamp', index=12,
|
||||||
|
number=13, type=11, cpp_type=10, label=1,
|
||||||
|
has_default_value=False, default_value=None,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='creation_timestamp', full_name='pb.SearchRequest.creation_timestamp', index=13,
|
||||||
|
number=14, type=11, cpp_type=10, label=1,
|
||||||
|
has_default_value=False, default_value=None,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='height', full_name='pb.SearchRequest.height', index=14,
|
||||||
|
number=15, type=11, cpp_type=10, label=1,
|
||||||
|
has_default_value=False, default_value=None,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='creation_height', full_name='pb.SearchRequest.creation_height', index=15,
|
||||||
|
number=16, type=11, cpp_type=10, label=1,
|
||||||
|
has_default_value=False, default_value=None,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='activation_height', full_name='pb.SearchRequest.activation_height', index=16,
|
||||||
|
number=17, type=11, cpp_type=10, label=1,
|
||||||
|
has_default_value=False, default_value=None,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='expiration_height', full_name='pb.SearchRequest.expiration_height', index=17,
|
||||||
|
number=18, type=11, cpp_type=10, label=1,
|
||||||
|
has_default_value=False, default_value=None,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='release_time', full_name='pb.SearchRequest.release_time', index=18,
|
||||||
|
number=19, type=11, cpp_type=10, label=1,
|
||||||
|
has_default_value=False, default_value=None,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='short_url', full_name='pb.SearchRequest.short_url', index=19,
|
||||||
|
number=20, type=9, cpp_type=9, label=1,
|
||||||
|
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='canonical_url', full_name='pb.SearchRequest.canonical_url', index=20,
|
||||||
|
number=21, type=9, cpp_type=9, label=1,
|
||||||
|
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='title', full_name='pb.SearchRequest.title', index=21,
|
||||||
|
number=22, type=9, cpp_type=9, label=1,
|
||||||
|
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='author', full_name='pb.SearchRequest.author', index=22,
|
||||||
|
number=23, type=9, cpp_type=9, label=1,
|
||||||
|
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='description', full_name='pb.SearchRequest.description', index=23,
|
||||||
|
number=24, type=9, cpp_type=9, label=1,
|
||||||
|
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='claim_type', full_name='pb.SearchRequest.claim_type', index=24,
|
||||||
|
number=25, type=9, cpp_type=9, label=3,
|
||||||
|
has_default_value=False, default_value=[],
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='repost_count', full_name='pb.SearchRequest.repost_count', index=25,
|
||||||
|
number=26, type=11, cpp_type=10, label=1,
|
||||||
|
has_default_value=False, default_value=None,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='stream_type', full_name='pb.SearchRequest.stream_type', index=26,
|
||||||
|
number=27, type=9, cpp_type=9, label=3,
|
||||||
|
has_default_value=False, default_value=[],
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='media_type', full_name='pb.SearchRequest.media_type', index=27,
|
||||||
|
number=28, type=9, cpp_type=9, label=3,
|
||||||
|
has_default_value=False, default_value=[],
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='fee_amount', full_name='pb.SearchRequest.fee_amount', index=28,
|
||||||
|
number=29, type=11, cpp_type=10, label=1,
|
||||||
|
has_default_value=False, default_value=None,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='fee_currency', full_name='pb.SearchRequest.fee_currency', index=29,
|
||||||
|
number=30, type=9, cpp_type=9, label=1,
|
||||||
|
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='duration', full_name='pb.SearchRequest.duration', index=30,
|
||||||
|
number=31, type=11, cpp_type=10, label=1,
|
||||||
|
has_default_value=False, default_value=None,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='reposted_claim_id', full_name='pb.SearchRequest.reposted_claim_id', index=31,
|
||||||
|
number=32, type=9, cpp_type=9, label=1,
|
||||||
|
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='censor_type', full_name='pb.SearchRequest.censor_type', index=32,
|
||||||
|
number=33, type=11, cpp_type=10, label=1,
|
||||||
|
has_default_value=False, default_value=None,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='claims_in_channel', full_name='pb.SearchRequest.claims_in_channel', index=33,
|
||||||
|
number=34, type=9, cpp_type=9, label=1,
|
||||||
|
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='channel_join', full_name='pb.SearchRequest.channel_join', index=34,
|
||||||
|
number=35, type=11, cpp_type=10, label=1,
|
||||||
|
has_default_value=False, default_value=None,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='is_signature_valid', full_name='pb.SearchRequest.is_signature_valid', index=35,
|
||||||
|
number=36, type=11, cpp_type=10, label=1,
|
||||||
|
has_default_value=False, default_value=None,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='effective_amount', full_name='pb.SearchRequest.effective_amount', index=36,
|
||||||
|
number=37, type=11, cpp_type=10, label=1,
|
||||||
|
has_default_value=False, default_value=None,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='support_amount', full_name='pb.SearchRequest.support_amount', index=37,
|
||||||
|
number=38, type=11, cpp_type=10, label=1,
|
||||||
|
has_default_value=False, default_value=None,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='trending_group', full_name='pb.SearchRequest.trending_group', index=38,
|
||||||
|
number=39, type=11, cpp_type=10, label=1,
|
||||||
|
has_default_value=False, default_value=None,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='trending_mixed', full_name='pb.SearchRequest.trending_mixed', index=39,
|
||||||
|
number=40, type=11, cpp_type=10, label=1,
|
||||||
|
has_default_value=False, default_value=None,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='trending_local', full_name='pb.SearchRequest.trending_local', index=40,
|
||||||
|
number=41, type=11, cpp_type=10, label=1,
|
||||||
|
has_default_value=False, default_value=None,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='trending_global', full_name='pb.SearchRequest.trending_global', index=41,
|
||||||
|
number=42, type=11, cpp_type=10, label=1,
|
||||||
|
has_default_value=False, default_value=None,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='tx_id', full_name='pb.SearchRequest.tx_id', index=42,
|
||||||
|
number=43, type=9, cpp_type=9, label=1,
|
||||||
|
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='tx_nout', full_name='pb.SearchRequest.tx_nout', index=43,
|
||||||
|
number=44, type=11, cpp_type=10, label=1,
|
||||||
|
has_default_value=False, default_value=None,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='signature', full_name='pb.SearchRequest.signature', index=44,
|
||||||
|
number=45, type=9, cpp_type=9, label=1,
|
||||||
|
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='signature_digest', full_name='pb.SearchRequest.signature_digest', index=45,
|
||||||
|
number=46, type=9, cpp_type=9, label=1,
|
||||||
|
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='public_key_bytes', full_name='pb.SearchRequest.public_key_bytes', index=46,
|
||||||
|
number=47, type=9, cpp_type=9, label=1,
|
||||||
|
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='public_key_id', full_name='pb.SearchRequest.public_key_id', index=47,
|
||||||
|
number=48, type=9, cpp_type=9, label=1,
|
||||||
|
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='any_tags', full_name='pb.SearchRequest.any_tags', index=48,
|
||||||
|
number=49, type=9, cpp_type=9, label=3,
|
||||||
|
has_default_value=False, default_value=[],
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='all_tags', full_name='pb.SearchRequest.all_tags', index=49,
|
||||||
|
number=50, type=9, cpp_type=9, label=3,
|
||||||
|
has_default_value=False, default_value=[],
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='not_tags', full_name='pb.SearchRequest.not_tags', index=50,
|
||||||
|
number=51, type=9, cpp_type=9, label=3,
|
||||||
|
has_default_value=False, default_value=[],
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='has_channel_signature', full_name='pb.SearchRequest.has_channel_signature', index=51,
|
||||||
|
number=52, type=8, cpp_type=7, label=1,
|
||||||
|
has_default_value=False, default_value=False,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='has_source', full_name='pb.SearchRequest.has_source', index=52,
|
||||||
|
number=53, type=11, cpp_type=10, label=1,
|
||||||
|
has_default_value=False, default_value=None,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='limit_claims_per_channel', full_name='pb.SearchRequest.limit_claims_per_channel', index=53,
|
||||||
|
number=54, type=13, cpp_type=3, label=1,
|
||||||
|
has_default_value=False, default_value=0,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='any_languages', full_name='pb.SearchRequest.any_languages', index=54,
|
||||||
|
number=55, type=9, cpp_type=9, label=3,
|
||||||
|
has_default_value=False, default_value=[],
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='all_languages', full_name='pb.SearchRequest.all_languages', index=55,
|
||||||
|
number=56, type=9, cpp_type=9, label=3,
|
||||||
|
has_default_value=False, default_value=[],
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='remove_duplicates', full_name='pb.SearchRequest.remove_duplicates', index=56,
|
||||||
|
number=57, type=8, cpp_type=7, label=1,
|
||||||
|
has_default_value=False, default_value=False,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='no_totals', full_name='pb.SearchRequest.no_totals', index=57,
|
||||||
|
number=58, type=8, cpp_type=7, label=1,
|
||||||
|
has_default_value=False, default_value=False,
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||||
|
],
|
||||||
|
extensions=[
|
||||||
|
],
|
||||||
|
nested_types=[],
|
||||||
|
enum_types=[
|
||||||
|
],
|
||||||
|
serialized_options=None,
|
||||||
|
is_extendable=False,
|
||||||
|
syntax='proto3',
|
||||||
|
extension_ranges=[],
|
||||||
|
oneofs=[
|
||||||
|
],
|
||||||
|
serialized_start=248,
|
||||||
|
serialized_end=1940,
|
||||||
|
)
|
||||||
|
|
||||||
|
_RANGEFIELD.fields_by_name['op'].enum_type = _RANGEFIELD_OP
|
||||||
|
_RANGEFIELD_OP.containing_type = _RANGEFIELD
|
||||||
|
_SEARCHREQUEST.fields_by_name['claim_id'].message_type = _INVERTIBLEFIELD
|
||||||
|
_SEARCHREQUEST.fields_by_name['channel_id'].message_type = _INVERTIBLEFIELD
|
||||||
|
_SEARCHREQUEST.fields_by_name['tx_position'].message_type = _RANGEFIELD
|
||||||
|
_SEARCHREQUEST.fields_by_name['amount'].message_type = _RANGEFIELD
|
||||||
|
_SEARCHREQUEST.fields_by_name['timestamp'].message_type = _RANGEFIELD
|
||||||
|
_SEARCHREQUEST.fields_by_name['creation_timestamp'].message_type = _RANGEFIELD
|
||||||
|
_SEARCHREQUEST.fields_by_name['height'].message_type = _RANGEFIELD
|
||||||
|
_SEARCHREQUEST.fields_by_name['creation_height'].message_type = _RANGEFIELD
|
||||||
|
_SEARCHREQUEST.fields_by_name['activation_height'].message_type = _RANGEFIELD
|
||||||
|
_SEARCHREQUEST.fields_by_name['expiration_height'].message_type = _RANGEFIELD
|
||||||
|
_SEARCHREQUEST.fields_by_name['release_time'].message_type = _RANGEFIELD
|
||||||
|
_SEARCHREQUEST.fields_by_name['repost_count'].message_type = _RANGEFIELD
|
||||||
|
_SEARCHREQUEST.fields_by_name['fee_amount'].message_type = _RANGEFIELD
|
||||||
|
_SEARCHREQUEST.fields_by_name['duration'].message_type = _RANGEFIELD
|
||||||
|
_SEARCHREQUEST.fields_by_name['censor_type'].message_type = _RANGEFIELD
|
||||||
|
_SEARCHREQUEST.fields_by_name['channel_join'].message_type = _RANGEFIELD
|
||||||
|
_SEARCHREQUEST.fields_by_name['is_signature_valid'].message_type = _BOOLVALUE
|
||||||
|
_SEARCHREQUEST.fields_by_name['effective_amount'].message_type = _RANGEFIELD
|
||||||
|
_SEARCHREQUEST.fields_by_name['support_amount'].message_type = _RANGEFIELD
|
||||||
|
_SEARCHREQUEST.fields_by_name['trending_group'].message_type = _RANGEFIELD
|
||||||
|
_SEARCHREQUEST.fields_by_name['trending_mixed'].message_type = _RANGEFIELD
|
||||||
|
_SEARCHREQUEST.fields_by_name['trending_local'].message_type = _RANGEFIELD
|
||||||
|
_SEARCHREQUEST.fields_by_name['trending_global'].message_type = _RANGEFIELD
|
||||||
|
_SEARCHREQUEST.fields_by_name['tx_nout'].message_type = _UINT32VALUE
|
||||||
|
_SEARCHREQUEST.fields_by_name['has_source'].message_type = _BOOLVALUE
|
||||||
|
DESCRIPTOR.message_types_by_name['InvertibleField'] = _INVERTIBLEFIELD
|
||||||
|
DESCRIPTOR.message_types_by_name['BoolValue'] = _BOOLVALUE
|
||||||
|
DESCRIPTOR.message_types_by_name['UInt32Value'] = _UINT32VALUE
|
||||||
|
DESCRIPTOR.message_types_by_name['RangeField'] = _RANGEFIELD
|
||||||
|
DESCRIPTOR.message_types_by_name['SearchRequest'] = _SEARCHREQUEST
|
||||||
|
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||||
|
|
||||||
|
InvertibleField = _reflection.GeneratedProtocolMessageType('InvertibleField', (_message.Message,), {
|
||||||
|
'DESCRIPTOR' : _INVERTIBLEFIELD,
|
||||||
|
'__module__' : 'hub_pb2'
|
||||||
|
# @@protoc_insertion_point(class_scope:pb.InvertibleField)
|
||||||
|
})
|
||||||
|
_sym_db.RegisterMessage(InvertibleField)
|
||||||
|
|
||||||
|
BoolValue = _reflection.GeneratedProtocolMessageType('BoolValue', (_message.Message,), {
|
||||||
|
'DESCRIPTOR' : _BOOLVALUE,
|
||||||
|
'__module__' : 'hub_pb2'
|
||||||
|
# @@protoc_insertion_point(class_scope:pb.BoolValue)
|
||||||
|
})
|
||||||
|
_sym_db.RegisterMessage(BoolValue)
|
||||||
|
|
||||||
|
UInt32Value = _reflection.GeneratedProtocolMessageType('UInt32Value', (_message.Message,), {
|
||||||
|
'DESCRIPTOR' : _UINT32VALUE,
|
||||||
|
'__module__' : 'hub_pb2'
|
||||||
|
# @@protoc_insertion_point(class_scope:pb.UInt32Value)
|
||||||
|
})
|
||||||
|
_sym_db.RegisterMessage(UInt32Value)
|
||||||
|
|
||||||
|
RangeField = _reflection.GeneratedProtocolMessageType('RangeField', (_message.Message,), {
|
||||||
|
'DESCRIPTOR' : _RANGEFIELD,
|
||||||
|
'__module__' : 'hub_pb2'
|
||||||
|
# @@protoc_insertion_point(class_scope:pb.RangeField)
|
||||||
|
})
|
||||||
|
_sym_db.RegisterMessage(RangeField)
|
||||||
|
|
||||||
|
SearchRequest = _reflection.GeneratedProtocolMessageType('SearchRequest', (_message.Message,), {
|
||||||
|
'DESCRIPTOR' : _SEARCHREQUEST,
|
||||||
|
'__module__' : 'hub_pb2'
|
||||||
|
# @@protoc_insertion_point(class_scope:pb.SearchRequest)
|
||||||
|
})
|
||||||
|
_sym_db.RegisterMessage(SearchRequest)
|
||||||
|
|
||||||
|
|
||||||
|
DESCRIPTOR._options = None
|
||||||
|
|
||||||
|
_HUB = _descriptor.ServiceDescriptor(
|
||||||
|
name='Hub',
|
||||||
|
full_name='pb.Hub',
|
||||||
|
file=DESCRIPTOR,
|
||||||
|
index=0,
|
||||||
|
serialized_options=None,
|
||||||
|
create_key=_descriptor._internal_create_key,
|
||||||
|
serialized_start=1942,
|
||||||
|
serialized_end=1991,
|
||||||
|
methods=[
|
||||||
|
_descriptor.MethodDescriptor(
|
||||||
|
name='Search',
|
||||||
|
full_name='pb.Hub.Search',
|
||||||
|
index=0,
|
||||||
|
containing_service=None,
|
||||||
|
input_type=_SEARCHREQUEST,
|
||||||
|
output_type=result__pb2._OUTPUTS,
|
||||||
|
serialized_options=None,
|
||||||
|
create_key=_descriptor._internal_create_key,
|
||||||
|
),
|
||||||
|
])
|
||||||
|
_sym_db.RegisterServiceDescriptor(_HUB)
|
||||||
|
|
||||||
|
DESCRIPTOR.services_by_name['Hub'] = _HUB
|
||||||
|
|
||||||
|
# @@protoc_insertion_point(module_scope)
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
|
||||||
|
"""Client and server classes corresponding to protobuf-defined services."""
|
||||||
|
import grpc
|
||||||
|
|
||||||
|
import lbry.schema.types.v2.hub_pb2 as hub__pb2
|
||||||
|
import lbry.schema.types.v2.result_pb2 as result__pb2
|
||||||
|
|
||||||
|
|
||||||
|
class HubStub(object):
|
||||||
|
"""Missing associated documentation comment in .proto file."""
|
||||||
|
|
||||||
|
def __init__(self, channel):
|
||||||
|
"""Constructor.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
channel: A grpc.Channel.
|
||||||
|
"""
|
||||||
|
self.Search = channel.unary_unary(
|
||||||
|
'/pb.Hub/Search',
|
||||||
|
request_serializer=hub__pb2.SearchRequest.SerializeToString,
|
||||||
|
response_deserializer=result__pb2.Outputs.FromString,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class HubServicer(object):
|
||||||
|
"""Missing associated documentation comment in .proto file."""
|
||||||
|
|
||||||
|
def Search(self, request, context):
|
||||||
|
"""Missing associated documentation comment in .proto file."""
|
||||||
|
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||||
|
context.set_details('Method not implemented!')
|
||||||
|
raise NotImplementedError('Method not implemented!')
|
||||||
|
|
||||||
|
|
||||||
|
def add_HubServicer_to_server(servicer, server):
|
||||||
|
rpc_method_handlers = {
|
||||||
|
'Search': grpc.unary_unary_rpc_method_handler(
|
||||||
|
servicer.Search,
|
||||||
|
request_deserializer=hub__pb2.SearchRequest.FromString,
|
||||||
|
response_serializer=result__pb2.Outputs.SerializeToString,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
generic_handler = grpc.method_handlers_generic_handler(
|
||||||
|
'pb.Hub', rpc_method_handlers)
|
||||||
|
server.add_generic_rpc_handlers((generic_handler,))
|
||||||
|
|
||||||
|
|
||||||
|
# This class is part of an EXPERIMENTAL API.
|
||||||
|
class Hub(object):
|
||||||
|
"""Missing associated documentation comment in .proto file."""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def Search(request,
|
||||||
|
target,
|
||||||
|
options=(),
|
||||||
|
channel_credentials=None,
|
||||||
|
call_credentials=None,
|
||||||
|
insecure=False,
|
||||||
|
compression=None,
|
||||||
|
wait_for_ready=None,
|
||||||
|
timeout=None,
|
||||||
|
metadata=None):
|
||||||
|
return grpc.experimental.unary_unary(request, target, '/pb.Hub/Search',
|
||||||
|
hub__pb2.SearchRequest.SerializeToString,
|
||||||
|
result__pb2.Outputs.FromString,
|
||||||
|
options, channel_credentials,
|
||||||
|
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
|
||||||
@@ -0,0 +1,76 @@
|
|||||||
|
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||||
|
# source: support.proto
|
||||||
|
|
||||||
|
import sys
|
||||||
|
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||||
|
from google.protobuf import descriptor as _descriptor
|
||||||
|
from google.protobuf import message as _message
|
||||||
|
from google.protobuf import reflection as _reflection
|
||||||
|
from google.protobuf import symbol_database as _symbol_database
|
||||||
|
from google.protobuf import descriptor_pb2
|
||||||
|
# @@protoc_insertion_point(imports)
|
||||||
|
|
||||||
|
_sym_db = _symbol_database.Default()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||||
|
name='support.proto',
|
||||||
|
package='pb',
|
||||||
|
syntax='proto3',
|
||||||
|
serialized_pb=_b('\n\rsupport.proto\x12\x02pb\")\n\x07Support\x12\r\n\x05\x65moji\x18\x01 \x01(\t\x12\x0f\n\x07\x63omment\x18\x02 \x01(\tb\x06proto3')
|
||||||
|
)
|
||||||
|
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
_SUPPORT = _descriptor.Descriptor(
|
||||||
|
name='Support',
|
||||||
|
full_name='pb.Support',
|
||||||
|
filename=None,
|
||||||
|
file=DESCRIPTOR,
|
||||||
|
containing_type=None,
|
||||||
|
fields=[
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='emoji', full_name='pb.Support.emoji', index=0,
|
||||||
|
number=1, type=9, cpp_type=9, label=1,
|
||||||
|
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
options=None),
|
||||||
|
_descriptor.FieldDescriptor(
|
||||||
|
name='comment', full_name='pb.Support.comment', index=1,
|
||||||
|
number=2, type=9, cpp_type=9, label=1,
|
||||||
|
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||||
|
message_type=None, enum_type=None, containing_type=None,
|
||||||
|
is_extension=False, extension_scope=None,
|
||||||
|
options=None),
|
||||||
|
],
|
||||||
|
extensions=[
|
||||||
|
],
|
||||||
|
nested_types=[],
|
||||||
|
enum_types=[
|
||||||
|
],
|
||||||
|
options=None,
|
||||||
|
is_extendable=False,
|
||||||
|
syntax='proto3',
|
||||||
|
extension_ranges=[],
|
||||||
|
oneofs=[
|
||||||
|
],
|
||||||
|
serialized_start=21,
|
||||||
|
serialized_end=62,
|
||||||
|
)
|
||||||
|
|
||||||
|
DESCRIPTOR.message_types_by_name['Support'] = _SUPPORT
|
||||||
|
|
||||||
|
Support = _reflection.GeneratedProtocolMessageType('Support', (_message.Message,), dict(
|
||||||
|
DESCRIPTOR = _SUPPORT,
|
||||||
|
__module__ = 'support_pb2'
|
||||||
|
# @@protoc_insertion_point(class_scope:pb.Support)
|
||||||
|
))
|
||||||
|
_sym_db.RegisterMessage(Support)
|
||||||
|
|
||||||
|
|
||||||
|
# @@protoc_insertion_point(module_scope)
|
||||||
+10
-9
@@ -22,8 +22,7 @@ def _create_url_regex():
|
|||||||
return _group(
|
return _group(
|
||||||
_named(name+"_name", prefix + invalid_names_regex) +
|
_named(name+"_name", prefix + invalid_names_regex) +
|
||||||
_oneof(
|
_oneof(
|
||||||
_group('#' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
|
_group('[:#]' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
|
||||||
_group(':' + _named(name+"_sequence", '[1-9][0-9]*')),
|
|
||||||
_group(r'\$' + _named(name+"_amount_order", '[1-9][0-9]*'))
|
_group(r'\$' + _named(name+"_amount_order", '[1-9][0-9]*'))
|
||||||
) + '?'
|
) + '?'
|
||||||
)
|
)
|
||||||
@@ -50,28 +49,31 @@ def normalize_name(name):
|
|||||||
class PathSegment(NamedTuple):
|
class PathSegment(NamedTuple):
|
||||||
name: str
|
name: str
|
||||||
claim_id: str = None
|
claim_id: str = None
|
||||||
sequence: int = None
|
|
||||||
amount_order: int = None
|
amount_order: int = None
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def normalized(self):
|
def normalized(self):
|
||||||
return normalize_name(self.name)
|
return normalize_name(self.name)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def is_shortid(self):
|
||||||
|
return self.claim_id is not None and len(self.claim_id) < 40
|
||||||
|
|
||||||
|
@property
|
||||||
|
def is_fullid(self):
|
||||||
|
return self.claim_id is not None and len(self.claim_id) == 40
|
||||||
|
|
||||||
def to_dict(self):
|
def to_dict(self):
|
||||||
q = {'name': self.name}
|
q = {'name': self.name}
|
||||||
if self.claim_id is not None:
|
if self.claim_id is not None:
|
||||||
q['claim_id'] = self.claim_id
|
q['claim_id'] = self.claim_id
|
||||||
if self.sequence is not None:
|
|
||||||
q['sequence'] = self.sequence
|
|
||||||
if self.amount_order is not None:
|
if self.amount_order is not None:
|
||||||
q['amount_order'] = self.amount_order
|
q['amount_order'] = self.amount_order
|
||||||
return q
|
return q
|
||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
if self.claim_id is not None:
|
if self.claim_id is not None:
|
||||||
return f"{self.name}#{self.claim_id}"
|
return f"{self.name}:{self.claim_id}"
|
||||||
elif self.sequence is not None:
|
|
||||||
return f"{self.name}:{self.sequence}"
|
|
||||||
elif self.amount_order is not None:
|
elif self.amount_order is not None:
|
||||||
return f"{self.name}${self.amount_order}"
|
return f"{self.name}${self.amount_order}"
|
||||||
return self.name
|
return self.name
|
||||||
@@ -118,7 +120,6 @@ class URL(NamedTuple):
|
|||||||
segments[segment] = PathSegment(
|
segments[segment] = PathSegment(
|
||||||
parts[f'{segment}_name'],
|
parts[f'{segment}_name'],
|
||||||
parts[f'{segment}_claim_id'],
|
parts[f'{segment}_claim_id'],
|
||||||
parts[f'{segment}_sequence'],
|
|
||||||
parts[f'{segment}_amount_order']
|
parts[f'{segment}_amount_order']
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
+36
-19
@@ -4,6 +4,7 @@ import binascii
|
|||||||
import logging
|
import logging
|
||||||
import typing
|
import typing
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import time
|
||||||
import re
|
import re
|
||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||||
@@ -44,18 +45,25 @@ def random_iv_generator() -> typing.Generator[bytes, None, None]:
|
|||||||
yield os.urandom(AES.block_size // 8)
|
yield os.urandom(AES.block_size // 8)
|
||||||
|
|
||||||
|
|
||||||
def file_reader(file_path: str):
|
def read_bytes(file_path: str, offset: int, to_read: int):
|
||||||
|
with open(file_path, 'rb') as f:
|
||||||
|
f.seek(offset)
|
||||||
|
return f.read(to_read)
|
||||||
|
|
||||||
|
|
||||||
|
async def file_reader(file_path: str):
|
||||||
length = int(os.stat(file_path).st_size)
|
length = int(os.stat(file_path).st_size)
|
||||||
offset = 0
|
offset = 0
|
||||||
|
|
||||||
with open(file_path, 'rb') as stream_file:
|
while offset < length:
|
||||||
while offset < length:
|
bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1)
|
||||||
bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1)
|
if not bytes_to_read:
|
||||||
if not bytes_to_read:
|
break
|
||||||
break
|
blob_bytes = await asyncio.get_event_loop().run_in_executor(
|
||||||
blob_bytes = stream_file.read(bytes_to_read)
|
None, read_bytes, file_path, offset, bytes_to_read
|
||||||
yield blob_bytes
|
)
|
||||||
offset += bytes_to_read
|
yield blob_bytes
|
||||||
|
offset += bytes_to_read
|
||||||
|
|
||||||
|
|
||||||
def sanitize_file_name(dirty_name: str, default_file_name: str = 'lbry_download'):
|
def sanitize_file_name(dirty_name: str, default_file_name: str = 'lbry_download'):
|
||||||
@@ -145,15 +153,19 @@ class StreamDescriptor:
|
|||||||
h.update(self.old_sort_json())
|
h.update(self.old_sort_json())
|
||||||
return h.hexdigest()
|
return h.hexdigest()
|
||||||
|
|
||||||
async def make_sd_blob(self, blob_file_obj: typing.Optional[AbstractBlob] = None,
|
async def make_sd_blob(
|
||||||
old_sort: typing.Optional[bool] = False,
|
self, blob_file_obj: typing.Optional[AbstractBlob] = None, old_sort: typing.Optional[bool] = False,
|
||||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None):
|
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None,
|
||||||
|
added_on: float = None, is_mine: bool = False
|
||||||
|
):
|
||||||
sd_hash = self.calculate_sd_hash() if not old_sort else self.calculate_old_sort_sd_hash()
|
sd_hash = self.calculate_sd_hash() if not old_sort else self.calculate_old_sort_sd_hash()
|
||||||
if not old_sort:
|
if not old_sort:
|
||||||
sd_data = self.as_json()
|
sd_data = self.as_json()
|
||||||
else:
|
else:
|
||||||
sd_data = self.old_sort_json()
|
sd_data = self.old_sort_json()
|
||||||
sd_blob = blob_file_obj or BlobFile(self.loop, sd_hash, len(sd_data), blob_completed_callback, self.blob_dir)
|
sd_blob = blob_file_obj or BlobFile(
|
||||||
|
self.loop, sd_hash, len(sd_data), blob_completed_callback, self.blob_dir, added_on, is_mine
|
||||||
|
)
|
||||||
if blob_file_obj:
|
if blob_file_obj:
|
||||||
blob_file_obj.set_length(len(sd_data))
|
blob_file_obj.set_length(len(sd_data))
|
||||||
if not sd_blob.get_is_verified():
|
if not sd_blob.get_is_verified():
|
||||||
@@ -176,11 +188,11 @@ class StreamDescriptor:
|
|||||||
raise InvalidStreamDescriptorError("Does not decode as valid JSON")
|
raise InvalidStreamDescriptorError("Does not decode as valid JSON")
|
||||||
if decoded['blobs'][-1]['length'] != 0:
|
if decoded['blobs'][-1]['length'] != 0:
|
||||||
raise InvalidStreamDescriptorError("Does not end with a zero-length blob.")
|
raise InvalidStreamDescriptorError("Does not end with a zero-length blob.")
|
||||||
if any([blob_info['length'] == 0 for blob_info in decoded['blobs'][:-1]]):
|
if any(blob_info['length'] == 0 for blob_info in decoded['blobs'][:-1]):
|
||||||
raise InvalidStreamDescriptorError("Contains zero-length data blob")
|
raise InvalidStreamDescriptorError("Contains zero-length data blob")
|
||||||
if 'blob_hash' in decoded['blobs'][-1]:
|
if 'blob_hash' in decoded['blobs'][-1]:
|
||||||
raise InvalidStreamDescriptorError("Stream terminator blob should not have a hash")
|
raise InvalidStreamDescriptorError("Stream terminator blob should not have a hash")
|
||||||
if any([i != blob_info['blob_num'] for i, blob_info in enumerate(decoded['blobs'])]):
|
if any(i != blob_info['blob_num'] for i, blob_info in enumerate(decoded['blobs'])):
|
||||||
raise InvalidStreamDescriptorError("Stream contains out of order or skipped blobs")
|
raise InvalidStreamDescriptorError("Stream contains out of order or skipped blobs")
|
||||||
descriptor = cls(
|
descriptor = cls(
|
||||||
loop, blob_dir,
|
loop, blob_dir,
|
||||||
@@ -245,20 +257,25 @@ class StreamDescriptor:
|
|||||||
iv_generator = iv_generator or random_iv_generator()
|
iv_generator = iv_generator or random_iv_generator()
|
||||||
key = key or os.urandom(AES.block_size // 8)
|
key = key or os.urandom(AES.block_size // 8)
|
||||||
blob_num = -1
|
blob_num = -1
|
||||||
for blob_bytes in file_reader(file_path):
|
added_on = time.time()
|
||||||
|
async for blob_bytes in file_reader(file_path):
|
||||||
blob_num += 1
|
blob_num += 1
|
||||||
blob_info = await BlobFile.create_from_unencrypted(
|
blob_info = await BlobFile.create_from_unencrypted(
|
||||||
loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num, blob_completed_callback
|
loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num, added_on, True, blob_completed_callback
|
||||||
)
|
)
|
||||||
blobs.append(blob_info)
|
blobs.append(blob_info)
|
||||||
blobs.append(
|
blobs.append(
|
||||||
BlobInfo(len(blobs), 0, binascii.hexlify(next(iv_generator)).decode())) # add the stream terminator
|
# add the stream terminator
|
||||||
|
BlobInfo(len(blobs), 0, binascii.hexlify(next(iv_generator)).decode(), None, added_on, True)
|
||||||
|
)
|
||||||
file_name = os.path.basename(file_path)
|
file_name = os.path.basename(file_path)
|
||||||
suggested_file_name = sanitize_file_name(file_name)
|
suggested_file_name = sanitize_file_name(file_name)
|
||||||
descriptor = cls(
|
descriptor = cls(
|
||||||
loop, blob_dir, file_name, binascii.hexlify(key).decode(), suggested_file_name, blobs
|
loop, blob_dir, file_name, binascii.hexlify(key).decode(), suggested_file_name, blobs
|
||||||
)
|
)
|
||||||
sd_blob = await descriptor.make_sd_blob(old_sort=old_sort, blob_completed_callback=blob_completed_callback)
|
sd_blob = await descriptor.make_sd_blob(
|
||||||
|
old_sort=old_sort, blob_completed_callback=blob_completed_callback, added_on=added_on, is_mine=True
|
||||||
|
)
|
||||||
descriptor.sd_hash = sd_blob.blob_hash
|
descriptor.sd_hash = sd_blob.blob_hash
|
||||||
return descriptor
|
return descriptor
|
||||||
|
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ class StreamDownloader:
|
|||||||
async def cached_read_blob(blob_info: 'BlobInfo') -> bytes:
|
async def cached_read_blob(blob_info: 'BlobInfo') -> bytes:
|
||||||
return await self.read_blob(blob_info, 2)
|
return await self.read_blob(blob_info, 2)
|
||||||
|
|
||||||
if self.blob_manager.decrypted_blob_lru_cache:
|
if self.blob_manager.decrypted_blob_lru_cache is not None:
|
||||||
cached_read_blob = lru_cache_concurrent(override_lru_cache=self.blob_manager.decrypted_blob_lru_cache)(
|
cached_read_blob = lru_cache_concurrent(override_lru_cache=self.blob_manager.decrypted_blob_lru_cache)(
|
||||||
cached_read_blob
|
cached_read_blob
|
||||||
)
|
)
|
||||||
@@ -51,15 +51,15 @@ class StreamDownloader:
|
|||||||
def _delayed_add_fixed_peers():
|
def _delayed_add_fixed_peers():
|
||||||
self.added_fixed_peers = True
|
self.added_fixed_peers = True
|
||||||
self.peer_queue.put_nowait([
|
self.peer_queue.put_nowait([
|
||||||
make_kademlia_peer(None, address, None, tcp_port=port + 1, allow_localhost=True)
|
make_kademlia_peer(None, address, None, tcp_port=port, allow_localhost=True)
|
||||||
for address, port in addresses
|
for address, port in addresses
|
||||||
])
|
])
|
||||||
|
|
||||||
if not self.config.reflector_servers:
|
if not self.config.fixed_peers:
|
||||||
return
|
return
|
||||||
addresses = [
|
addresses = [
|
||||||
(await resolve_host(url, port + 1, proto='tcp'), port)
|
(await resolve_host(url, port, proto='tcp'), port)
|
||||||
for url, port in self.config.reflector_servers
|
for url, port in self.config.fixed_peers
|
||||||
]
|
]
|
||||||
if 'dht' in self.config.components_to_skip or not self.node or not \
|
if 'dht' in self.config.components_to_skip or not self.node or not \
|
||||||
len(self.node.protocol.routing_table.get_peers()) > 0:
|
len(self.node.protocol.routing_table.get_peers()) > 0:
|
||||||
@@ -92,8 +92,8 @@ class StreamDownloader:
|
|||||||
|
|
||||||
async def start(self, node: typing.Optional['Node'] = None, connection_id: int = 0):
|
async def start(self, node: typing.Optional['Node'] = None, connection_id: int = 0):
|
||||||
# set up peer accumulation
|
# set up peer accumulation
|
||||||
if node:
|
self.node = node or self.node # fixme: this shouldnt be set here!
|
||||||
self.node = node
|
if self.node:
|
||||||
if self.accumulate_task and not self.accumulate_task.done():
|
if self.accumulate_task and not self.accumulate_task.done():
|
||||||
self.accumulate_task.cancel()
|
self.accumulate_task.cancel()
|
||||||
_, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue)
|
_, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue)
|
||||||
|
|||||||
+83
-177
@@ -3,9 +3,8 @@ import asyncio
|
|||||||
import time
|
import time
|
||||||
import typing
|
import typing
|
||||||
import logging
|
import logging
|
||||||
import binascii
|
from typing import Optional
|
||||||
from aiohttp.web import Request, StreamResponse, HTTPRequestRangeNotSatisfiable
|
from aiohttp.web import Request, StreamResponse, HTTPRequestRangeNotSatisfiable
|
||||||
from lbry.utils import generate_id
|
|
||||||
from lbry.error import DownloadSDTimeoutError
|
from lbry.error import DownloadSDTimeoutError
|
||||||
from lbry.schema.mime_types import guess_media_type
|
from lbry.schema.mime_types import guess_media_type
|
||||||
from lbry.stream.downloader import StreamDownloader
|
from lbry.stream.downloader import StreamDownloader
|
||||||
@@ -13,6 +12,7 @@ from lbry.stream.descriptor import StreamDescriptor, sanitize_file_name
|
|||||||
from lbry.stream.reflector.client import StreamReflectorClient
|
from lbry.stream.reflector.client import StreamReflectorClient
|
||||||
from lbry.extras.daemon.storage import StoredContentClaim
|
from lbry.extras.daemon.storage import StoredContentClaim
|
||||||
from lbry.blob import MAX_BLOB_SIZE
|
from lbry.blob import MAX_BLOB_SIZE
|
||||||
|
from lbry.file.source import ManagedDownloadSource
|
||||||
|
|
||||||
if typing.TYPE_CHECKING:
|
if typing.TYPE_CHECKING:
|
||||||
from lbry.conf import Config
|
from lbry.conf import Config
|
||||||
@@ -40,76 +40,35 @@ async def get_next_available_file_name(loop: asyncio.AbstractEventLoop, download
|
|||||||
return await loop.run_in_executor(None, _get_next_available_file_name, download_directory, file_name)
|
return await loop.run_in_executor(None, _get_next_available_file_name, download_directory, file_name)
|
||||||
|
|
||||||
|
|
||||||
class ManagedStream:
|
class ManagedStream(ManagedDownloadSource):
|
||||||
STATUS_RUNNING = "running"
|
|
||||||
STATUS_STOPPED = "stopped"
|
|
||||||
STATUS_FINISHED = "finished"
|
|
||||||
|
|
||||||
SAVING_ID = 1
|
|
||||||
STREAMING_ID = 2
|
|
||||||
|
|
||||||
__slots__ = [
|
|
||||||
'loop',
|
|
||||||
'config',
|
|
||||||
'blob_manager',
|
|
||||||
'sd_hash',
|
|
||||||
'download_directory',
|
|
||||||
'_file_name',
|
|
||||||
'_added_on',
|
|
||||||
'_status',
|
|
||||||
'stream_claim_info',
|
|
||||||
'download_id',
|
|
||||||
'rowid',
|
|
||||||
'content_fee',
|
|
||||||
'purchase_receipt',
|
|
||||||
'downloader',
|
|
||||||
'analytics_manager',
|
|
||||||
'fully_reflected',
|
|
||||||
'file_output_task',
|
|
||||||
'delayed_stop_task',
|
|
||||||
'streaming_responses',
|
|
||||||
'streaming',
|
|
||||||
'_running',
|
|
||||||
'saving',
|
|
||||||
'finished_writing',
|
|
||||||
'started_writing',
|
|
||||||
'finished_write_attempt'
|
|
||||||
]
|
|
||||||
|
|
||||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||||
sd_hash: str, download_directory: typing.Optional[str] = None, file_name: typing.Optional[str] = None,
|
sd_hash: str, download_directory: Optional[str] = None, file_name: Optional[str] = None,
|
||||||
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional[StoredContentClaim] = None,
|
status: Optional[str] = ManagedDownloadSource.STATUS_STOPPED,
|
||||||
download_id: typing.Optional[str] = None, rowid: typing.Optional[int] = None,
|
claim: Optional[StoredContentClaim] = None,
|
||||||
descriptor: typing.Optional[StreamDescriptor] = None,
|
download_id: Optional[str] = None, rowid: Optional[int] = None,
|
||||||
content_fee: typing.Optional['Transaction'] = None,
|
descriptor: Optional[StreamDescriptor] = None,
|
||||||
analytics_manager: typing.Optional['AnalyticsManager'] = None,
|
content_fee: Optional['Transaction'] = None,
|
||||||
added_on: typing.Optional[int] = None):
|
analytics_manager: Optional['AnalyticsManager'] = None,
|
||||||
self.loop = loop
|
added_on: Optional[int] = None):
|
||||||
self.config = config
|
super().__init__(loop, config, blob_manager.storage, sd_hash, file_name, download_directory, status, claim,
|
||||||
|
download_id, rowid, content_fee, analytics_manager, added_on)
|
||||||
self.blob_manager = blob_manager
|
self.blob_manager = blob_manager
|
||||||
self.sd_hash = sd_hash
|
|
||||||
self.download_directory = download_directory
|
|
||||||
self._file_name = file_name
|
|
||||||
self._status = status
|
|
||||||
self.stream_claim_info = claim
|
|
||||||
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
|
|
||||||
self.rowid = rowid
|
|
||||||
self.content_fee = content_fee
|
|
||||||
self.purchase_receipt = None
|
self.purchase_receipt = None
|
||||||
self._added_on = added_on
|
|
||||||
self.downloader = StreamDownloader(self.loop, self.config, self.blob_manager, sd_hash, descriptor)
|
self.downloader = StreamDownloader(self.loop, self.config, self.blob_manager, sd_hash, descriptor)
|
||||||
self.analytics_manager = analytics_manager
|
self.analytics_manager = analytics_manager
|
||||||
|
|
||||||
self.fully_reflected = asyncio.Event(loop=self.loop)
|
self.reflector_progress = 0
|
||||||
|
self.uploading_to_reflector = False
|
||||||
self.file_output_task: typing.Optional[asyncio.Task] = None
|
self.file_output_task: typing.Optional[asyncio.Task] = None
|
||||||
self.delayed_stop_task: typing.Optional[asyncio.Task] = None
|
self.delayed_stop_task: typing.Optional[asyncio.Task] = None
|
||||||
self.streaming_responses: typing.List[typing.Tuple[Request, StreamResponse]] = []
|
self.streaming_responses: typing.List[typing.Tuple[Request, StreamResponse]] = []
|
||||||
|
self.fully_reflected = asyncio.Event(loop=self.loop)
|
||||||
self.streaming = asyncio.Event(loop=self.loop)
|
self.streaming = asyncio.Event(loop=self.loop)
|
||||||
self._running = asyncio.Event(loop=self.loop)
|
self._running = asyncio.Event(loop=self.loop)
|
||||||
self.saving = asyncio.Event(loop=self.loop)
|
|
||||||
self.finished_writing = asyncio.Event(loop=self.loop)
|
@property
|
||||||
self.started_writing = asyncio.Event(loop=self.loop)
|
def sd_hash(self) -> str:
|
||||||
self.finished_write_attempt = asyncio.Event(loop=self.loop)
|
return self.identifier
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def is_fully_reflected(self) -> bool:
|
def is_fully_reflected(self) -> bool:
|
||||||
@@ -124,17 +83,9 @@ class ManagedStream:
|
|||||||
return self.descriptor.stream_hash
|
return self.descriptor.stream_hash
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def file_name(self) -> typing.Optional[str]:
|
def file_name(self) -> Optional[str]:
|
||||||
return self._file_name or (self.descriptor.suggested_file_name if self.descriptor else None)
|
return self._file_name or (self.descriptor.suggested_file_name if self.descriptor else None)
|
||||||
|
|
||||||
@property
|
|
||||||
def added_on(self) -> typing.Optional[int]:
|
|
||||||
return self._added_on
|
|
||||||
|
|
||||||
@property
|
|
||||||
def status(self) -> str:
|
|
||||||
return self._status
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def written_bytes(self) -> int:
|
def written_bytes(self) -> int:
|
||||||
return 0 if not self.output_file_exists else os.stat(self.full_path).st_size
|
return 0 if not self.output_file_exists else os.stat(self.full_path).st_size
|
||||||
@@ -152,55 +103,6 @@ class ManagedStream:
|
|||||||
self._status = status
|
self._status = status
|
||||||
await self.blob_manager.storage.change_file_status(self.stream_hash, status)
|
await self.blob_manager.storage.change_file_status(self.stream_hash, status)
|
||||||
|
|
||||||
@property
|
|
||||||
def finished(self) -> bool:
|
|
||||||
return self.status == self.STATUS_FINISHED
|
|
||||||
|
|
||||||
@property
|
|
||||||
def running(self) -> bool:
|
|
||||||
return self.status == self.STATUS_RUNNING
|
|
||||||
|
|
||||||
@property
|
|
||||||
def claim_id(self) -> typing.Optional[str]:
|
|
||||||
return None if not self.stream_claim_info else self.stream_claim_info.claim_id
|
|
||||||
|
|
||||||
@property
|
|
||||||
def txid(self) -> typing.Optional[str]:
|
|
||||||
return None if not self.stream_claim_info else self.stream_claim_info.txid
|
|
||||||
|
|
||||||
@property
|
|
||||||
def nout(self) -> typing.Optional[int]:
|
|
||||||
return None if not self.stream_claim_info else self.stream_claim_info.nout
|
|
||||||
|
|
||||||
@property
|
|
||||||
def outpoint(self) -> typing.Optional[str]:
|
|
||||||
return None if not self.stream_claim_info else self.stream_claim_info.outpoint
|
|
||||||
|
|
||||||
@property
|
|
||||||
def claim_height(self) -> typing.Optional[int]:
|
|
||||||
return None if not self.stream_claim_info else self.stream_claim_info.height
|
|
||||||
|
|
||||||
@property
|
|
||||||
def channel_claim_id(self) -> typing.Optional[str]:
|
|
||||||
return None if not self.stream_claim_info else self.stream_claim_info.channel_claim_id
|
|
||||||
|
|
||||||
@property
|
|
||||||
def channel_name(self) -> typing.Optional[str]:
|
|
||||||
return None if not self.stream_claim_info else self.stream_claim_info.channel_name
|
|
||||||
|
|
||||||
@property
|
|
||||||
def claim_name(self) -> typing.Optional[str]:
|
|
||||||
return None if not self.stream_claim_info else self.stream_claim_info.claim_name
|
|
||||||
|
|
||||||
@property
|
|
||||||
def metadata(self) -> typing.Optional[typing.Dict]:
|
|
||||||
return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict()
|
|
||||||
|
|
||||||
@property
|
|
||||||
def metadata_protobuf(self) -> bytes:
|
|
||||||
if self.stream_claim_info:
|
|
||||||
return binascii.hexlify(self.stream_claim_info.claim.to_bytes())
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def blobs_completed(self) -> int:
|
def blobs_completed(self) -> int:
|
||||||
return sum([1 if b.blob_hash in self.blob_manager.completed_blob_hashes else 0
|
return sum([1 if b.blob_hash in self.blob_manager.completed_blob_hashes else 0
|
||||||
@@ -214,39 +116,34 @@ class ManagedStream:
|
|||||||
def blobs_remaining(self) -> int:
|
def blobs_remaining(self) -> int:
|
||||||
return self.blobs_in_stream - self.blobs_completed
|
return self.blobs_in_stream - self.blobs_completed
|
||||||
|
|
||||||
@property
|
|
||||||
def full_path(self) -> typing.Optional[str]:
|
|
||||||
return os.path.join(self.download_directory, os.path.basename(self.file_name)) \
|
|
||||||
if self.file_name and self.download_directory else None
|
|
||||||
|
|
||||||
@property
|
|
||||||
def output_file_exists(self):
|
|
||||||
return os.path.isfile(self.full_path) if self.full_path else False
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def mime_type(self):
|
def mime_type(self):
|
||||||
return guess_media_type(os.path.basename(self.descriptor.suggested_file_name))[0]
|
return guess_media_type(os.path.basename(self.descriptor.suggested_file_name))[0]
|
||||||
|
|
||||||
@classmethod
|
@property
|
||||||
async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
def download_path(self):
|
||||||
file_path: str, key: typing.Optional[bytes] = None,
|
return f"{self.download_directory}/{self._file_name}" if self.download_directory and self._file_name else None
|
||||||
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedStream':
|
|
||||||
"""
|
|
||||||
Generate a stream from a file and save it to the db
|
|
||||||
"""
|
|
||||||
descriptor = await StreamDescriptor.create_stream(
|
|
||||||
loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
|
|
||||||
blob_completed_callback=blob_manager.blob_completed
|
|
||||||
)
|
|
||||||
await blob_manager.storage.store_stream(
|
|
||||||
blob_manager.get_blob(descriptor.sd_hash), descriptor
|
|
||||||
)
|
|
||||||
row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path),
|
|
||||||
os.path.dirname(file_path), 0)
|
|
||||||
return cls(loop, config, blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
|
|
||||||
os.path.basename(file_path), status=cls.STATUS_FINISHED, rowid=row_id, descriptor=descriptor)
|
|
||||||
|
|
||||||
async def start(self, node: typing.Optional['Node'] = None, timeout: typing.Optional[float] = None,
|
# @classmethod
|
||||||
|
# async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config',
|
||||||
|
# file_path: str, key: Optional[bytes] = None,
|
||||||
|
# iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedDownloadSource':
|
||||||
|
# """
|
||||||
|
# Generate a stream from a file and save it to the db
|
||||||
|
# """
|
||||||
|
# descriptor = await StreamDescriptor.create_stream(
|
||||||
|
# loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
|
||||||
|
# blob_completed_callback=blob_manager.blob_completed
|
||||||
|
# )
|
||||||
|
# await blob_manager.storage.store_stream(
|
||||||
|
# blob_manager.get_blob(descriptor.sd_hash), descriptor
|
||||||
|
# )
|
||||||
|
# row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path),
|
||||||
|
# os.path.dirname(file_path), 0)
|
||||||
|
# return cls(loop, config, blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
|
||||||
|
# os.path.basename(file_path), status=cls.STATUS_FINISHED, rowid=row_id, descriptor=descriptor)
|
||||||
|
|
||||||
|
async def start(self, timeout: Optional[float] = None,
|
||||||
save_now: bool = False):
|
save_now: bool = False):
|
||||||
timeout = timeout or self.config.download_timeout
|
timeout = timeout or self.config.download_timeout
|
||||||
if self._running.is_set():
|
if self._running.is_set():
|
||||||
@@ -254,7 +151,7 @@ class ManagedStream:
|
|||||||
log.info("start downloader for stream (sd hash: %s)", self.sd_hash)
|
log.info("start downloader for stream (sd hash: %s)", self.sd_hash)
|
||||||
self._running.set()
|
self._running.set()
|
||||||
try:
|
try:
|
||||||
await asyncio.wait_for(self.downloader.start(node), timeout, loop=self.loop)
|
await asyncio.wait_for(self.downloader.start(), timeout, loop=self.loop)
|
||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
self._running.clear()
|
self._running.clear()
|
||||||
raise DownloadSDTimeoutError(self.sd_hash)
|
raise DownloadSDTimeoutError(self.sd_hash)
|
||||||
@@ -264,6 +161,11 @@ class ManagedStream:
|
|||||||
self.delayed_stop_task = self.loop.create_task(self._delayed_stop())
|
self.delayed_stop_task = self.loop.create_task(self._delayed_stop())
|
||||||
if not await self.blob_manager.storage.file_exists(self.sd_hash):
|
if not await self.blob_manager.storage.file_exists(self.sd_hash):
|
||||||
if save_now:
|
if save_now:
|
||||||
|
if not self._file_name:
|
||||||
|
self._file_name = await get_next_available_file_name(
|
||||||
|
self.loop, self.download_directory,
|
||||||
|
self._file_name or sanitize_file_name(self.descriptor.suggested_file_name)
|
||||||
|
)
|
||||||
file_name, download_dir = self._file_name, self.download_directory
|
file_name, download_dir = self._file_name, self.download_directory
|
||||||
else:
|
else:
|
||||||
file_name, download_dir = None, None
|
file_name, download_dir = None, None
|
||||||
@@ -283,7 +185,7 @@ class ManagedStream:
|
|||||||
if (finished and self.status != self.STATUS_FINISHED) or self.status == self.STATUS_RUNNING:
|
if (finished and self.status != self.STATUS_FINISHED) or self.status == self.STATUS_RUNNING:
|
||||||
await self.update_status(self.STATUS_FINISHED if finished else self.STATUS_STOPPED)
|
await self.update_status(self.STATUS_FINISHED if finished else self.STATUS_STOPPED)
|
||||||
|
|
||||||
async def _aiter_read_stream(self, start_blob_num: typing.Optional[int] = 0, connection_id: int = 0)\
|
async def _aiter_read_stream(self, start_blob_num: Optional[int] = 0, connection_id: int = 0)\
|
||||||
-> typing.AsyncIterator[typing.Tuple['BlobInfo', bytes]]:
|
-> typing.AsyncIterator[typing.Tuple['BlobInfo', bytes]]:
|
||||||
if start_blob_num >= len(self.descriptor.blobs[:-1]):
|
if start_blob_num >= len(self.descriptor.blobs[:-1]):
|
||||||
raise IndexError(start_blob_num)
|
raise IndexError(start_blob_num)
|
||||||
@@ -295,13 +197,13 @@ class ManagedStream:
|
|||||||
decrypted = await self.downloader.read_blob(blob_info, connection_id)
|
decrypted = await self.downloader.read_blob(blob_info, connection_id)
|
||||||
yield (blob_info, decrypted)
|
yield (blob_info, decrypted)
|
||||||
|
|
||||||
async def stream_file(self, request: Request, node: typing.Optional['Node'] = None) -> StreamResponse:
|
async def stream_file(self, request: Request) -> StreamResponse:
|
||||||
log.info("stream file to browser for lbry://%s#%s (sd hash %s...)", self.claim_name, self.claim_id,
|
log.info("stream file to browser for lbry://%s#%s (sd hash %s...)", self.claim_name, self.claim_id,
|
||||||
self.sd_hash[:6])
|
self.sd_hash[:6])
|
||||||
headers, size, skip_blobs, first_blob_start_offset = self._prepare_range_response_headers(
|
headers, size, skip_blobs, first_blob_start_offset = self._prepare_range_response_headers(
|
||||||
request.headers.get('range', 'bytes=0-')
|
request.headers.get('range', 'bytes=0-')
|
||||||
)
|
)
|
||||||
await self.start(node)
|
await self.start()
|
||||||
response = StreamResponse(
|
response = StreamResponse(
|
||||||
status=206,
|
status=206,
|
||||||
headers=headers
|
headers=headers
|
||||||
@@ -339,9 +241,10 @@ class ManagedStream:
|
|||||||
self.streaming.clear()
|
self.streaming.clear()
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _write_decrypted_blob(handle: typing.IO, data: bytes):
|
def _write_decrypted_blob(output_path: str, data: bytes):
|
||||||
handle.write(data)
|
with open(output_path, 'ab') as handle:
|
||||||
handle.flush()
|
handle.write(data)
|
||||||
|
handle.flush()
|
||||||
|
|
||||||
async def _save_file(self, output_path: str):
|
async def _save_file(self, output_path: str):
|
||||||
log.info("save file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id, self.sd_hash[:6],
|
log.info("save file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id, self.sd_hash[:6],
|
||||||
@@ -351,12 +254,12 @@ class ManagedStream:
|
|||||||
self.finished_writing.clear()
|
self.finished_writing.clear()
|
||||||
self.started_writing.clear()
|
self.started_writing.clear()
|
||||||
try:
|
try:
|
||||||
with open(output_path, 'wb') as file_write_handle:
|
open(output_path, 'wb').close() # pylint: disable=consider-using-with
|
||||||
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
|
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
|
||||||
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
|
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
|
||||||
await self.loop.run_in_executor(None, self._write_decrypted_blob, file_write_handle, decrypted)
|
await self.loop.run_in_executor(None, self._write_decrypted_blob, output_path, decrypted)
|
||||||
if not self.started_writing.is_set():
|
if not self.started_writing.is_set():
|
||||||
self.started_writing.set()
|
self.started_writing.set()
|
||||||
await self.update_status(ManagedStream.STATUS_FINISHED)
|
await self.update_status(ManagedStream.STATUS_FINISHED)
|
||||||
if self.analytics_manager:
|
if self.analytics_manager:
|
||||||
self.loop.create_task(self.analytics_manager.send_download_finished(
|
self.loop.create_task(self.analytics_manager.send_download_finished(
|
||||||
@@ -386,9 +289,8 @@ class ManagedStream:
|
|||||||
self.saving.clear()
|
self.saving.clear()
|
||||||
self.finished_write_attempt.set()
|
self.finished_write_attempt.set()
|
||||||
|
|
||||||
async def save_file(self, file_name: typing.Optional[str] = None, download_directory: typing.Optional[str] = None,
|
async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
|
||||||
node: typing.Optional['Node'] = None):
|
await self.start()
|
||||||
await self.start(node)
|
|
||||||
if self.file_output_task and not self.file_output_task.done(): # cancel an already running save task
|
if self.file_output_task and not self.file_output_task.done(): # cancel an already running save task
|
||||||
self.file_output_task.cancel()
|
self.file_output_task.cancel()
|
||||||
self.download_directory = download_directory or self.download_directory or self.config.download_dir
|
self.download_directory = download_directory or self.download_directory or self.config.download_dir
|
||||||
@@ -430,6 +332,7 @@ class ManagedStream:
|
|||||||
sent = []
|
sent = []
|
||||||
protocol = StreamReflectorClient(self.blob_manager, self.descriptor)
|
protocol = StreamReflectorClient(self.blob_manager, self.descriptor)
|
||||||
try:
|
try:
|
||||||
|
self.uploading_to_reflector = True
|
||||||
await self.loop.create_connection(lambda: protocol, host, port)
|
await self.loop.create_connection(lambda: protocol, host, port)
|
||||||
await protocol.send_handshake()
|
await protocol.send_handshake()
|
||||||
sent_sd, needed = await protocol.send_descriptor()
|
sent_sd, needed = await protocol.send_descriptor()
|
||||||
@@ -445,30 +348,33 @@ class ManagedStream:
|
|||||||
]
|
]
|
||||||
log.info("we have %i/%i needed blobs needed by reflector for lbry://%s#%s", len(we_have), len(needed),
|
log.info("we have %i/%i needed blobs needed by reflector for lbry://%s#%s", len(we_have), len(needed),
|
||||||
self.claim_name, self.claim_id)
|
self.claim_name, self.claim_id)
|
||||||
for blob_hash in we_have:
|
for i, blob_hash in enumerate(we_have):
|
||||||
await protocol.send_blob(blob_hash)
|
await protocol.send_blob(blob_hash)
|
||||||
sent.append(blob_hash)
|
sent.append(blob_hash)
|
||||||
|
self.reflector_progress = int((i + 1) / len(we_have) * 100)
|
||||||
except (asyncio.TimeoutError, ValueError):
|
except (asyncio.TimeoutError, ValueError):
|
||||||
return sent
|
return sent
|
||||||
except ConnectionRefusedError:
|
except ConnectionError:
|
||||||
|
return sent
|
||||||
|
except (OSError, Exception) as err:
|
||||||
|
if isinstance(err, asyncio.CancelledError):
|
||||||
|
log.warning("stopped uploading %s#%s to reflector", self.claim_name, self.claim_id)
|
||||||
|
elif isinstance(err, OSError):
|
||||||
|
log.warning(
|
||||||
|
"stopped uploading %s#%s to reflector because blobs were deleted or moved", self.claim_name,
|
||||||
|
self.claim_id
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
log.exception("unexpected error reflecting %s#%s", self.claim_name, self.claim_id)
|
||||||
return sent
|
return sent
|
||||||
finally:
|
finally:
|
||||||
if protocol.transport:
|
if protocol.transport:
|
||||||
protocol.transport.close()
|
protocol.transport.close()
|
||||||
if not self.fully_reflected.is_set():
|
self.uploading_to_reflector = False
|
||||||
self.fully_reflected.set()
|
|
||||||
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
|
|
||||||
return sent
|
return sent
|
||||||
|
|
||||||
def set_claim(self, claim_info: typing.Dict, claim: 'Claim'):
|
async def update_content_claim(self, claim_info: Optional[typing.Dict] = None):
|
||||||
self.stream_claim_info = StoredContentClaim(
|
|
||||||
f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
|
|
||||||
claim_info['name'], claim_info['amount'], claim_info['height'],
|
|
||||||
binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'],
|
|
||||||
claim_info['claim_sequence'], claim_info.get('channel_name')
|
|
||||||
)
|
|
||||||
|
|
||||||
async def update_content_claim(self, claim_info: typing.Optional[typing.Dict] = None):
|
|
||||||
if not claim_info:
|
if not claim_info:
|
||||||
claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash)
|
claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash)
|
||||||
self.set_claim(claim_info, claim_info['value'])
|
self.set_claim(claim_info, claim_info['value'])
|
||||||
|
|||||||
@@ -35,6 +35,8 @@ class StreamReflectorClient(asyncio.Protocol):
|
|||||||
def connection_lost(self, exc: typing.Optional[Exception]):
|
def connection_lost(self, exc: typing.Optional[Exception]):
|
||||||
self.transport = None
|
self.transport = None
|
||||||
self.connected.clear()
|
self.connected.clear()
|
||||||
|
if self.pending_request:
|
||||||
|
self.pending_request.cancel()
|
||||||
if self.reflected_blobs:
|
if self.reflected_blobs:
|
||||||
log.info("Finished sending reflector %i blobs", len(self.reflected_blobs))
|
log.info("Finished sending reflector %i blobs", len(self.reflected_blobs))
|
||||||
|
|
||||||
@@ -56,12 +58,18 @@ class StreamReflectorClient(asyncio.Protocol):
|
|||||||
self.response_buff = b''
|
self.response_buff = b''
|
||||||
return
|
return
|
||||||
|
|
||||||
async def send_request(self, request_dict: typing.Dict):
|
async def send_request(self, request_dict: typing.Dict, timeout: int = 180):
|
||||||
msg = json.dumps(request_dict)
|
msg = json.dumps(request_dict, sort_keys=True)
|
||||||
self.transport.write(msg.encode())
|
|
||||||
try:
|
try:
|
||||||
self.pending_request = self.loop.create_task(self.response_queue.get())
|
self.transport.write(msg.encode())
|
||||||
|
self.pending_request = self.loop.create_task(asyncio.wait_for(self.response_queue.get(), timeout))
|
||||||
return await self.pending_request
|
return await self.pending_request
|
||||||
|
except (AttributeError, asyncio.CancelledError) as err:
|
||||||
|
# attribute error happens when we transport.write after disconnect
|
||||||
|
# cancelled error happens when the pending_request task is cancelled by a disconnect
|
||||||
|
if self.transport:
|
||||||
|
self.transport.close()
|
||||||
|
raise err if isinstance(err, asyncio.CancelledError) else asyncio.CancelledError()
|
||||||
finally:
|
finally:
|
||||||
self.pending_request = None
|
self.pending_request = None
|
||||||
|
|
||||||
@@ -86,8 +94,16 @@ class StreamReflectorClient(asyncio.Protocol):
|
|||||||
needed = response.get('needed_blobs', [])
|
needed = response.get('needed_blobs', [])
|
||||||
sent_sd = False
|
sent_sd = False
|
||||||
if response['send_sd_blob']:
|
if response['send_sd_blob']:
|
||||||
await sd_blob.sendfile(self)
|
try:
|
||||||
received = await self.response_queue.get()
|
sent = await sd_blob.sendfile(self)
|
||||||
|
if sent == -1:
|
||||||
|
log.warning("failed to send sd blob")
|
||||||
|
raise asyncio.CancelledError()
|
||||||
|
received = await asyncio.wait_for(self.response_queue.get(), 30)
|
||||||
|
except asyncio.CancelledError as err:
|
||||||
|
if self.transport:
|
||||||
|
self.transport.close()
|
||||||
|
raise err
|
||||||
if received.get('received_sd_blob'):
|
if received.get('received_sd_blob'):
|
||||||
sent_sd = True
|
sent_sd = True
|
||||||
if not needed:
|
if not needed:
|
||||||
@@ -110,8 +126,16 @@ class StreamReflectorClient(asyncio.Protocol):
|
|||||||
if 'send_blob' not in response:
|
if 'send_blob' not in response:
|
||||||
raise ValueError("I don't know whether to send the blob or not!")
|
raise ValueError("I don't know whether to send the blob or not!")
|
||||||
if response['send_blob']:
|
if response['send_blob']:
|
||||||
await blob.sendfile(self)
|
try:
|
||||||
received = await self.response_queue.get()
|
sent = await blob.sendfile(self)
|
||||||
|
if sent == -1:
|
||||||
|
log.warning("failed to send blob")
|
||||||
|
raise asyncio.CancelledError()
|
||||||
|
received = await asyncio.wait_for(self.response_queue.get(), 30)
|
||||||
|
except asyncio.CancelledError as err:
|
||||||
|
if self.transport:
|
||||||
|
self.transport.close()
|
||||||
|
raise err
|
||||||
if received.get('received_blob'):
|
if received.get('received_blob'):
|
||||||
self.reflected_blobs.append(blob.blob_hash)
|
self.reflected_blobs.append(blob.blob_hash)
|
||||||
log.info("Sent reflector blob %s", blob.blob_hash[:8])
|
log.info("Sent reflector blob %s", blob.blob_hash[:8])
|
||||||
|
|||||||
@@ -15,7 +15,9 @@ log = logging.getLogger(__name__)
|
|||||||
|
|
||||||
|
|
||||||
class ReflectorServerProtocol(asyncio.Protocol):
|
class ReflectorServerProtocol(asyncio.Protocol):
|
||||||
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000):
|
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000,
|
||||||
|
stop_event: asyncio.Event = None, incoming_event: asyncio.Event = None,
|
||||||
|
not_incoming_event: asyncio.Event = None, partial_event: asyncio.Event = None):
|
||||||
self.loop = asyncio.get_event_loop()
|
self.loop = asyncio.get_event_loop()
|
||||||
self.blob_manager = blob_manager
|
self.blob_manager = blob_manager
|
||||||
self.server_task: asyncio.Task = None
|
self.server_task: asyncio.Task = None
|
||||||
@@ -27,11 +29,26 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
|||||||
self.descriptor: typing.Optional['StreamDescriptor'] = None
|
self.descriptor: typing.Optional['StreamDescriptor'] = None
|
||||||
self.sd_blob: typing.Optional['BlobFile'] = None
|
self.sd_blob: typing.Optional['BlobFile'] = None
|
||||||
self.received = []
|
self.received = []
|
||||||
self.incoming = asyncio.Event(loop=self.loop)
|
self.incoming = incoming_event or asyncio.Event(loop=self.loop)
|
||||||
|
self.not_incoming = not_incoming_event or asyncio.Event(loop=self.loop)
|
||||||
|
self.stop_event = stop_event or asyncio.Event(loop=self.loop)
|
||||||
self.chunk_size = response_chunk_size
|
self.chunk_size = response_chunk_size
|
||||||
|
self.wait_for_stop_task: typing.Optional[asyncio.Task] = None
|
||||||
|
self.partial_event = partial_event
|
||||||
|
|
||||||
|
async def wait_for_stop(self):
|
||||||
|
await self.stop_event.wait()
|
||||||
|
if self.transport:
|
||||||
|
self.transport.close()
|
||||||
|
|
||||||
def connection_made(self, transport):
|
def connection_made(self, transport):
|
||||||
self.transport = transport
|
self.transport = transport
|
||||||
|
self.wait_for_stop_task = self.loop.create_task(self.wait_for_stop())
|
||||||
|
|
||||||
|
def connection_lost(self, exc):
|
||||||
|
if self.wait_for_stop_task:
|
||||||
|
self.wait_for_stop_task.cancel()
|
||||||
|
self.wait_for_stop_task = None
|
||||||
|
|
||||||
def data_received(self, data: bytes):
|
def data_received(self, data: bytes):
|
||||||
if self.incoming.is_set():
|
if self.incoming.is_set():
|
||||||
@@ -73,6 +90,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
|||||||
self.sd_blob = self.blob_manager.get_blob(request['sd_blob_hash'], request['sd_blob_size'])
|
self.sd_blob = self.blob_manager.get_blob(request['sd_blob_hash'], request['sd_blob_size'])
|
||||||
if not self.sd_blob.get_is_verified():
|
if not self.sd_blob.get_is_verified():
|
||||||
self.writer = self.sd_blob.get_blob_writer(self.transport.get_extra_info('peername'))
|
self.writer = self.sd_blob.get_blob_writer(self.transport.get_extra_info('peername'))
|
||||||
|
self.not_incoming.clear()
|
||||||
self.incoming.set()
|
self.incoming.set()
|
||||||
self.send_response({"send_sd_blob": True})
|
self.send_response({"send_sd_blob": True})
|
||||||
try:
|
try:
|
||||||
@@ -86,6 +104,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
|||||||
self.transport.close()
|
self.transport.close()
|
||||||
finally:
|
finally:
|
||||||
self.incoming.clear()
|
self.incoming.clear()
|
||||||
|
self.not_incoming.set()
|
||||||
self.writer.close_handle()
|
self.writer.close_handle()
|
||||||
self.writer = None
|
self.writer = None
|
||||||
else:
|
else:
|
||||||
@@ -93,13 +112,18 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
|||||||
self.loop, self.blob_manager.blob_dir, self.sd_blob
|
self.loop, self.blob_manager.blob_dir, self.sd_blob
|
||||||
)
|
)
|
||||||
self.incoming.clear()
|
self.incoming.clear()
|
||||||
|
self.not_incoming.set()
|
||||||
if self.writer:
|
if self.writer:
|
||||||
self.writer.close_handle()
|
self.writer.close_handle()
|
||||||
self.writer = None
|
self.writer = None
|
||||||
self.send_response({"send_sd_blob": False, 'needed': [
|
|
||||||
blob.blob_hash for blob in self.descriptor.blobs[:-1]
|
needs = [blob.blob_hash
|
||||||
if not self.blob_manager.get_blob(blob.blob_hash).get_is_verified()
|
for blob in self.descriptor.blobs[:-1]
|
||||||
]})
|
if not self.blob_manager.get_blob(blob.blob_hash).get_is_verified()]
|
||||||
|
if needs and not self.partial_event.is_set():
|
||||||
|
needs = needs[:3]
|
||||||
|
self.partial_event.set()
|
||||||
|
self.send_response({"send_sd_blob": False, 'needed_blobs': needs})
|
||||||
return
|
return
|
||||||
return
|
return
|
||||||
elif self.descriptor:
|
elif self.descriptor:
|
||||||
@@ -112,6 +136,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
|||||||
blob = self.blob_manager.get_blob(request['blob_hash'], request['blob_size'])
|
blob = self.blob_manager.get_blob(request['blob_hash'], request['blob_size'])
|
||||||
if not blob.get_is_verified():
|
if not blob.get_is_verified():
|
||||||
self.writer = blob.get_blob_writer(self.transport.get_extra_info('peername'))
|
self.writer = blob.get_blob_writer(self.transport.get_extra_info('peername'))
|
||||||
|
self.not_incoming.clear()
|
||||||
self.incoming.set()
|
self.incoming.set()
|
||||||
self.send_response({"send_blob": True})
|
self.send_response({"send_blob": True})
|
||||||
try:
|
try:
|
||||||
@@ -120,6 +145,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
|||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
self.send_response({"received_blob": False})
|
self.send_response({"received_blob": False})
|
||||||
self.incoming.clear()
|
self.incoming.clear()
|
||||||
|
self.not_incoming.set()
|
||||||
self.writer.close_handle()
|
self.writer.close_handle()
|
||||||
self.writer = None
|
self.writer = None
|
||||||
else:
|
else:
|
||||||
@@ -130,26 +156,39 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
|||||||
|
|
||||||
|
|
||||||
class ReflectorServer:
|
class ReflectorServer:
|
||||||
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000):
|
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000,
|
||||||
|
stop_event: asyncio.Event = None, incoming_event: asyncio.Event = None,
|
||||||
|
not_incoming_event: asyncio.Event = None, partial_needs=False):
|
||||||
self.loop = asyncio.get_event_loop()
|
self.loop = asyncio.get_event_loop()
|
||||||
self.blob_manager = blob_manager
|
self.blob_manager = blob_manager
|
||||||
self.server_task: typing.Optional[asyncio.Task] = None
|
self.server_task: typing.Optional[asyncio.Task] = None
|
||||||
self.started_listening = asyncio.Event(loop=self.loop)
|
self.started_listening = asyncio.Event(loop=self.loop)
|
||||||
|
self.stopped_listening = asyncio.Event(loop=self.loop)
|
||||||
|
self.incoming_event = incoming_event or asyncio.Event(loop=self.loop)
|
||||||
|
self.not_incoming_event = not_incoming_event or asyncio.Event(loop=self.loop)
|
||||||
self.response_chunk_size = response_chunk_size
|
self.response_chunk_size = response_chunk_size
|
||||||
|
self.stop_event = stop_event
|
||||||
|
self.partial_needs = partial_needs # for testing cases where it doesn't know what it wants
|
||||||
|
|
||||||
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
|
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
|
||||||
if self.server_task is not None:
|
if self.server_task is not None:
|
||||||
raise Exception("already running")
|
raise Exception("already running")
|
||||||
|
|
||||||
async def _start_server():
|
async def _start_server():
|
||||||
server = await self.loop.create_server(
|
partial_event = asyncio.Event()
|
||||||
lambda: ReflectorServerProtocol(self.blob_manager, self.response_chunk_size),
|
if not self.partial_needs:
|
||||||
interface, port
|
partial_event.set()
|
||||||
)
|
server = await self.loop.create_server(lambda: ReflectorServerProtocol(
|
||||||
|
self.blob_manager, self.response_chunk_size, self.stop_event, self.incoming_event,
|
||||||
|
self.not_incoming_event, partial_event), interface, port)
|
||||||
self.started_listening.set()
|
self.started_listening.set()
|
||||||
|
self.stopped_listening.clear()
|
||||||
log.info("Reflector server listening on TCP %s:%i", interface, port)
|
log.info("Reflector server listening on TCP %s:%i", interface, port)
|
||||||
async with server:
|
try:
|
||||||
await server.serve_forever()
|
async with server:
|
||||||
|
await server.serve_forever()
|
||||||
|
finally:
|
||||||
|
self.stopped_listening.set()
|
||||||
|
|
||||||
self.server_task = self.loop.create_task(_start_server())
|
self.server_task = self.loop.create_task(_start_server())
|
||||||
|
|
||||||
|
|||||||
+103
-323
@@ -6,83 +6,67 @@ import random
|
|||||||
import typing
|
import typing
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
from aiohttp.web import Request
|
from aiohttp.web import Request
|
||||||
from lbry.error import ResolveError, InvalidStreamDescriptorError, DownloadSDTimeoutError, InsufficientFundsError
|
from lbry.error import InvalidStreamDescriptorError
|
||||||
from lbry.error import ResolveTimeoutError, DownloadDataTimeoutError, KeyFeeAboveMaxAllowedError
|
from lbry.file.source_manager import SourceManager
|
||||||
from lbry.utils import cache_concurrent
|
|
||||||
from lbry.stream.descriptor import StreamDescriptor
|
from lbry.stream.descriptor import StreamDescriptor
|
||||||
from lbry.stream.managed_stream import ManagedStream
|
from lbry.stream.managed_stream import ManagedStream
|
||||||
from lbry.schema.claim import Claim
|
from lbry.file.source import ManagedDownloadSource
|
||||||
from lbry.schema.url import URL
|
|
||||||
from lbry.wallet.dewies import dewies_to_lbc
|
|
||||||
from lbry.wallet import Output
|
|
||||||
|
|
||||||
if typing.TYPE_CHECKING:
|
if typing.TYPE_CHECKING:
|
||||||
from lbry.conf import Config
|
from lbry.conf import Config
|
||||||
from lbry.blob.blob_manager import BlobManager
|
from lbry.blob.blob_manager import BlobManager
|
||||||
from lbry.dht.node import Node
|
from lbry.dht.node import Node
|
||||||
|
from lbry.wallet.wallet import WalletManager
|
||||||
|
from lbry.wallet.transaction import Transaction
|
||||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||||
from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim
|
from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim
|
||||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
FILTER_FIELDS = [
|
|
||||||
'rowid',
|
|
||||||
'status',
|
|
||||||
'file_name',
|
|
||||||
'added_on',
|
|
||||||
'sd_hash',
|
|
||||||
'stream_hash',
|
|
||||||
'claim_name',
|
|
||||||
'claim_height',
|
|
||||||
'claim_id',
|
|
||||||
'outpoint',
|
|
||||||
'txid',
|
|
||||||
'nout',
|
|
||||||
'channel_claim_id',
|
|
||||||
'channel_name',
|
|
||||||
'full_status', # TODO: remove
|
|
||||||
'blobs_remaining',
|
|
||||||
'blobs_in_stream'
|
|
||||||
]
|
|
||||||
|
|
||||||
COMPARISON_OPERATORS = {
|
def path_or_none(encoded_path) -> Optional[str]:
|
||||||
'eq': lambda a, b: a == b,
|
if not encoded_path:
|
||||||
'ne': lambda a, b: a != b,
|
|
||||||
'g': lambda a, b: a > b,
|
|
||||||
'l': lambda a, b: a < b,
|
|
||||||
'ge': lambda a, b: a >= b,
|
|
||||||
'le': lambda a, b: a <= b,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def path_or_none(path) -> Optional[str]:
|
|
||||||
if not path:
|
|
||||||
return
|
return
|
||||||
return binascii.unhexlify(path).decode()
|
return binascii.unhexlify(encoded_path).decode()
|
||||||
|
|
||||||
|
|
||||||
class StreamManager:
|
class StreamManager(SourceManager):
|
||||||
|
_sources: typing.Dict[str, ManagedStream]
|
||||||
|
|
||||||
|
filter_fields = SourceManager.filter_fields
|
||||||
|
filter_fields.update({
|
||||||
|
'sd_hash',
|
||||||
|
'stream_hash',
|
||||||
|
'full_status', # TODO: remove
|
||||||
|
'blobs_remaining',
|
||||||
|
'blobs_in_stream',
|
||||||
|
'uploading_to_reflector',
|
||||||
|
'is_fully_reflected'
|
||||||
|
})
|
||||||
|
|
||||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||||
wallet_manager: 'WalletManager', storage: 'SQLiteStorage', node: Optional['Node'],
|
wallet_manager: 'WalletManager', storage: 'SQLiteStorage', node: Optional['Node'],
|
||||||
analytics_manager: Optional['AnalyticsManager'] = None):
|
analytics_manager: Optional['AnalyticsManager'] = None):
|
||||||
self.loop = loop
|
super().__init__(loop, config, storage, analytics_manager)
|
||||||
self.config = config
|
|
||||||
self.blob_manager = blob_manager
|
self.blob_manager = blob_manager
|
||||||
self.wallet_manager = wallet_manager
|
self.wallet_manager = wallet_manager
|
||||||
self.storage = storage
|
|
||||||
self.node = node
|
self.node = node
|
||||||
self.analytics_manager = analytics_manager
|
|
||||||
self.streams: typing.Dict[str, ManagedStream] = {}
|
|
||||||
self.resume_saving_task: Optional[asyncio.Task] = None
|
self.resume_saving_task: Optional[asyncio.Task] = None
|
||||||
self.re_reflect_task: Optional[asyncio.Task] = None
|
self.re_reflect_task: Optional[asyncio.Task] = None
|
||||||
self.update_stream_finished_futs: typing.List[asyncio.Future] = []
|
self.update_stream_finished_futs: typing.List[asyncio.Future] = []
|
||||||
self.running_reflector_uploads: typing.Dict[str, asyncio.Task] = {}
|
self.running_reflector_uploads: typing.Dict[str, asyncio.Task] = {}
|
||||||
self.started = asyncio.Event(loop=self.loop)
|
self.started = asyncio.Event(loop=self.loop)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def streams(self):
|
||||||
|
return self._sources
|
||||||
|
|
||||||
|
def add(self, source: ManagedStream):
|
||||||
|
super().add(source)
|
||||||
|
self.storage.content_claim_callbacks[source.stream_hash] = lambda: self._update_content_claim(source)
|
||||||
|
|
||||||
async def _update_content_claim(self, stream: ManagedStream):
|
async def _update_content_claim(self, stream: ManagedStream):
|
||||||
claim_info = await self.storage.get_content_claim(stream.stream_hash)
|
claim_info = await self.storage.get_content_claim(stream.stream_hash)
|
||||||
self.streams.setdefault(stream.sd_hash, stream).set_claim(claim_info, claim_info['value'])
|
self._sources.setdefault(stream.sd_hash, stream).set_claim(claim_info, claim_info['value'])
|
||||||
|
|
||||||
async def recover_streams(self, file_infos: typing.List[typing.Dict]):
|
async def recover_streams(self, file_infos: typing.List[typing.Dict]):
|
||||||
to_restore = []
|
to_restore = []
|
||||||
@@ -113,10 +97,10 @@ class StreamManager:
|
|||||||
# if self.blob_manager._save_blobs:
|
# if self.blob_manager._save_blobs:
|
||||||
# log.info("Recovered %i/%i attempted streams", len(to_restore), len(file_infos))
|
# log.info("Recovered %i/%i attempted streams", len(to_restore), len(file_infos))
|
||||||
|
|
||||||
async def add_stream(self, rowid: int, sd_hash: str, file_name: Optional[str],
|
async def _load_stream(self, rowid: int, sd_hash: str, file_name: Optional[str],
|
||||||
download_directory: Optional[str], status: str,
|
download_directory: Optional[str], status: str,
|
||||||
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
|
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
|
||||||
added_on: Optional[int], fully_reflected: bool):
|
added_on: Optional[int], fully_reflected: Optional[bool]):
|
||||||
try:
|
try:
|
||||||
descriptor = await self.blob_manager.get_stream_descriptor(sd_hash)
|
descriptor = await self.blob_manager.get_stream_descriptor(sd_hash)
|
||||||
except InvalidStreamDescriptorError as err:
|
except InvalidStreamDescriptorError as err:
|
||||||
@@ -129,10 +113,9 @@ class StreamManager:
|
|||||||
)
|
)
|
||||||
if fully_reflected:
|
if fully_reflected:
|
||||||
stream.fully_reflected.set()
|
stream.fully_reflected.set()
|
||||||
self.streams[sd_hash] = stream
|
self.add(stream)
|
||||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
|
||||||
|
|
||||||
async def load_and_resume_streams_from_database(self):
|
async def initialize_from_database(self):
|
||||||
to_recover = []
|
to_recover = []
|
||||||
to_start = []
|
to_start = []
|
||||||
|
|
||||||
@@ -146,7 +129,6 @@ class StreamManager:
|
|||||||
to_recover.append(file_info)
|
to_recover.append(file_info)
|
||||||
to_start.append(file_info)
|
to_start.append(file_info)
|
||||||
if to_recover:
|
if to_recover:
|
||||||
log.info("Recover %i files", len(to_recover))
|
|
||||||
await self.recover_streams(to_recover)
|
await self.recover_streams(to_recover)
|
||||||
|
|
||||||
log.info("Initializing %i files", len(to_start))
|
log.info("Initializing %i files", len(to_start))
|
||||||
@@ -157,7 +139,7 @@ class StreamManager:
|
|||||||
download_directory = path_or_none(file_info['download_directory'])
|
download_directory = path_or_none(file_info['download_directory'])
|
||||||
if file_name and download_directory and not file_info['saved_file'] and file_info['status'] == 'running':
|
if file_name and download_directory and not file_info['saved_file'] and file_info['status'] == 'running':
|
||||||
to_resume_saving.append((file_name, download_directory, file_info['sd_hash']))
|
to_resume_saving.append((file_name, download_directory, file_info['sd_hash']))
|
||||||
add_stream_tasks.append(self.loop.create_task(self.add_stream(
|
add_stream_tasks.append(self.loop.create_task(self._load_stream(
|
||||||
file_info['rowid'], file_info['sd_hash'], file_name,
|
file_info['rowid'], file_info['sd_hash'], file_name,
|
||||||
download_directory, file_info['status'],
|
download_directory, file_info['status'],
|
||||||
file_info['claim'], file_info['content_fee'],
|
file_info['claim'], file_info['content_fee'],
|
||||||
@@ -165,25 +147,32 @@ class StreamManager:
|
|||||||
)))
|
)))
|
||||||
if add_stream_tasks:
|
if add_stream_tasks:
|
||||||
await asyncio.gather(*add_stream_tasks, loop=self.loop)
|
await asyncio.gather(*add_stream_tasks, loop=self.loop)
|
||||||
log.info("Started stream manager with %i files", len(self.streams))
|
log.info("Started stream manager with %i files", len(self._sources))
|
||||||
if not self.node:
|
if not self.node:
|
||||||
log.info("no DHT node given, resuming downloads trusting that we can contact reflector")
|
log.info("no DHT node given, resuming downloads trusting that we can contact reflector")
|
||||||
if to_resume_saving:
|
if to_resume_saving:
|
||||||
self.resume_saving_task = self.loop.create_task(self.resume(to_resume_saving))
|
log.info("Resuming saving %i files", len(to_resume_saving))
|
||||||
|
self.resume_saving_task = asyncio.ensure_future(asyncio.gather(
|
||||||
async def resume(self, to_resume_saving):
|
*(self._sources[sd_hash].save_file(file_name, download_directory)
|
||||||
log.info("Resuming saving %i files", len(to_resume_saving))
|
for (file_name, download_directory, sd_hash) in to_resume_saving),
|
||||||
await asyncio.gather(
|
loop=self.loop
|
||||||
*(self.streams[sd_hash].save_file(file_name, download_directory, node=self.node)
|
))
|
||||||
for (file_name, download_directory, sd_hash) in to_resume_saving),
|
|
||||||
loop=self.loop
|
|
||||||
)
|
|
||||||
|
|
||||||
async def reflect_streams(self):
|
async def reflect_streams(self):
|
||||||
|
try:
|
||||||
|
return await self._reflect_streams()
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
raise
|
||||||
|
except Exception:
|
||||||
|
log.exception("reflector task encountered an unexpected error!")
|
||||||
|
|
||||||
|
async def _reflect_streams(self):
|
||||||
|
# todo: those debug statements are temporary for #2987 - remove them if its closed
|
||||||
while True:
|
while True:
|
||||||
if self.config.reflect_streams and self.config.reflector_servers:
|
if self.config.reflect_streams and self.config.reflector_servers:
|
||||||
|
log.debug("collecting streams to reflect")
|
||||||
sd_hashes = await self.storage.get_streams_to_re_reflect()
|
sd_hashes = await self.storage.get_streams_to_re_reflect()
|
||||||
sd_hashes = [sd for sd in sd_hashes if sd in self.streams]
|
sd_hashes = [sd for sd in sd_hashes if sd in self._sources]
|
||||||
batch = []
|
batch = []
|
||||||
while sd_hashes:
|
while sd_hashes:
|
||||||
stream = self.streams[sd_hashes.pop()]
|
stream = self.streams[sd_hashes.pop()]
|
||||||
@@ -192,25 +181,26 @@ class StreamManager:
|
|||||||
stream.fully_reflected.is_set():
|
stream.fully_reflected.is_set():
|
||||||
batch.append(self.reflect_stream(stream))
|
batch.append(self.reflect_stream(stream))
|
||||||
if len(batch) >= self.config.concurrent_reflector_uploads:
|
if len(batch) >= self.config.concurrent_reflector_uploads:
|
||||||
|
log.debug("waiting for batch of %s reflecting streams", len(batch))
|
||||||
await asyncio.gather(*batch, loop=self.loop)
|
await asyncio.gather(*batch, loop=self.loop)
|
||||||
|
log.debug("done processing %s streams", len(batch))
|
||||||
batch = []
|
batch = []
|
||||||
if batch:
|
if batch:
|
||||||
|
log.debug("waiting for batch of %s reflecting streams", len(batch))
|
||||||
await asyncio.gather(*batch, loop=self.loop)
|
await asyncio.gather(*batch, loop=self.loop)
|
||||||
|
log.debug("done processing %s streams", len(batch))
|
||||||
await asyncio.sleep(300, loop=self.loop)
|
await asyncio.sleep(300, loop=self.loop)
|
||||||
|
|
||||||
async def start(self):
|
async def start(self):
|
||||||
await self.load_and_resume_streams_from_database()
|
await super().start()
|
||||||
self.re_reflect_task = self.loop.create_task(self.reflect_streams())
|
self.re_reflect_task = self.loop.create_task(self.reflect_streams())
|
||||||
self.started.set()
|
|
||||||
|
|
||||||
def stop(self):
|
def stop(self):
|
||||||
|
super().stop()
|
||||||
if self.resume_saving_task and not self.resume_saving_task.done():
|
if self.resume_saving_task and not self.resume_saving_task.done():
|
||||||
self.resume_saving_task.cancel()
|
self.resume_saving_task.cancel()
|
||||||
if self.re_reflect_task and not self.re_reflect_task.done():
|
if self.re_reflect_task and not self.re_reflect_task.done():
|
||||||
self.re_reflect_task.cancel()
|
self.re_reflect_task.cancel()
|
||||||
while self.streams:
|
|
||||||
_, stream = self.streams.popitem()
|
|
||||||
stream.stop_tasks()
|
|
||||||
while self.update_stream_finished_futs:
|
while self.update_stream_finished_futs:
|
||||||
self.update_stream_finished_futs.pop().cancel()
|
self.update_stream_finished_futs.pop().cancel()
|
||||||
while self.running_reflector_uploads:
|
while self.running_reflector_uploads:
|
||||||
@@ -225,7 +215,7 @@ class StreamManager:
|
|||||||
server, port = random.choice(self.config.reflector_servers)
|
server, port = random.choice(self.config.reflector_servers)
|
||||||
if stream.sd_hash in self.running_reflector_uploads:
|
if stream.sd_hash in self.running_reflector_uploads:
|
||||||
return self.running_reflector_uploads[stream.sd_hash]
|
return self.running_reflector_uploads[stream.sd_hash]
|
||||||
task = self.loop.create_task(stream.upload_to_reflector(server, port))
|
task = self.loop.create_task(self._retriable_reflect_stream(stream, server, port))
|
||||||
self.running_reflector_uploads[stream.sd_hash] = task
|
self.running_reflector_uploads[stream.sd_hash] = task
|
||||||
task.add_done_callback(
|
task.add_done_callback(
|
||||||
lambda _: None if stream.sd_hash not in self.running_reflector_uploads else
|
lambda _: None if stream.sd_hash not in self.running_reflector_uploads else
|
||||||
@@ -233,261 +223,51 @@ class StreamManager:
|
|||||||
)
|
)
|
||||||
return task
|
return task
|
||||||
|
|
||||||
async def create_stream(self, file_path: str, key: Optional[bytes] = None,
|
async def _retriable_reflect_stream(self, stream, host, port):
|
||||||
iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
|
sent = await stream.upload_to_reflector(host, port)
|
||||||
stream = await ManagedStream.create(self.loop, self.config, self.blob_manager, file_path, key, iv_generator)
|
while not stream.is_fully_reflected and stream.reflector_progress > 0 and len(sent) > 0:
|
||||||
|
stream.reflector_progress = 0
|
||||||
|
sent = await stream.upload_to_reflector(host, port)
|
||||||
|
|
||||||
|
async def create(self, file_path: str, key: Optional[bytes] = None,
|
||||||
|
iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
|
||||||
|
descriptor = await StreamDescriptor.create_stream(
|
||||||
|
self.loop, self.blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
|
||||||
|
blob_completed_callback=self.blob_manager.blob_completed
|
||||||
|
)
|
||||||
|
await self.storage.store_stream(
|
||||||
|
self.blob_manager.get_blob(descriptor.sd_hash, is_mine=True), descriptor
|
||||||
|
)
|
||||||
|
row_id = await self.storage.save_published_file(
|
||||||
|
descriptor.stream_hash, os.path.basename(file_path), os.path.dirname(file_path), 0
|
||||||
|
)
|
||||||
|
stream = ManagedStream(
|
||||||
|
self.loop, self.config, self.blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
|
||||||
|
os.path.basename(file_path), status=ManagedDownloadSource.STATUS_FINISHED,
|
||||||
|
rowid=row_id, descriptor=descriptor
|
||||||
|
)
|
||||||
self.streams[stream.sd_hash] = stream
|
self.streams[stream.sd_hash] = stream
|
||||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||||
if self.config.reflect_streams and self.config.reflector_servers:
|
if self.config.reflect_streams and self.config.reflector_servers:
|
||||||
self.reflect_stream(stream)
|
self.reflect_stream(stream)
|
||||||
return stream
|
return stream
|
||||||
|
|
||||||
async def delete_stream(self, stream: ManagedStream, delete_file: Optional[bool] = False):
|
async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
|
||||||
if stream.sd_hash in self.running_reflector_uploads:
|
if not isinstance(source, ManagedStream):
|
||||||
self.running_reflector_uploads[stream.sd_hash].cancel()
|
return
|
||||||
stream.stop_tasks()
|
if source.identifier in self.running_reflector_uploads:
|
||||||
if stream.sd_hash in self.streams:
|
self.running_reflector_uploads[source.identifier].cancel()
|
||||||
del self.streams[stream.sd_hash]
|
source.stop_tasks()
|
||||||
blob_hashes = [stream.sd_hash] + [b.blob_hash for b in stream.descriptor.blobs[:-1]]
|
if source.identifier in self.streams:
|
||||||
|
del self.streams[source.identifier]
|
||||||
|
blob_hashes = [source.identifier] + [b.blob_hash for b in source.descriptor.blobs[:-1]]
|
||||||
await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False)
|
await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False)
|
||||||
await self.storage.delete_stream(stream.descriptor)
|
await self.storage.delete_stream(source.descriptor)
|
||||||
if delete_file and stream.output_file_exists:
|
if delete_file and source.output_file_exists:
|
||||||
os.remove(stream.full_path)
|
os.remove(source.full_path)
|
||||||
|
|
||||||
def get_stream_by_stream_hash(self, stream_hash: str) -> Optional[ManagedStream]:
|
|
||||||
streams = tuple(filter(lambda stream: stream.stream_hash == stream_hash, self.streams.values()))
|
|
||||||
if streams:
|
|
||||||
return streams[0]
|
|
||||||
|
|
||||||
def get_filtered_streams(self, sort_by: Optional[str] = None, reverse: Optional[bool] = False,
|
|
||||||
comparison: Optional[str] = None,
|
|
||||||
**search_by) -> typing.List[ManagedStream]:
|
|
||||||
"""
|
|
||||||
Get a list of filtered and sorted ManagedStream objects
|
|
||||||
|
|
||||||
:param sort_by: field to sort by
|
|
||||||
:param reverse: reverse sorting
|
|
||||||
:param comparison: comparison operator used for filtering
|
|
||||||
:param search_by: fields and values to filter by
|
|
||||||
"""
|
|
||||||
if sort_by and sort_by not in FILTER_FIELDS:
|
|
||||||
raise ValueError(f"'{sort_by}' is not a valid field to sort by")
|
|
||||||
if comparison and comparison not in COMPARISON_OPERATORS:
|
|
||||||
raise ValueError(f"'{comparison}' is not a valid comparison")
|
|
||||||
if 'full_status' in search_by:
|
|
||||||
del search_by['full_status']
|
|
||||||
for search in search_by:
|
|
||||||
if search not in FILTER_FIELDS:
|
|
||||||
raise ValueError(f"'{search}' is not a valid search operation")
|
|
||||||
if search_by:
|
|
||||||
comparison = comparison or 'eq'
|
|
||||||
streams = []
|
|
||||||
for stream in self.streams.values():
|
|
||||||
for search, val in search_by.items():
|
|
||||||
if COMPARISON_OPERATORS[comparison](getattr(stream, search), val):
|
|
||||||
streams.append(stream)
|
|
||||||
break
|
|
||||||
else:
|
|
||||||
streams = list(self.streams.values())
|
|
||||||
if sort_by:
|
|
||||||
streams.sort(key=lambda s: getattr(s, sort_by))
|
|
||||||
if reverse:
|
|
||||||
streams.reverse()
|
|
||||||
return streams
|
|
||||||
|
|
||||||
async def _check_update_or_replace(self, outpoint: str, claim_id: str, claim: Claim
|
|
||||||
) -> typing.Tuple[Optional[ManagedStream], Optional[ManagedStream]]:
|
|
||||||
existing = self.get_filtered_streams(outpoint=outpoint)
|
|
||||||
if existing:
|
|
||||||
return existing[0], None
|
|
||||||
existing = self.get_filtered_streams(sd_hash=claim.stream.source.sd_hash)
|
|
||||||
if existing and existing[0].claim_id != claim_id:
|
|
||||||
raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {claim_id}")
|
|
||||||
if existing:
|
|
||||||
log.info("claim contains a metadata only update to a stream we have")
|
|
||||||
await self.storage.save_content_claim(
|
|
||||||
existing[0].stream_hash, outpoint
|
|
||||||
)
|
|
||||||
await self._update_content_claim(existing[0])
|
|
||||||
return existing[0], None
|
|
||||||
else:
|
|
||||||
existing_for_claim_id = self.get_filtered_streams(claim_id=claim_id)
|
|
||||||
if existing_for_claim_id:
|
|
||||||
log.info("claim contains an update to a stream we have, downloading it")
|
|
||||||
return None, existing_for_claim_id[0]
|
|
||||||
return None, None
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _convert_to_old_resolve_output(wallet_manager, resolves):
|
|
||||||
result = {}
|
|
||||||
for url, txo in resolves.items():
|
|
||||||
if isinstance(txo, Output):
|
|
||||||
tx_height = txo.tx_ref.height
|
|
||||||
best_height = wallet_manager.ledger.headers.height
|
|
||||||
result[url] = {
|
|
||||||
'name': txo.claim_name,
|
|
||||||
'value': txo.claim,
|
|
||||||
'protobuf': binascii.hexlify(txo.claim.to_bytes()),
|
|
||||||
'claim_id': txo.claim_id,
|
|
||||||
'txid': txo.tx_ref.id,
|
|
||||||
'nout': txo.position,
|
|
||||||
'amount': dewies_to_lbc(txo.amount),
|
|
||||||
'effective_amount': txo.meta.get('effective_amount', 0),
|
|
||||||
'height': tx_height,
|
|
||||||
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
|
|
||||||
'claim_sequence': -1,
|
|
||||||
'address': txo.get_address(wallet_manager.ledger),
|
|
||||||
'valid_at_height': txo.meta.get('activation_height', None),
|
|
||||||
'timestamp': wallet_manager.ledger.headers[tx_height]['timestamp'],
|
|
||||||
'supports': []
|
|
||||||
}
|
|
||||||
else:
|
|
||||||
result[url] = txo
|
|
||||||
return result
|
|
||||||
|
|
||||||
@cache_concurrent
|
|
||||||
async def download_stream_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
|
|
||||||
timeout: Optional[float] = None,
|
|
||||||
file_name: Optional[str] = None,
|
|
||||||
download_directory: Optional[str] = None,
|
|
||||||
save_file: Optional[bool] = None,
|
|
||||||
resolve_timeout: float = 3.0,
|
|
||||||
wallet: Optional['Wallet'] = None) -> ManagedStream:
|
|
||||||
manager = self.wallet_manager
|
|
||||||
wallet = wallet or manager.default_wallet
|
|
||||||
timeout = timeout or self.config.download_timeout
|
|
||||||
start_time = self.loop.time()
|
|
||||||
resolved_time = None
|
|
||||||
stream = None
|
|
||||||
txo: Optional[Output] = None
|
|
||||||
error = None
|
|
||||||
outpoint = None
|
|
||||||
if save_file is None:
|
|
||||||
save_file = self.config.save_files
|
|
||||||
if file_name and not save_file:
|
|
||||||
save_file = True
|
|
||||||
if save_file:
|
|
||||||
download_directory = download_directory or self.config.download_dir
|
|
||||||
else:
|
|
||||||
download_directory = None
|
|
||||||
|
|
||||||
payment = None
|
|
||||||
try:
|
|
||||||
# resolve the claim
|
|
||||||
if not URL.parse(uri).has_stream:
|
|
||||||
raise ResolveError("cannot download a channel claim, specify a /path")
|
|
||||||
try:
|
|
||||||
response = await asyncio.wait_for(
|
|
||||||
manager.ledger.resolve(wallet.accounts, [uri]),
|
|
||||||
resolve_timeout
|
|
||||||
)
|
|
||||||
resolved_result = self._convert_to_old_resolve_output(manager, response)
|
|
||||||
except asyncio.TimeoutError:
|
|
||||||
raise ResolveTimeoutError(uri)
|
|
||||||
except Exception as err:
|
|
||||||
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
|
|
||||||
raise
|
|
||||||
log.exception("Unexpected error resolving stream:")
|
|
||||||
raise ResolveError(f"Unexpected error resolving stream: {str(err)}")
|
|
||||||
await self.storage.save_claims_for_resolve([
|
|
||||||
value for value in resolved_result.values() if 'error' not in value
|
|
||||||
])
|
|
||||||
resolved = resolved_result.get(uri, {})
|
|
||||||
resolved = resolved if 'value' in resolved else resolved.get('claim')
|
|
||||||
if not resolved:
|
|
||||||
raise ResolveError(f"Failed to resolve stream at '{uri}'")
|
|
||||||
if 'error' in resolved:
|
|
||||||
raise ResolveError(f"error resolving stream: {resolved['error']}")
|
|
||||||
txo = response[uri]
|
|
||||||
|
|
||||||
claim = Claim.from_bytes(binascii.unhexlify(resolved['protobuf']))
|
|
||||||
outpoint = f"{resolved['txid']}:{resolved['nout']}"
|
|
||||||
resolved_time = self.loop.time() - start_time
|
|
||||||
|
|
||||||
# resume or update an existing stream, if the stream changed: download it and delete the old one after
|
|
||||||
updated_stream, to_replace = await self._check_update_or_replace(outpoint, resolved['claim_id'], claim)
|
|
||||||
if updated_stream:
|
|
||||||
log.info("already have stream for %s", uri)
|
|
||||||
if save_file and updated_stream.output_file_exists:
|
|
||||||
save_file = False
|
|
||||||
await updated_stream.start(node=self.node, timeout=timeout, save_now=save_file)
|
|
||||||
if not updated_stream.output_file_exists and (save_file or file_name or download_directory):
|
|
||||||
await updated_stream.save_file(
|
|
||||||
file_name=file_name, download_directory=download_directory, node=self.node
|
|
||||||
)
|
|
||||||
return updated_stream
|
|
||||||
|
|
||||||
if not to_replace and txo.has_price and not txo.purchase_receipt:
|
|
||||||
payment = await manager.create_purchase_transaction(
|
|
||||||
wallet.accounts, txo, exchange_rate_manager
|
|
||||||
)
|
|
||||||
|
|
||||||
stream = ManagedStream(
|
|
||||||
self.loop, self.config, self.blob_manager, claim.stream.source.sd_hash, download_directory,
|
|
||||||
file_name, ManagedStream.STATUS_RUNNING, content_fee=payment,
|
|
||||||
analytics_manager=self.analytics_manager
|
|
||||||
)
|
|
||||||
log.info("starting download for %s", uri)
|
|
||||||
|
|
||||||
before_download = self.loop.time()
|
|
||||||
await stream.start(self.node, timeout)
|
|
||||||
stream.set_claim(resolved, claim)
|
|
||||||
if to_replace: # delete old stream now that the replacement has started downloading
|
|
||||||
await self.delete_stream(to_replace)
|
|
||||||
|
|
||||||
if payment is not None:
|
|
||||||
await manager.broadcast_or_release(payment)
|
|
||||||
payment = None # to avoid releasing in `finally` later
|
|
||||||
log.info("paid fee of %s for %s", dewies_to_lbc(stream.content_fee.outputs[0].amount), uri)
|
|
||||||
await self.storage.save_content_fee(stream.stream_hash, stream.content_fee)
|
|
||||||
|
|
||||||
self.streams[stream.sd_hash] = stream
|
|
||||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
|
||||||
await self.storage.save_content_claim(stream.stream_hash, outpoint)
|
|
||||||
if save_file:
|
|
||||||
await asyncio.wait_for(stream.save_file(node=self.node), timeout - (self.loop.time() - before_download),
|
|
||||||
loop=self.loop)
|
|
||||||
return stream
|
|
||||||
except asyncio.TimeoutError:
|
|
||||||
error = DownloadDataTimeoutError(stream.sd_hash)
|
|
||||||
raise error
|
|
||||||
except Exception as err: # forgive data timeout, don't delete stream
|
|
||||||
expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError,
|
|
||||||
KeyFeeAboveMaxAllowedError)
|
|
||||||
if isinstance(err, expected):
|
|
||||||
log.warning("Failed to download %s: %s", uri, str(err))
|
|
||||||
elif isinstance(err, asyncio.CancelledError):
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
log.exception("Unexpected error downloading stream:")
|
|
||||||
error = err
|
|
||||||
raise
|
|
||||||
finally:
|
|
||||||
if payment is not None:
|
|
||||||
# payment is set to None after broadcasting, if we're here an exception probably happened
|
|
||||||
await manager.ledger.release_tx(payment)
|
|
||||||
if self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or
|
|
||||||
stream.downloader.time_to_first_bytes))):
|
|
||||||
server = self.wallet_manager.ledger.network.client.server
|
|
||||||
self.loop.create_task(
|
|
||||||
self.analytics_manager.send_time_to_first_bytes(
|
|
||||||
resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id,
|
|
||||||
uri, outpoint,
|
|
||||||
None if not stream else len(stream.downloader.blob_downloader.active_connections),
|
|
||||||
None if not stream else len(stream.downloader.blob_downloader.scores),
|
|
||||||
None if not stream else len(stream.downloader.blob_downloader.connection_failures),
|
|
||||||
False if not stream else stream.downloader.added_fixed_peers,
|
|
||||||
self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay,
|
|
||||||
None if not stream else stream.sd_hash,
|
|
||||||
None if not stream else stream.downloader.time_to_descriptor,
|
|
||||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
|
|
||||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
|
|
||||||
None if not stream else stream.downloader.time_to_first_bytes,
|
|
||||||
None if not error else error.__class__.__name__,
|
|
||||||
None if not error else str(error),
|
|
||||||
None if not server else f"{server[0]}:{server[1]}"
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
async def stream_partial_content(self, request: Request, sd_hash: str):
|
async def stream_partial_content(self, request: Request, sd_hash: str):
|
||||||
return await self.streams[sd_hash].stream_file(request, self.node)
|
stream = self._sources[sd_hash]
|
||||||
|
if not stream.downloader.node:
|
||||||
|
stream.downloader.node = self.node
|
||||||
|
return await stream.stream_file(request)
|
||||||
+109
-23
@@ -18,17 +18,18 @@ from lbry.wallet import WalletManager, Wallet, Ledger, Account, Transaction
|
|||||||
from lbry.conf import Config
|
from lbry.conf import Config
|
||||||
from lbry.wallet.util import satoshis_to_coins
|
from lbry.wallet.util import satoshis_to_coins
|
||||||
from lbry.wallet.orchstr8 import Conductor
|
from lbry.wallet.orchstr8 import Conductor
|
||||||
from lbry.wallet.orchstr8.node import BlockchainNode, WalletNode
|
from lbry.wallet.orchstr8.node import BlockchainNode, WalletNode, HubNode
|
||||||
|
|
||||||
from lbry.extras.daemon.daemon import Daemon, jsonrpc_dumps_pretty
|
from lbry.extras.daemon.daemon import Daemon, jsonrpc_dumps_pretty
|
||||||
from lbry.extras.daemon.components import Component, WalletComponent
|
from lbry.extras.daemon.components import Component, WalletComponent
|
||||||
from lbry.extras.daemon.components import (
|
from lbry.extras.daemon.components import (
|
||||||
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
DHT_COMPONENT,
|
||||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||||
|
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, LIBTORRENT_COMPONENT
|
||||||
)
|
)
|
||||||
from lbry.extras.daemon.componentmanager import ComponentManager
|
from lbry.extras.daemon.componentmanager import ComponentManager
|
||||||
from lbry.extras.daemon.exchange_rate_manager import (
|
from lbry.extras.daemon.exchange_rate_manager import (
|
||||||
ExchangeRateManager, ExchangeRate, LBRYFeed, LBRYBTCFeed
|
ExchangeRateManager, ExchangeRate, BittrexBTCFeed, BittrexUSDFeed
|
||||||
)
|
)
|
||||||
from lbry.extras.daemon.storage import SQLiteStorage
|
from lbry.extras.daemon.storage import SQLiteStorage
|
||||||
from lbry.blob.blob_manager import BlobManager
|
from lbry.blob.blob_manager import BlobManager
|
||||||
@@ -83,6 +84,7 @@ class AsyncioTestCase(unittest.TestCase):
|
|||||||
# https://bugs.python.org/issue32972
|
# https://bugs.python.org/issue32972
|
||||||
|
|
||||||
LOOP_SLOW_CALLBACK_DURATION = 0.2
|
LOOP_SLOW_CALLBACK_DURATION = 0.2
|
||||||
|
TIMEOUT = 120.0
|
||||||
|
|
||||||
maxDiff = None
|
maxDiff = None
|
||||||
|
|
||||||
@@ -136,6 +138,8 @@ class AsyncioTestCase(unittest.TestCase):
|
|||||||
with outcome.testPartExecutor(self, isTest=True):
|
with outcome.testPartExecutor(self, isTest=True):
|
||||||
maybe_coroutine = testMethod()
|
maybe_coroutine = testMethod()
|
||||||
if asyncio.iscoroutine(maybe_coroutine):
|
if asyncio.iscoroutine(maybe_coroutine):
|
||||||
|
if self.TIMEOUT:
|
||||||
|
self.loop.call_later(self.TIMEOUT, self.cancel)
|
||||||
self.loop.run_until_complete(maybe_coroutine)
|
self.loop.run_until_complete(maybe_coroutine)
|
||||||
outcome.expecting_failure = False
|
outcome.expecting_failure = False
|
||||||
with outcome.testPartExecutor(self):
|
with outcome.testPartExecutor(self):
|
||||||
@@ -188,6 +192,12 @@ class AsyncioTestCase(unittest.TestCase):
|
|||||||
if asyncio.iscoroutine(maybe_coroutine):
|
if asyncio.iscoroutine(maybe_coroutine):
|
||||||
self.loop.run_until_complete(maybe_coroutine)
|
self.loop.run_until_complete(maybe_coroutine)
|
||||||
|
|
||||||
|
def cancel(self):
|
||||||
|
for task in asyncio.all_tasks(self.loop):
|
||||||
|
if not task.done():
|
||||||
|
task.print_stack()
|
||||||
|
task.cancel()
|
||||||
|
|
||||||
|
|
||||||
class AdvanceTimeTestCase(AsyncioTestCase):
|
class AdvanceTimeTestCase(AsyncioTestCase):
|
||||||
|
|
||||||
@@ -213,6 +223,7 @@ class IntegrationTestCase(AsyncioTestCase):
|
|||||||
super().__init__(*args, **kwargs)
|
super().__init__(*args, **kwargs)
|
||||||
self.conductor: Optional[Conductor] = None
|
self.conductor: Optional[Conductor] = None
|
||||||
self.blockchain: Optional[BlockchainNode] = None
|
self.blockchain: Optional[BlockchainNode] = None
|
||||||
|
self.hub: Optional[HubNode] = None
|
||||||
self.wallet_node: Optional[WalletNode] = None
|
self.wallet_node: Optional[WalletNode] = None
|
||||||
self.manager: Optional[WalletManager] = None
|
self.manager: Optional[WalletManager] = None
|
||||||
self.ledger: Optional[Ledger] = None
|
self.ledger: Optional[Ledger] = None
|
||||||
@@ -227,7 +238,10 @@ class IntegrationTestCase(AsyncioTestCase):
|
|||||||
self.addCleanup(self.conductor.stop_spv)
|
self.addCleanup(self.conductor.stop_spv)
|
||||||
await self.conductor.start_wallet()
|
await self.conductor.start_wallet()
|
||||||
self.addCleanup(self.conductor.stop_wallet)
|
self.addCleanup(self.conductor.stop_wallet)
|
||||||
|
await self.conductor.start_hub()
|
||||||
|
self.addCleanup(self.conductor.stop_hub)
|
||||||
self.blockchain = self.conductor.blockchain_node
|
self.blockchain = self.conductor.blockchain_node
|
||||||
|
self.hub = self.conductor.hub_node
|
||||||
self.wallet_node = self.conductor.wallet_node
|
self.wallet_node = self.conductor.wallet_node
|
||||||
self.manager = self.wallet_node.manager
|
self.manager = self.wallet_node.manager
|
||||||
self.ledger = self.wallet_node.ledger
|
self.ledger = self.wallet_node.ledger
|
||||||
@@ -281,8 +295,8 @@ class FakeExchangeRateManager(ExchangeRateManager):
|
|||||||
|
|
||||||
def get_fake_exchange_rate_manager(rates=None):
|
def get_fake_exchange_rate_manager(rates=None):
|
||||||
return FakeExchangeRateManager(
|
return FakeExchangeRateManager(
|
||||||
[LBRYFeed(), LBRYBTCFeed()],
|
[BittrexBTCFeed(), BittrexUSDFeed()],
|
||||||
rates or {'BTCLBC': 3.0, 'USDBTC': 2.0}
|
rates or {'BTCLBC': 3.0, 'USDLBC': 2.0}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -320,6 +334,7 @@ class CommandTestCase(IntegrationTestCase):
|
|||||||
self.server_blob_manager = None
|
self.server_blob_manager = None
|
||||||
self.server = None
|
self.server = None
|
||||||
self.reflector = None
|
self.reflector = None
|
||||||
|
self.skip_libtorrent = True
|
||||||
|
|
||||||
async def asyncSetUp(self):
|
async def asyncSetUp(self):
|
||||||
await super().asyncSetUp()
|
await super().asyncSetUp()
|
||||||
@@ -339,7 +354,13 @@ class CommandTestCase(IntegrationTestCase):
|
|||||||
|
|
||||||
server_tmp_dir = tempfile.mkdtemp()
|
server_tmp_dir = tempfile.mkdtemp()
|
||||||
self.addCleanup(shutil.rmtree, server_tmp_dir)
|
self.addCleanup(shutil.rmtree, server_tmp_dir)
|
||||||
self.server_config = Config()
|
self.server_config = Config(
|
||||||
|
data_dir=server_tmp_dir,
|
||||||
|
wallet_dir=server_tmp_dir,
|
||||||
|
save_files=True,
|
||||||
|
download_dir=server_tmp_dir
|
||||||
|
)
|
||||||
|
self.server_config.transaction_cache_size = 10000
|
||||||
self.server_storage = SQLiteStorage(self.server_config, ':memory:')
|
self.server_storage = SQLiteStorage(self.server_config, ':memory:')
|
||||||
await self.server_storage.open()
|
await self.server_storage.open()
|
||||||
|
|
||||||
@@ -362,6 +383,7 @@ class CommandTestCase(IntegrationTestCase):
|
|||||||
await daemon.stop()
|
await daemon.stop()
|
||||||
|
|
||||||
async def add_daemon(self, wallet_node=None, seed=None):
|
async def add_daemon(self, wallet_node=None, seed=None):
|
||||||
|
start_wallet_node = False
|
||||||
if wallet_node is None:
|
if wallet_node is None:
|
||||||
wallet_node = WalletNode(
|
wallet_node = WalletNode(
|
||||||
self.wallet_node.manager_class,
|
self.wallet_node.manager_class,
|
||||||
@@ -369,30 +391,42 @@ class CommandTestCase(IntegrationTestCase):
|
|||||||
port=self.extra_wallet_node_port
|
port=self.extra_wallet_node_port
|
||||||
)
|
)
|
||||||
self.extra_wallet_node_port += 1
|
self.extra_wallet_node_port += 1
|
||||||
await wallet_node.start(self.conductor.spv_node, seed=seed)
|
start_wallet_node = True
|
||||||
self.extra_wallet_nodes.append(wallet_node)
|
|
||||||
|
|
||||||
upload_dir = os.path.join(wallet_node.data_path, 'uploads')
|
upload_dir = os.path.join(wallet_node.data_path, 'uploads')
|
||||||
os.mkdir(upload_dir)
|
os.mkdir(upload_dir)
|
||||||
|
|
||||||
conf = Config()
|
conf = Config(
|
||||||
conf.data_dir = wallet_node.data_path
|
# needed during instantiation to access known_hubs path
|
||||||
conf.wallet_dir = wallet_node.data_path
|
data_dir=wallet_node.data_path,
|
||||||
conf.download_dir = wallet_node.data_path
|
wallet_dir=wallet_node.data_path,
|
||||||
|
save_files=True,
|
||||||
|
download_dir=wallet_node.data_path
|
||||||
|
)
|
||||||
conf.upload_dir = upload_dir # not a real conf setting
|
conf.upload_dir = upload_dir # not a real conf setting
|
||||||
conf.share_usage_data = False
|
conf.share_usage_data = False
|
||||||
conf.use_upnp = False
|
conf.use_upnp = False
|
||||||
conf.reflect_streams = True
|
conf.reflect_streams = True
|
||||||
conf.blockchain_name = 'lbrycrd_regtest'
|
conf.blockchain_name = 'lbrycrd_regtest'
|
||||||
conf.lbryum_servers = [('127.0.0.1', 50001)]
|
conf.lbryum_servers = [(self.conductor.spv_node.hostname, self.conductor.spv_node.port)]
|
||||||
conf.reflector_servers = [('127.0.0.1', 5566)]
|
conf.reflector_servers = [('127.0.0.1', 5566)]
|
||||||
|
conf.fixed_peers = [('127.0.0.1', 5567)]
|
||||||
conf.known_dht_nodes = []
|
conf.known_dht_nodes = []
|
||||||
conf.blob_lru_cache_size = self.blob_lru_cache_size
|
conf.blob_lru_cache_size = self.blob_lru_cache_size
|
||||||
|
conf.transaction_cache_size = 10000
|
||||||
conf.components_to_skip = [
|
conf.components_to_skip = [
|
||||||
DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
|
DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
|
||||||
PEER_PROTOCOL_SERVER_COMPONENT
|
PEER_PROTOCOL_SERVER_COMPONENT
|
||||||
]
|
]
|
||||||
wallet_node.manager.config = conf
|
if self.skip_libtorrent:
|
||||||
|
conf.components_to_skip.append(LIBTORRENT_COMPONENT)
|
||||||
|
|
||||||
|
if start_wallet_node:
|
||||||
|
await wallet_node.start(self.conductor.spv_node, seed=seed, config=conf)
|
||||||
|
self.extra_wallet_nodes.append(wallet_node)
|
||||||
|
else:
|
||||||
|
wallet_node.manager.config = conf
|
||||||
|
wallet_node.manager.ledger.config['known_hubs'] = conf.known_hubs
|
||||||
|
|
||||||
def wallet_maker(component_manager):
|
def wallet_maker(component_manager):
|
||||||
wallet_component = WalletComponent(component_manager)
|
wallet_component = WalletComponent(component_manager)
|
||||||
@@ -403,7 +437,7 @@ class CommandTestCase(IntegrationTestCase):
|
|||||||
daemon = Daemon(conf, ComponentManager(
|
daemon = Daemon(conf, ComponentManager(
|
||||||
conf, skip_components=conf.components_to_skip, wallet=wallet_maker,
|
conf, skip_components=conf.components_to_skip, wallet=wallet_maker,
|
||||||
exchange_rate_manager=partial(ExchangeRateManagerComponent, rates={
|
exchange_rate_manager=partial(ExchangeRateManagerComponent, rates={
|
||||||
'BTCLBC': 1.0, 'USDBTC': 2.0
|
'BTCLBC': 1.0, 'USDLBC': 2.0
|
||||||
})
|
})
|
||||||
))
|
))
|
||||||
await daemon.initialize()
|
await daemon.initialize()
|
||||||
@@ -432,8 +466,9 @@ class CommandTestCase(IntegrationTestCase):
|
|||||||
|
|
||||||
async def generate(self, blocks):
|
async def generate(self, blocks):
|
||||||
""" Ask lbrycrd to generate some blocks and wait until ledger has them. """
|
""" Ask lbrycrd to generate some blocks and wait until ledger has them. """
|
||||||
|
prepare = self.ledger.on_header.where(self.blockchain.is_expected_block)
|
||||||
await self.blockchain.generate(blocks)
|
await self.blockchain.generate(blocks)
|
||||||
await self.ledger.on_header.where(self.blockchain.is_expected_block)
|
await prepare # no guarantee that it didn't happen already, so start waiting from before calling generate
|
||||||
|
|
||||||
async def blockchain_claim_name(self, name: str, value: str, amount: str, confirm=True):
|
async def blockchain_claim_name(self, name: str, value: str, amount: str, confirm=True):
|
||||||
txid = await self.blockchain._cli_cmnd('claimname', name, value, amount)
|
txid = await self.blockchain._cli_cmnd('claimname', name, value, amount)
|
||||||
@@ -473,7 +508,7 @@ class CommandTestCase(IntegrationTestCase):
|
|||||||
async def stream_create(
|
async def stream_create(
|
||||||
self, name='hovercraft', bid='1.0', file_path=None,
|
self, name='hovercraft', bid='1.0', file_path=None,
|
||||||
data=b'hi!', confirm=True, prefix=None, suffix=None, **kwargs):
|
data=b'hi!', confirm=True, prefix=None, suffix=None, **kwargs):
|
||||||
if file_path is None:
|
if file_path is None and data is not None:
|
||||||
file_path = self.create_upload_file(data=data, prefix=prefix, suffix=suffix)
|
file_path = self.create_upload_file(data=data, prefix=prefix, suffix=suffix)
|
||||||
return await self.confirm_and_render(
|
return await self.confirm_and_render(
|
||||||
self.daemon.jsonrpc_stream_create(name, bid, file_path=file_path, **kwargs), confirm
|
self.daemon.jsonrpc_stream_create(name, bid, file_path=file_path, **kwargs), confirm
|
||||||
@@ -490,8 +525,8 @@ class CommandTestCase(IntegrationTestCase):
|
|||||||
self.daemon.jsonrpc_stream_update(claim_id, **kwargs), confirm
|
self.daemon.jsonrpc_stream_update(claim_id, **kwargs), confirm
|
||||||
)
|
)
|
||||||
|
|
||||||
def stream_repost(self, claim_id, name='repost', bid='1.0', confirm=True, **kwargs):
|
async def stream_repost(self, claim_id, name='repost', bid='1.0', confirm=True, **kwargs):
|
||||||
return self.confirm_and_render(
|
return await self.confirm_and_render(
|
||||||
self.daemon.jsonrpc_stream_repost(claim_id=claim_id, name=name, bid=bid, **kwargs), confirm
|
self.daemon.jsonrpc_stream_repost(claim_id=claim_id, name=name, bid=bid, **kwargs), confirm
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -502,6 +537,11 @@ class CommandTestCase(IntegrationTestCase):
|
|||||||
self.daemon.jsonrpc_stream_abandon(*args, **kwargs), confirm
|
self.daemon.jsonrpc_stream_abandon(*args, **kwargs), confirm
|
||||||
)
|
)
|
||||||
|
|
||||||
|
async def purchase_create(self, *args, confirm=True, **kwargs):
|
||||||
|
return await self.confirm_and_render(
|
||||||
|
self.daemon.jsonrpc_purchase_create(*args, **kwargs), confirm
|
||||||
|
)
|
||||||
|
|
||||||
async def publish(self, name, *args, confirm=True, **kwargs):
|
async def publish(self, name, *args, confirm=True, **kwargs):
|
||||||
return await self.confirm_and_render(
|
return await self.confirm_and_render(
|
||||||
self.daemon.jsonrpc_publish(name, *args, **kwargs), confirm
|
self.daemon.jsonrpc_publish(name, *args, **kwargs), confirm
|
||||||
@@ -548,8 +588,39 @@ class CommandTestCase(IntegrationTestCase):
|
|||||||
self.daemon.jsonrpc_support_create(claim_id, bid, **kwargs), confirm
|
self.daemon.jsonrpc_support_create(claim_id, bid, **kwargs), confirm
|
||||||
)
|
)
|
||||||
|
|
||||||
async def resolve(self, uri):
|
async def support_abandon(self, *args, confirm=True, **kwargs):
|
||||||
return await self.out(self.daemon.jsonrpc_resolve(uri))
|
if 'blocking' not in kwargs:
|
||||||
|
kwargs['blocking'] = False
|
||||||
|
return await self.confirm_and_render(
|
||||||
|
self.daemon.jsonrpc_support_abandon(*args, **kwargs), confirm
|
||||||
|
)
|
||||||
|
|
||||||
|
async def account_send(self, *args, confirm=True, **kwargs):
|
||||||
|
return await self.confirm_and_render(
|
||||||
|
self.daemon.jsonrpc_account_send(*args, **kwargs), confirm
|
||||||
|
)
|
||||||
|
|
||||||
|
async def wallet_send(self, *args, confirm=True, **kwargs):
|
||||||
|
return await self.confirm_and_render(
|
||||||
|
self.daemon.jsonrpc_wallet_send(*args, **kwargs), confirm
|
||||||
|
)
|
||||||
|
|
||||||
|
async def txo_spend(self, *args, confirm=True, **kwargs):
|
||||||
|
txs = await self.daemon.jsonrpc_txo_spend(*args, **kwargs)
|
||||||
|
if confirm:
|
||||||
|
await asyncio.wait([self.ledger.wait(tx) for tx in txs])
|
||||||
|
await self.generate(1)
|
||||||
|
await asyncio.wait([self.ledger.wait(tx, self.blockchain.block_expected) for tx in txs])
|
||||||
|
return self.sout(txs)
|
||||||
|
|
||||||
|
async def blob_clean(self):
|
||||||
|
return await self.out(self.daemon.jsonrpc_blob_clean())
|
||||||
|
|
||||||
|
async def status(self):
|
||||||
|
return await self.out(self.daemon.jsonrpc_status())
|
||||||
|
|
||||||
|
async def resolve(self, uri, **kwargs):
|
||||||
|
return (await self.out(self.daemon.jsonrpc_resolve(uri, **kwargs)))[uri]
|
||||||
|
|
||||||
async def claim_search(self, **kwargs):
|
async def claim_search(self, **kwargs):
|
||||||
return (await self.out(self.daemon.jsonrpc_claim_search(**kwargs)))['items']
|
return (await self.out(self.daemon.jsonrpc_claim_search(**kwargs)))['items']
|
||||||
@@ -557,6 +628,15 @@ class CommandTestCase(IntegrationTestCase):
|
|||||||
async def file_list(self, *args, **kwargs):
|
async def file_list(self, *args, **kwargs):
|
||||||
return (await self.out(self.daemon.jsonrpc_file_list(*args, **kwargs)))['items']
|
return (await self.out(self.daemon.jsonrpc_file_list(*args, **kwargs)))['items']
|
||||||
|
|
||||||
|
async def txo_list(self, *args, **kwargs):
|
||||||
|
return (await self.out(self.daemon.jsonrpc_txo_list(*args, **kwargs)))['items']
|
||||||
|
|
||||||
|
async def txo_sum(self, *args, **kwargs):
|
||||||
|
return await self.out(self.daemon.jsonrpc_txo_sum(*args, **kwargs))
|
||||||
|
|
||||||
|
async def txo_plot(self, *args, **kwargs):
|
||||||
|
return await self.out(self.daemon.jsonrpc_txo_plot(*args, **kwargs))
|
||||||
|
|
||||||
async def claim_list(self, *args, **kwargs):
|
async def claim_list(self, *args, **kwargs):
|
||||||
return (await self.out(self.daemon.jsonrpc_claim_list(*args, **kwargs)))['items']
|
return (await self.out(self.daemon.jsonrpc_claim_list(*args, **kwargs)))['items']
|
||||||
|
|
||||||
@@ -566,9 +646,15 @@ class CommandTestCase(IntegrationTestCase):
|
|||||||
async def channel_list(self, *args, **kwargs):
|
async def channel_list(self, *args, **kwargs):
|
||||||
return (await self.out(self.daemon.jsonrpc_channel_list(*args, **kwargs)))['items']
|
return (await self.out(self.daemon.jsonrpc_channel_list(*args, **kwargs)))['items']
|
||||||
|
|
||||||
|
async def transaction_list(self, *args, **kwargs):
|
||||||
|
return (await self.out(self.daemon.jsonrpc_transaction_list(*args, **kwargs)))['items']
|
||||||
|
|
||||||
|
async def blob_list(self, *args, **kwargs):
|
||||||
|
return (await self.out(self.daemon.jsonrpc_blob_list(*args, **kwargs)))['items']
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_claim_id(tx):
|
def get_claim_id(tx):
|
||||||
return tx['outputs'][0]['claim_id']
|
return tx['outputs'][0]['claim_id']
|
||||||
|
|
||||||
def assertItemCount(self, result, count): # pylint: disable=invalid-name
|
def assertItemCount(self, result, count): # pylint: disable=invalid-name
|
||||||
self.assertEqual(result['total_items'], count)
|
self.assertEqual(count, result['total_items'])
|
||||||
Whitespace-only changes.
@@ -0,0 +1,290 @@
|
|||||||
|
import asyncio
|
||||||
|
import binascii
|
||||||
|
import os
|
||||||
|
import logging
|
||||||
|
import random
|
||||||
|
from hashlib import sha1
|
||||||
|
from tempfile import mkdtemp
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
import libtorrent
|
||||||
|
|
||||||
|
|
||||||
|
NOTIFICATION_MASKS = [
|
||||||
|
"error",
|
||||||
|
"peer",
|
||||||
|
"port_mapping",
|
||||||
|
"storage",
|
||||||
|
"tracker",
|
||||||
|
"debug",
|
||||||
|
"status",
|
||||||
|
"progress",
|
||||||
|
"ip_block",
|
||||||
|
"dht",
|
||||||
|
"stats",
|
||||||
|
"session_log",
|
||||||
|
"torrent_log",
|
||||||
|
"peer_log",
|
||||||
|
"incoming_request",
|
||||||
|
"dht_log",
|
||||||
|
"dht_operation",
|
||||||
|
"port_mapping_log",
|
||||||
|
"picker_log",
|
||||||
|
"file_progress",
|
||||||
|
"piece_progress",
|
||||||
|
"upload",
|
||||||
|
"block_progress"
|
||||||
|
]
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_FLAGS = ( # fixme: somehow the logic here is inverted?
|
||||||
|
libtorrent.add_torrent_params_flags_t.flag_auto_managed
|
||||||
|
| libtorrent.add_torrent_params_flags_t.flag_update_subscribe
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def get_notification_type(notification) -> str:
|
||||||
|
for i, notification_type in enumerate(NOTIFICATION_MASKS):
|
||||||
|
if (1 << i) & notification:
|
||||||
|
return notification_type
|
||||||
|
raise ValueError("unrecognized notification type")
|
||||||
|
|
||||||
|
|
||||||
|
class TorrentHandle:
|
||||||
|
def __init__(self, loop, executor, handle):
|
||||||
|
self._loop = loop
|
||||||
|
self._executor = executor
|
||||||
|
self._handle: libtorrent.torrent_handle = handle
|
||||||
|
self.started = asyncio.Event(loop=loop)
|
||||||
|
self.finished = asyncio.Event(loop=loop)
|
||||||
|
self.metadata_completed = asyncio.Event(loop=loop)
|
||||||
|
self.size = 0
|
||||||
|
self.total_wanted_done = 0
|
||||||
|
self.name = ''
|
||||||
|
self.tasks = []
|
||||||
|
self.torrent_file: Optional[libtorrent.file_storage] = None
|
||||||
|
self._base_path = None
|
||||||
|
self._handle.set_sequential_download(1)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def largest_file(self) -> Optional[str]:
|
||||||
|
if not self.torrent_file:
|
||||||
|
return None
|
||||||
|
index = self.largest_file_index
|
||||||
|
return os.path.join(self._base_path, self.torrent_file.at(index).path)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def largest_file_index(self):
|
||||||
|
largest_size, index = 0, 0
|
||||||
|
for file_num in range(self.torrent_file.num_files()):
|
||||||
|
if self.torrent_file.file_size(file_num) > largest_size:
|
||||||
|
largest_size = self.torrent_file.file_size(file_num)
|
||||||
|
index = file_num
|
||||||
|
return index
|
||||||
|
|
||||||
|
def stop_tasks(self):
|
||||||
|
while self.tasks:
|
||||||
|
self.tasks.pop().cancel()
|
||||||
|
|
||||||
|
def _show_status(self):
|
||||||
|
# fixme: cleanup
|
||||||
|
if not self._handle.is_valid():
|
||||||
|
return
|
||||||
|
status = self._handle.status()
|
||||||
|
if status.has_metadata:
|
||||||
|
self.size = status.total_wanted
|
||||||
|
self.total_wanted_done = status.total_wanted_done
|
||||||
|
self.name = status.name
|
||||||
|
if not self.metadata_completed.is_set():
|
||||||
|
self.metadata_completed.set()
|
||||||
|
log.info("Metadata completed for btih:%s - %s", status.info_hash, self.name)
|
||||||
|
self.torrent_file = self._handle.get_torrent_info().files()
|
||||||
|
self._base_path = status.save_path
|
||||||
|
first_piece = self.torrent_file.at(self.largest_file_index).offset
|
||||||
|
if not self.started.is_set():
|
||||||
|
if self._handle.have_piece(first_piece):
|
||||||
|
self.started.set()
|
||||||
|
else:
|
||||||
|
# prioritize it
|
||||||
|
self._handle.set_piece_deadline(first_piece, 100)
|
||||||
|
if not status.is_seeding:
|
||||||
|
log.debug('%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d seeds: %d) %s - %s',
|
||||||
|
status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000,
|
||||||
|
status.num_peers, status.num_seeds, status.state, status.save_path)
|
||||||
|
elif not self.finished.is_set():
|
||||||
|
self.finished.set()
|
||||||
|
log.info("Torrent finished: %s", self.name)
|
||||||
|
|
||||||
|
async def status_loop(self):
|
||||||
|
while True:
|
||||||
|
self._show_status()
|
||||||
|
if self.finished.is_set():
|
||||||
|
break
|
||||||
|
await asyncio.sleep(0.1, loop=self._loop)
|
||||||
|
|
||||||
|
async def pause(self):
|
||||||
|
await self._loop.run_in_executor(
|
||||||
|
self._executor, self._handle.pause
|
||||||
|
)
|
||||||
|
|
||||||
|
async def resume(self):
|
||||||
|
await self._loop.run_in_executor(
|
||||||
|
self._executor, lambda: self._handle.resume() # pylint: disable=unnecessary-lambda
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TorrentSession:
|
||||||
|
def __init__(self, loop, executor):
|
||||||
|
self._loop = loop
|
||||||
|
self._executor = executor
|
||||||
|
self._session: Optional[libtorrent.session] = None
|
||||||
|
self._handles = {}
|
||||||
|
self.tasks = []
|
||||||
|
self.wait_start = True
|
||||||
|
|
||||||
|
async def add_fake_torrent(self):
|
||||||
|
tmpdir = mkdtemp()
|
||||||
|
info, btih = _create_fake_torrent(tmpdir)
|
||||||
|
flags = libtorrent.add_torrent_params_flags_t.flag_seed_mode
|
||||||
|
handle = self._session.add_torrent({
|
||||||
|
'ti': info, 'save_path': tmpdir, 'flags': flags
|
||||||
|
})
|
||||||
|
self._handles[btih] = TorrentHandle(self._loop, self._executor, handle)
|
||||||
|
return btih
|
||||||
|
|
||||||
|
async def bind(self, interface: str = '0.0.0.0', port: int = 10889):
|
||||||
|
settings = {
|
||||||
|
'listen_interfaces': f"{interface}:{port}",
|
||||||
|
'enable_outgoing_utp': True,
|
||||||
|
'enable_incoming_utp': True,
|
||||||
|
'enable_outgoing_tcp': False,
|
||||||
|
'enable_incoming_tcp': False
|
||||||
|
}
|
||||||
|
self._session = await self._loop.run_in_executor(
|
||||||
|
self._executor, libtorrent.session, settings # pylint: disable=c-extension-no-member
|
||||||
|
)
|
||||||
|
self.tasks.append(self._loop.create_task(self.process_alerts()))
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
while self.tasks:
|
||||||
|
self.tasks.pop().cancel()
|
||||||
|
self._session.save_state()
|
||||||
|
self._session.pause()
|
||||||
|
self._session.stop_dht()
|
||||||
|
self._session.stop_lsd()
|
||||||
|
self._session.stop_natpmp()
|
||||||
|
self._session.stop_upnp()
|
||||||
|
self._session = None
|
||||||
|
|
||||||
|
def _pop_alerts(self):
|
||||||
|
for alert in self._session.pop_alerts():
|
||||||
|
log.info("torrent alert: %s", alert)
|
||||||
|
|
||||||
|
async def process_alerts(self):
|
||||||
|
while True:
|
||||||
|
await self._loop.run_in_executor(
|
||||||
|
self._executor, self._pop_alerts
|
||||||
|
)
|
||||||
|
await asyncio.sleep(1, loop=self._loop)
|
||||||
|
|
||||||
|
async def pause(self):
|
||||||
|
await self._loop.run_in_executor(
|
||||||
|
self._executor, lambda: self._session.save_state() # pylint: disable=unnecessary-lambda
|
||||||
|
)
|
||||||
|
await self._loop.run_in_executor(
|
||||||
|
self._executor, lambda: self._session.pause() # pylint: disable=unnecessary-lambda
|
||||||
|
)
|
||||||
|
|
||||||
|
async def resume(self):
|
||||||
|
await self._loop.run_in_executor(
|
||||||
|
self._executor, self._session.resume
|
||||||
|
)
|
||||||
|
|
||||||
|
def _add_torrent(self, btih: str, download_directory: Optional[str]):
|
||||||
|
params = {'info_hash': binascii.unhexlify(btih.encode()), 'flags': DEFAULT_FLAGS}
|
||||||
|
if download_directory:
|
||||||
|
params['save_path'] = download_directory
|
||||||
|
handle = self._session.add_torrent(params)
|
||||||
|
handle.force_dht_announce()
|
||||||
|
self._handles[btih] = TorrentHandle(self._loop, self._executor, handle)
|
||||||
|
|
||||||
|
def full_path(self, btih):
|
||||||
|
return self._handles[btih].largest_file
|
||||||
|
|
||||||
|
async def add_torrent(self, btih, download_path):
|
||||||
|
await self._loop.run_in_executor(
|
||||||
|
self._executor, self._add_torrent, btih, download_path
|
||||||
|
)
|
||||||
|
self._handles[btih].tasks.append(self._loop.create_task(self._handles[btih].status_loop()))
|
||||||
|
await self._handles[btih].metadata_completed.wait()
|
||||||
|
if self.wait_start:
|
||||||
|
# fixme: temporary until we add streaming support, otherwise playback fails!
|
||||||
|
await self._handles[btih].started.wait()
|
||||||
|
|
||||||
|
def remove_torrent(self, btih, remove_files=False):
|
||||||
|
if btih in self._handles:
|
||||||
|
handle = self._handles[btih]
|
||||||
|
handle.stop_tasks()
|
||||||
|
self._session.remove_torrent(handle._handle, 1 if remove_files else 0)
|
||||||
|
self._handles.pop(btih)
|
||||||
|
|
||||||
|
async def save_file(self, btih, download_directory):
|
||||||
|
handle = self._handles[btih]
|
||||||
|
await handle.resume()
|
||||||
|
|
||||||
|
def get_size(self, btih):
|
||||||
|
return self._handles[btih].size
|
||||||
|
|
||||||
|
def get_name(self, btih):
|
||||||
|
return self._handles[btih].name
|
||||||
|
|
||||||
|
def get_downloaded(self, btih):
|
||||||
|
return self._handles[btih].total_wanted_done
|
||||||
|
|
||||||
|
def is_completed(self, btih):
|
||||||
|
return self._handles[btih].finished.is_set()
|
||||||
|
|
||||||
|
|
||||||
|
def get_magnet_uri(btih):
|
||||||
|
return f"magnet:?xt=urn:btih:{btih}"
|
||||||
|
|
||||||
|
|
||||||
|
def _create_fake_torrent(tmpdir):
|
||||||
|
# beware, that's just for testing
|
||||||
|
path = os.path.join(tmpdir, 'tmp')
|
||||||
|
with open(path, 'wb') as myfile:
|
||||||
|
size = myfile.write(bytes([random.randint(0, 255) for _ in range(40)]) * 1024)
|
||||||
|
file_storage = libtorrent.file_storage()
|
||||||
|
file_storage.add_file('tmp', size)
|
||||||
|
t = libtorrent.create_torrent(file_storage, 0, 4 * 1024 * 1024)
|
||||||
|
libtorrent.set_piece_hashes(t, tmpdir)
|
||||||
|
info = libtorrent.torrent_info(t.generate())
|
||||||
|
btih = sha1(info.metadata()).hexdigest()
|
||||||
|
return info, btih
|
||||||
|
|
||||||
|
|
||||||
|
async def main():
|
||||||
|
if os.path.exists("~/Downloads/ubuntu-18.04.3-live-server-amd64.torrent"):
|
||||||
|
os.remove("~/Downloads/ubuntu-18.04.3-live-server-amd64.torrent")
|
||||||
|
if os.path.exists("~/Downloads/ubuntu-18.04.3-live-server-amd64.iso"):
|
||||||
|
os.remove("~/Downloads/ubuntu-18.04.3-live-server-amd64.iso")
|
||||||
|
|
||||||
|
btih = "dd8255ecdc7ca55fb0bbf81323d87062db1f6d1c"
|
||||||
|
|
||||||
|
executor = None
|
||||||
|
session = TorrentSession(asyncio.get_event_loop(), executor)
|
||||||
|
session2 = TorrentSession(asyncio.get_event_loop(), executor)
|
||||||
|
await session.bind('localhost', port=4040)
|
||||||
|
await session2.bind('localhost', port=4041)
|
||||||
|
btih = await session.add_fake_torrent()
|
||||||
|
session2._session.add_dht_node(('localhost', 4040))
|
||||||
|
await session2.add_torrent(btih, "/tmp/down")
|
||||||
|
while True:
|
||||||
|
await asyncio.sleep(100)
|
||||||
|
await session.pause()
|
||||||
|
executor.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
asyncio.run(main())
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
import typing
|
||||||
|
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class TorrentInfo:
|
||||||
|
__slots__ = ('dht_seeds', 'http_seeds', 'trackers', 'total_size')
|
||||||
|
|
||||||
|
def __init__(self, dht_seeds: typing.Tuple[typing.Tuple[str, int]],
|
||||||
|
http_seeds: typing.Tuple[typing.Dict[str, typing.Any]],
|
||||||
|
trackers: typing.Tuple[typing.Tuple[str, int]], total_size: int):
|
||||||
|
self.dht_seeds = dht_seeds
|
||||||
|
self.http_seeds = http_seeds
|
||||||
|
self.trackers = trackers
|
||||||
|
self.total_size = total_size
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_libtorrent_info(cls, torrent_info):
|
||||||
|
return cls(
|
||||||
|
torrent_info.nodes(), tuple(
|
||||||
|
{
|
||||||
|
'url': web_seed['url'],
|
||||||
|
'type': web_seed['type'],
|
||||||
|
'auth': web_seed['auth']
|
||||||
|
} for web_seed in torrent_info.web_seeds()
|
||||||
|
), tuple(
|
||||||
|
(tracker.url, tracker.tier) for tracker in torrent_info.trackers()
|
||||||
|
), torrent_info.total_size()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class Torrent:
|
||||||
|
def __init__(self, loop, handle):
|
||||||
|
self._loop = loop
|
||||||
|
self._handle = handle
|
||||||
|
self.finished = asyncio.Event(loop=loop)
|
||||||
|
|
||||||
|
def _threaded_update_status(self):
|
||||||
|
status = self._handle.status()
|
||||||
|
if not status.is_seeding:
|
||||||
|
log.info(
|
||||||
|
'%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d) %s',
|
||||||
|
status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000,
|
||||||
|
status.num_peers, status.state
|
||||||
|
)
|
||||||
|
elif not self.finished.is_set():
|
||||||
|
self.finished.set()
|
||||||
|
|
||||||
|
async def wait_for_finished(self):
|
||||||
|
while True:
|
||||||
|
await self._loop.run_in_executor(
|
||||||
|
None, self._threaded_update_status
|
||||||
|
)
|
||||||
|
if self.finished.is_set():
|
||||||
|
log.info("finished downloading torrent!")
|
||||||
|
await self.pause()
|
||||||
|
break
|
||||||
|
await asyncio.sleep(1, loop=self._loop)
|
||||||
|
|
||||||
|
async def pause(self):
|
||||||
|
log.info("pause torrent")
|
||||||
|
await self._loop.run_in_executor(
|
||||||
|
None, self._handle.pause
|
||||||
|
)
|
||||||
|
|
||||||
|
async def resume(self):
|
||||||
|
await self._loop.run_in_executor(
|
||||||
|
None, self._handle.resume
|
||||||
|
)
|
||||||
@@ -0,0 +1,140 @@
|
|||||||
|
import asyncio
|
||||||
|
import binascii
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import typing
|
||||||
|
from typing import Optional
|
||||||
|
from aiohttp.web import Request
|
||||||
|
from lbry.file.source_manager import SourceManager
|
||||||
|
from lbry.file.source import ManagedDownloadSource
|
||||||
|
|
||||||
|
if typing.TYPE_CHECKING:
|
||||||
|
from lbry.torrent.session import TorrentSession
|
||||||
|
from lbry.conf import Config
|
||||||
|
from lbry.wallet.transaction import Transaction
|
||||||
|
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||||
|
from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim
|
||||||
|
from lbry.extras.daemon.storage import StoredContentClaim
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def path_or_none(encoded_path) -> Optional[str]:
|
||||||
|
if not encoded_path:
|
||||||
|
return
|
||||||
|
return binascii.unhexlify(encoded_path).decode()
|
||||||
|
|
||||||
|
|
||||||
|
class TorrentSource(ManagedDownloadSource):
|
||||||
|
STATUS_STOPPED = "stopped"
|
||||||
|
filter_fields = SourceManager.filter_fields
|
||||||
|
filter_fields.update({
|
||||||
|
'bt_infohash'
|
||||||
|
})
|
||||||
|
|
||||||
|
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage', identifier: str,
|
||||||
|
file_name: Optional[str] = None, download_directory: Optional[str] = None,
|
||||||
|
status: Optional[str] = STATUS_STOPPED, claim: Optional['StoredContentClaim'] = None,
|
||||||
|
download_id: Optional[str] = None, rowid: Optional[int] = None,
|
||||||
|
content_fee: Optional['Transaction'] = None,
|
||||||
|
analytics_manager: Optional['AnalyticsManager'] = None,
|
||||||
|
added_on: Optional[int] = None, torrent_session: Optional['TorrentSession'] = None):
|
||||||
|
super().__init__(loop, config, storage, identifier, file_name, download_directory, status, claim, download_id,
|
||||||
|
rowid, content_fee, analytics_manager, added_on)
|
||||||
|
self.torrent_session = torrent_session
|
||||||
|
|
||||||
|
@property
|
||||||
|
def full_path(self) -> Optional[str]:
|
||||||
|
full_path = self.torrent_session.full_path(self.identifier)
|
||||||
|
self.download_directory = os.path.dirname(full_path)
|
||||||
|
return full_path
|
||||||
|
|
||||||
|
async def start(self, timeout: Optional[float] = None, save_now: Optional[bool] = False):
|
||||||
|
await self.torrent_session.add_torrent(self.identifier, self.download_directory)
|
||||||
|
|
||||||
|
async def stop(self, finished: bool = False):
|
||||||
|
await self.torrent_session.remove_torrent(self.identifier)
|
||||||
|
|
||||||
|
async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
|
||||||
|
await self.torrent_session.save_file(self.identifier, download_directory)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def torrent_length(self):
|
||||||
|
return self.torrent_session.get_size(self.identifier)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def written_bytes(self):
|
||||||
|
return self.torrent_session.get_downloaded(self.identifier)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def torrent_name(self):
|
||||||
|
return self.torrent_session.get_name(self.identifier)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def bt_infohash(self):
|
||||||
|
return self.identifier
|
||||||
|
|
||||||
|
def stop_tasks(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
@property
|
||||||
|
def completed(self):
|
||||||
|
return self.torrent_session.is_completed(self.identifier)
|
||||||
|
|
||||||
|
|
||||||
|
class TorrentManager(SourceManager):
|
||||||
|
_sources: typing.Dict[str, ManagedDownloadSource]
|
||||||
|
|
||||||
|
filter_fields = set(SourceManager.filter_fields)
|
||||||
|
filter_fields.update({
|
||||||
|
'bt_infohash',
|
||||||
|
'blobs_remaining', # TODO: here they call them "parts", but its pretty much the same concept
|
||||||
|
'blobs_in_stream'
|
||||||
|
})
|
||||||
|
|
||||||
|
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', torrent_session: 'TorrentSession',
|
||||||
|
storage: 'SQLiteStorage', analytics_manager: Optional['AnalyticsManager'] = None):
|
||||||
|
super().__init__(loop, config, storage, analytics_manager)
|
||||||
|
self.torrent_session: 'TorrentSession' = torrent_session
|
||||||
|
|
||||||
|
async def recover_streams(self, file_infos: typing.List[typing.Dict]):
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
async def _load_stream(self, rowid: int, bt_infohash: str, file_name: Optional[str],
|
||||||
|
download_directory: Optional[str], status: str,
|
||||||
|
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
|
||||||
|
added_on: Optional[int]):
|
||||||
|
stream = TorrentSource(
|
||||||
|
self.loop, self.config, self.storage, identifier=bt_infohash, file_name=file_name,
|
||||||
|
download_directory=download_directory, status=status, claim=claim, rowid=rowid,
|
||||||
|
content_fee=content_fee, analytics_manager=self.analytics_manager, added_on=added_on,
|
||||||
|
torrent_session=self.torrent_session
|
||||||
|
)
|
||||||
|
self.add(stream)
|
||||||
|
|
||||||
|
async def initialize_from_database(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
async def start(self):
|
||||||
|
await super().start()
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
super().stop()
|
||||||
|
log.info("finished stopping the torrent manager")
|
||||||
|
|
||||||
|
async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
|
||||||
|
await super().delete(source, delete_file)
|
||||||
|
self.torrent_session.remove_torrent(source.identifier, delete_file)
|
||||||
|
|
||||||
|
async def create(self, file_path: str, key: Optional[bytes] = None,
|
||||||
|
iv_generator: Optional[typing.Generator[bytes, None, None]] = None):
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
async def _delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
|
||||||
|
raise NotImplementedError
|
||||||
|
# blob_hashes = [source.sd_hash] + [b.blob_hash for b in source.descriptor.blobs[:-1]]
|
||||||
|
# await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False)
|
||||||
|
# await self.storage.delete_stream(source.descriptor)
|
||||||
|
|
||||||
|
async def stream_partial_content(self, request: Request, sd_hash: str):
|
||||||
|
raise NotImplementedError
|
||||||
+216
-18
@@ -3,7 +3,9 @@ import codecs
|
|||||||
import datetime
|
import datetime
|
||||||
import random
|
import random
|
||||||
import socket
|
import socket
|
||||||
|
import time
|
||||||
import string
|
import string
|
||||||
|
import sys
|
||||||
import json
|
import json
|
||||||
import typing
|
import typing
|
||||||
import asyncio
|
import asyncio
|
||||||
@@ -18,8 +20,10 @@ import pkg_resources
|
|||||||
|
|
||||||
import certifi
|
import certifi
|
||||||
import aiohttp
|
import aiohttp
|
||||||
|
from prometheus_client import Counter
|
||||||
from lbry.schema.claim import Claim
|
from lbry.schema.claim import Claim
|
||||||
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
@@ -100,10 +104,6 @@ def check_connection(server="lbry.com", port=80, timeout=5) -> bool:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
async def async_check_connection(server="lbry.com", port=80, timeout=1) -> bool:
|
|
||||||
return await asyncio.get_event_loop().run_in_executor(None, check_connection, server, port, timeout)
|
|
||||||
|
|
||||||
|
|
||||||
def random_string(length=10, chars=string.ascii_lowercase):
|
def random_string(length=10, chars=string.ascii_lowercase):
|
||||||
return ''.join([random.choice(chars) for _ in range(length)])
|
return ''.join([random.choice(chars) for _ in range(length)])
|
||||||
|
|
||||||
@@ -155,7 +155,7 @@ def async_timed_cache(duration: int):
|
|||||||
async def _inner(*args, **kwargs) -> typing.Any:
|
async def _inner(*args, **kwargs) -> typing.Any:
|
||||||
loop = asyncio.get_running_loop()
|
loop = asyncio.get_running_loop()
|
||||||
time_now = loop.time()
|
time_now = loop.time()
|
||||||
key = tuple([args, tuple([tuple([k, kwargs[k]]) for k in kwargs])])
|
key = (args, tuple(kwargs.items()))
|
||||||
if key in cache and (time_now - cache[key][1] < duration):
|
if key in cache and (time_now - cache[key][1] < duration):
|
||||||
return cache[key][0]
|
return cache[key][0]
|
||||||
to_cache = await func(*args, **kwargs)
|
to_cache = await func(*args, **kwargs)
|
||||||
@@ -173,7 +173,7 @@ def cache_concurrent(async_fn):
|
|||||||
|
|
||||||
@functools.wraps(async_fn)
|
@functools.wraps(async_fn)
|
||||||
async def wrapper(*args, **kwargs):
|
async def wrapper(*args, **kwargs):
|
||||||
key = tuple([args, tuple([tuple([k, kwargs[k]]) for k in kwargs])])
|
key = (args, tuple(kwargs.items()))
|
||||||
cache[key] = cache.get(key) or asyncio.create_task(async_fn(*args, **kwargs))
|
cache[key] = cache.get(key) or asyncio.create_task(async_fn(*args, **kwargs))
|
||||||
try:
|
try:
|
||||||
return await cache[key]
|
return await cache[key]
|
||||||
@@ -187,6 +187,8 @@ def cache_concurrent(async_fn):
|
|||||||
async def resolve_host(url: str, port: int, proto: str) -> str:
|
async def resolve_host(url: str, port: int, proto: str) -> str:
|
||||||
if proto not in ['udp', 'tcp']:
|
if proto not in ['udp', 'tcp']:
|
||||||
raise Exception("invalid protocol")
|
raise Exception("invalid protocol")
|
||||||
|
if url.lower() == 'localhost':
|
||||||
|
return '127.0.0.1'
|
||||||
try:
|
try:
|
||||||
if ipaddress.ip_address(url):
|
if ipaddress.ip_address(url):
|
||||||
return url
|
return url
|
||||||
@@ -201,18 +203,44 @@ async def resolve_host(url: str, port: int, proto: str) -> str:
|
|||||||
))[0][4][0]
|
))[0][4][0]
|
||||||
|
|
||||||
|
|
||||||
class LRUCache:
|
class LRUCacheWithMetrics:
|
||||||
__slots__ = [
|
__slots__ = [
|
||||||
'capacity',
|
'capacity',
|
||||||
'cache'
|
'cache',
|
||||||
|
'_track_metrics',
|
||||||
|
'hits',
|
||||||
|
'misses'
|
||||||
]
|
]
|
||||||
|
|
||||||
def __init__(self, capacity):
|
def __init__(self, capacity: int, metric_name: typing.Optional[str] = None, namespace: str = "daemon_cache"):
|
||||||
self.capacity = capacity
|
self.capacity = capacity
|
||||||
self.cache = collections.OrderedDict()
|
self.cache = collections.OrderedDict()
|
||||||
|
if metric_name is None:
|
||||||
|
self._track_metrics = False
|
||||||
|
self.hits = self.misses = None
|
||||||
|
else:
|
||||||
|
self._track_metrics = True
|
||||||
|
try:
|
||||||
|
self.hits = Counter(
|
||||||
|
f"{metric_name}_cache_hit_count", "Number of cache hits", namespace=namespace
|
||||||
|
)
|
||||||
|
self.misses = Counter(
|
||||||
|
f"{metric_name}_cache_miss_count", "Number of cache misses", namespace=namespace
|
||||||
|
)
|
||||||
|
except ValueError as err:
|
||||||
|
log.debug("failed to set up prometheus %s_cache_miss_count metric: %s", metric_name, err)
|
||||||
|
self._track_metrics = False
|
||||||
|
self.hits = self.misses = None
|
||||||
|
|
||||||
def get(self, key):
|
def get(self, key, default=None):
|
||||||
value = self.cache.pop(key)
|
try:
|
||||||
|
value = self.cache.pop(key)
|
||||||
|
if self._track_metrics:
|
||||||
|
self.hits.inc()
|
||||||
|
except KeyError:
|
||||||
|
if self._track_metrics:
|
||||||
|
self.misses.inc()
|
||||||
|
return default
|
||||||
self.cache[key] = value
|
self.cache[key] = value
|
||||||
return value
|
return value
|
||||||
|
|
||||||
@@ -224,22 +252,97 @@ class LRUCache:
|
|||||||
self.cache.popitem(last=False)
|
self.cache.popitem(last=False)
|
||||||
self.cache[key] = value
|
self.cache[key] = value
|
||||||
|
|
||||||
|
def clear(self):
|
||||||
|
self.cache.clear()
|
||||||
|
|
||||||
|
def pop(self, key):
|
||||||
|
return self.cache.pop(key)
|
||||||
|
|
||||||
|
def __setitem__(self, key, value):
|
||||||
|
return self.set(key, value)
|
||||||
|
|
||||||
|
def __getitem__(self, item):
|
||||||
|
return self.get(item)
|
||||||
|
|
||||||
def __contains__(self, item) -> bool:
|
def __contains__(self, item) -> bool:
|
||||||
return item in self.cache
|
return item in self.cache
|
||||||
|
|
||||||
|
def __len__(self):
|
||||||
|
return len(self.cache)
|
||||||
|
|
||||||
|
def __delitem__(self, key):
|
||||||
|
self.cache.pop(key)
|
||||||
|
|
||||||
|
def __del__(self):
|
||||||
|
self.clear()
|
||||||
|
|
||||||
|
|
||||||
|
class LRUCache:
|
||||||
|
__slots__ = [
|
||||||
|
'capacity',
|
||||||
|
'cache'
|
||||||
|
]
|
||||||
|
|
||||||
|
def __init__(self, capacity: int):
|
||||||
|
self.capacity = capacity
|
||||||
|
self.cache = collections.OrderedDict()
|
||||||
|
|
||||||
|
def get(self, key, default=None):
|
||||||
|
try:
|
||||||
|
value = self.cache.pop(key)
|
||||||
|
except KeyError:
|
||||||
|
return default
|
||||||
|
self.cache[key] = value
|
||||||
|
return value
|
||||||
|
|
||||||
|
def set(self, key, value):
|
||||||
|
try:
|
||||||
|
self.cache.pop(key)
|
||||||
|
except KeyError:
|
||||||
|
if len(self.cache) >= self.capacity:
|
||||||
|
self.cache.popitem(last=False)
|
||||||
|
self.cache[key] = value
|
||||||
|
|
||||||
|
def items(self):
|
||||||
|
return self.cache.items()
|
||||||
|
|
||||||
|
def clear(self):
|
||||||
|
self.cache.clear()
|
||||||
|
|
||||||
|
def pop(self, key, default=None):
|
||||||
|
return self.cache.pop(key, default)
|
||||||
|
|
||||||
|
def __setitem__(self, key, value):
|
||||||
|
return self.set(key, value)
|
||||||
|
|
||||||
|
def __getitem__(self, item):
|
||||||
|
return self.get(item)
|
||||||
|
|
||||||
|
def __contains__(self, item) -> bool:
|
||||||
|
return item in self.cache
|
||||||
|
|
||||||
|
def __len__(self):
|
||||||
|
return len(self.cache)
|
||||||
|
|
||||||
|
def __delitem__(self, key):
|
||||||
|
self.cache.pop(key)
|
||||||
|
|
||||||
|
def __del__(self):
|
||||||
|
self.clear()
|
||||||
|
|
||||||
|
|
||||||
def lru_cache_concurrent(cache_size: typing.Optional[int] = None,
|
def lru_cache_concurrent(cache_size: typing.Optional[int] = None,
|
||||||
override_lru_cache: typing.Optional[LRUCache] = None):
|
override_lru_cache: typing.Optional[LRUCacheWithMetrics] = None):
|
||||||
if not cache_size and override_lru_cache is None:
|
if not cache_size and override_lru_cache is None:
|
||||||
raise ValueError("invalid cache size")
|
raise ValueError("invalid cache size")
|
||||||
concurrent_cache = {}
|
concurrent_cache = {}
|
||||||
lru_cache = override_lru_cache or LRUCache(cache_size)
|
lru_cache = override_lru_cache if override_lru_cache is not None else LRUCacheWithMetrics(cache_size)
|
||||||
|
|
||||||
def wrapper(async_fn):
|
def wrapper(async_fn):
|
||||||
|
|
||||||
@functools.wraps(async_fn)
|
@functools.wraps(async_fn)
|
||||||
async def _inner(*args, **kwargs):
|
async def _inner(*args, **kwargs):
|
||||||
key = tuple([args, tuple([tuple([k, kwargs[k]]) for k in kwargs])])
|
key = (args, tuple(kwargs.items()))
|
||||||
if key in lru_cache:
|
if key in lru_cache:
|
||||||
return lru_cache.get(key)
|
return lru_cache.get(key)
|
||||||
|
|
||||||
@@ -264,15 +367,110 @@ def get_ssl_context() -> ssl.SSLContext:
|
|||||||
@contextlib.asynccontextmanager
|
@contextlib.asynccontextmanager
|
||||||
async def aiohttp_request(method, url, **kwargs) -> typing.AsyncContextManager[aiohttp.ClientResponse]:
|
async def aiohttp_request(method, url, **kwargs) -> typing.AsyncContextManager[aiohttp.ClientResponse]:
|
||||||
async with aiohttp.ClientSession() as session:
|
async with aiohttp.ClientSession() as session:
|
||||||
async with session.request(method, url, ssl=get_ssl_context(), **kwargs) as response:
|
async with session.request(method, url, **kwargs) as response:
|
||||||
yield response
|
yield response
|
||||||
|
|
||||||
|
|
||||||
async def get_external_ip() -> typing.Optional[str]: # used if upnp is disabled or non-functioning
|
# the ipaddress module does not show these subnets as reserved
|
||||||
|
CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
|
||||||
|
IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
|
||||||
|
|
||||||
|
|
||||||
|
def is_valid_public_ipv4(address, allow_localhost: bool = False, allow_lan: bool = False):
|
||||||
|
try:
|
||||||
|
parsed_ip = ipaddress.ip_address(address)
|
||||||
|
if parsed_ip.is_loopback and allow_localhost:
|
||||||
|
return True
|
||||||
|
if allow_lan and parsed_ip.is_private:
|
||||||
|
return True
|
||||||
|
if any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local, parsed_ip.is_loopback,
|
||||||
|
parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private)):
|
||||||
|
return False
|
||||||
|
else:
|
||||||
|
return not any((CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
|
||||||
|
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
|
||||||
|
except (ipaddress.AddressValueError, ValueError):
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
async def fallback_get_external_ip(): # used if spv servers can't be used for ip detection
|
||||||
try:
|
try:
|
||||||
async with aiohttp_request("get", "https://api.lbry.com/ip") as resp:
|
async with aiohttp_request("get", "https://api.lbry.com/ip") as resp:
|
||||||
response = await resp.json()
|
response = await resp.json()
|
||||||
if response['success']:
|
if response['success']:
|
||||||
return response['data']['ip']
|
return response['data']['ip'], None
|
||||||
except Exception:
|
except Exception:
|
||||||
return
|
return None, None
|
||||||
|
|
||||||
|
|
||||||
|
async def _get_external_ip(default_servers) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]:
|
||||||
|
# used if upnp is disabled or non-functioning
|
||||||
|
from lbry.wallet.server.udp import SPVStatusClientProtocol # pylint: disable=C0415
|
||||||
|
|
||||||
|
hostname_to_ip = {}
|
||||||
|
ip_to_hostnames = collections.defaultdict(list)
|
||||||
|
|
||||||
|
async def resolve_spv(server, port):
|
||||||
|
try:
|
||||||
|
server_addr = await resolve_host(server, port, 'udp')
|
||||||
|
hostname_to_ip[server] = (server_addr, port)
|
||||||
|
ip_to_hostnames[(server_addr, port)].append(server)
|
||||||
|
except Exception:
|
||||||
|
log.exception("error looking up dns for spv servers")
|
||||||
|
|
||||||
|
# accumulate the dns results
|
||||||
|
await asyncio.gather(*(resolve_spv(server, port) for (server, port) in default_servers))
|
||||||
|
|
||||||
|
loop = asyncio.get_event_loop()
|
||||||
|
pong_responses = asyncio.Queue()
|
||||||
|
connection = SPVStatusClientProtocol(pong_responses)
|
||||||
|
try:
|
||||||
|
await loop.create_datagram_endpoint(lambda: connection, ('0.0.0.0', 0))
|
||||||
|
# could raise OSError if it cant bind
|
||||||
|
randomized_servers = list(ip_to_hostnames.keys())
|
||||||
|
random.shuffle(randomized_servers)
|
||||||
|
for server in randomized_servers:
|
||||||
|
connection.ping(server)
|
||||||
|
try:
|
||||||
|
_, pong = await asyncio.wait_for(pong_responses.get(), 1)
|
||||||
|
if is_valid_public_ipv4(pong.ip_address):
|
||||||
|
return pong.ip_address, ip_to_hostnames[server][0]
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
pass
|
||||||
|
return None, None
|
||||||
|
finally:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
|
||||||
|
async def get_external_ip(default_servers) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]:
|
||||||
|
ip_from_spv_servers = await _get_external_ip(default_servers)
|
||||||
|
if not ip_from_spv_servers[1]:
|
||||||
|
return await fallback_get_external_ip()
|
||||||
|
return ip_from_spv_servers
|
||||||
|
|
||||||
|
|
||||||
|
def is_running_from_bundle():
|
||||||
|
# see https://pyinstaller.readthedocs.io/en/stable/runtime-information.html
|
||||||
|
return getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS')
|
||||||
|
|
||||||
|
|
||||||
|
class LockWithMetrics(asyncio.Lock):
|
||||||
|
def __init__(self, acquire_metric, held_time_metric, loop=None):
|
||||||
|
super().__init__(loop=loop)
|
||||||
|
self._acquire_metric = acquire_metric
|
||||||
|
self._lock_held_time_metric = held_time_metric
|
||||||
|
self._lock_acquired_time = None
|
||||||
|
|
||||||
|
async def acquire(self):
|
||||||
|
start = time.perf_counter()
|
||||||
|
try:
|
||||||
|
return await super().acquire()
|
||||||
|
finally:
|
||||||
|
self._lock_acquired_time = time.perf_counter()
|
||||||
|
self._acquire_metric.observe(self._lock_acquired_time - start)
|
||||||
|
|
||||||
|
def release(self):
|
||||||
|
try:
|
||||||
|
return super().release()
|
||||||
|
finally:
|
||||||
|
self._lock_held_time_metric.observe(time.perf_counter() - self._lock_acquired_time)
|
||||||
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
|
|||||||
__node_cli__ = 'lbrycrd-cli'
|
__node_cli__ = 'lbrycrd-cli'
|
||||||
__node_bin__ = ''
|
__node_bin__ = ''
|
||||||
__node_url__ = (
|
__node_url__ = (
|
||||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.3.2/lbrycrd-linux-1732.zip'
|
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.6/lbrycrd-linux-1746.zip'
|
||||||
)
|
)
|
||||||
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
|
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
|
||||||
|
|
||||||
|
|||||||
Binary file not shown.
+44
-50
@@ -5,7 +5,6 @@ import logging
|
|||||||
import typing
|
import typing
|
||||||
import asyncio
|
import asyncio
|
||||||
import random
|
import random
|
||||||
from functools import partial
|
|
||||||
from hashlib import sha256
|
from hashlib import sha256
|
||||||
from string import hexdigits
|
from string import hexdigits
|
||||||
from typing import Type, Dict, Tuple, Optional, Any, List
|
from typing import Type, Dict, Tuple, Optional, Any, List
|
||||||
@@ -16,7 +15,7 @@ from lbry.crypto.crypt import aes_encrypt, aes_decrypt
|
|||||||
|
|
||||||
from .bip32 import PrivateKey, PubKey, from_extended_key_string
|
from .bip32 import PrivateKey, PubKey, from_extended_key_string
|
||||||
from .mnemonic import Mnemonic
|
from .mnemonic import Mnemonic
|
||||||
from .constants import COIN, CLAIM_TYPES, TXO_TYPES
|
from .constants import COIN, TXO_TYPES
|
||||||
from .transaction import Transaction, Input, Output
|
from .transaction import Transaction, Input, Output
|
||||||
|
|
||||||
if typing.TYPE_CHECKING:
|
if typing.TYPE_CHECKING:
|
||||||
@@ -71,6 +70,7 @@ class AddressManager:
|
|||||||
|
|
||||||
def _query_addresses(self, **constraints):
|
def _query_addresses(self, **constraints):
|
||||||
return self.account.ledger.db.get_addresses(
|
return self.account.ledger.db.get_addresses(
|
||||||
|
read_only=constraints.pop("read_only", False),
|
||||||
accounts=[self.account],
|
accounts=[self.account],
|
||||||
chain=self.chain_number,
|
chain=self.chain_number,
|
||||||
**constraints
|
**constraints
|
||||||
@@ -96,7 +96,8 @@ class AddressManager:
|
|||||||
return [r['address'] for r in records]
|
return [r['address'] for r in records]
|
||||||
|
|
||||||
async def get_or_create_usable_address(self) -> str:
|
async def get_or_create_usable_address(self) -> str:
|
||||||
addresses = await self.get_addresses(only_usable=True, limit=10)
|
async with self.address_generator_lock:
|
||||||
|
addresses = await self.get_addresses(only_usable=True, limit=10)
|
||||||
if addresses:
|
if addresses:
|
||||||
return random.choice(addresses)
|
return random.choice(addresses)
|
||||||
addresses = await self.ensure_address_gap()
|
addresses = await self.ensure_address_gap()
|
||||||
@@ -249,7 +250,7 @@ class Account:
|
|||||||
generator_name = address_generator.get('name', HierarchicalDeterministic.name)
|
generator_name = address_generator.get('name', HierarchicalDeterministic.name)
|
||||||
self.address_generator = self.address_generators[generator_name]
|
self.address_generator = self.address_generators[generator_name]
|
||||||
self.receiving, self.change = self.address_generator.from_dict(self, address_generator)
|
self.receiving, self.change = self.address_generator.from_dict(self, address_generator)
|
||||||
self.address_managers = {am.chain_number: am for am in {self.receiving, self.change}}
|
self.address_managers = {am.chain_number: am for am in (self.receiving, self.change)}
|
||||||
self.channel_keys = channel_keys
|
self.channel_keys = channel_keys
|
||||||
ledger.add_account(self)
|
ledger.add_account(self)
|
||||||
wallet.add_account(self)
|
wallet.add_account(self)
|
||||||
@@ -310,7 +311,7 @@ class Account:
|
|||||||
private_key=private_key,
|
private_key=private_key,
|
||||||
public_key=public_key,
|
public_key=public_key,
|
||||||
address_generator=d.get('address_generator', {}),
|
address_generator=d.get('address_generator', {}),
|
||||||
modified_on=d.get('modified_on', time.time()),
|
modified_on=int(d.get('modified_on', time.time())),
|
||||||
channel_keys=d.get('certificates', {})
|
channel_keys=d.get('certificates', {})
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -342,7 +343,7 @@ class Account:
|
|||||||
def merge(self, d: dict):
|
def merge(self, d: dict):
|
||||||
if d.get('modified_on', 0) > self.modified_on:
|
if d.get('modified_on', 0) > self.modified_on:
|
||||||
self.name = d['name']
|
self.name = d['name']
|
||||||
self.modified_on = d.get('modified_on', time.time())
|
self.modified_on = int(d.get('modified_on', time.time()))
|
||||||
assert self.address_generator.name == d['address_generator']['name']
|
assert self.address_generator.name == d['address_generator']['name']
|
||||||
for chain_name in ('change', 'receiving'):
|
for chain_name in ('change', 'receiving'):
|
||||||
if chain_name in d['address_generator']:
|
if chain_name in d['address_generator']:
|
||||||
@@ -434,9 +435,9 @@ class Account:
|
|||||||
addresses.extend(new_addresses)
|
addresses.extend(new_addresses)
|
||||||
return addresses
|
return addresses
|
||||||
|
|
||||||
async def get_addresses(self, **constraints) -> List[str]:
|
async def get_addresses(self, read_only=False, **constraints) -> List[str]:
|
||||||
rows = await self.ledger.db.select_addresses('address', accounts=[self], **constraints)
|
rows = await self.ledger.db.select_addresses('address', read_only=read_only, accounts=[self], **constraints)
|
||||||
return [r[0] for r in rows]
|
return [r['address'] for r in rows]
|
||||||
|
|
||||||
def get_address_records(self, **constraints):
|
def get_address_records(self, **constraints):
|
||||||
return self.ledger.db.get_addresses(accounts=[self], **constraints)
|
return self.ledger.db.get_addresses(accounts=[self], **constraints)
|
||||||
@@ -451,13 +452,13 @@ class Account:
|
|||||||
def get_public_key(self, chain: int, index: int) -> PubKey:
|
def get_public_key(self, chain: int, index: int) -> PubKey:
|
||||||
return self.address_managers[chain].get_public_key(index)
|
return self.address_managers[chain].get_public_key(index)
|
||||||
|
|
||||||
def get_balance(self, confirmations: int = 0, include_claims=False, **constraints):
|
def get_balance(self, confirmations=0, include_claims=False, read_only=False, **constraints):
|
||||||
if not include_claims:
|
if not include_claims:
|
||||||
constraints.update({'txo_type__in': (0, TXO_TYPES['purchase'])})
|
constraints.update({'txo_type__in': (TXO_TYPES['other'], TXO_TYPES['purchase'])})
|
||||||
if confirmations > 0:
|
if confirmations > 0:
|
||||||
height = self.ledger.headers.height - (confirmations-1)
|
height = self.ledger.headers.height - (confirmations-1)
|
||||||
constraints.update({'height__lte': height, 'height__gt': 0})
|
constraints.update({'height__lte': height, 'height__gt': 0})
|
||||||
return self.ledger.db.get_balance(accounts=[self], **constraints)
|
return self.ledger.db.get_balance(accounts=[self], read_only=read_only, **constraints)
|
||||||
|
|
||||||
async def get_max_gap(self):
|
async def get_max_gap(self):
|
||||||
change_gap = await self.change.get_max_gap()
|
change_gap = await self.change.get_max_gap()
|
||||||
@@ -467,6 +468,12 @@ class Account:
|
|||||||
'max_receiving_gap': receiving_gap,
|
'max_receiving_gap': receiving_gap,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def get_txos(self, **constraints):
|
||||||
|
return self.ledger.get_txos(wallet=self.wallet, accounts=[self], **constraints)
|
||||||
|
|
||||||
|
def get_txo_count(self, **constraints):
|
||||||
|
return self.ledger.get_txo_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||||
|
|
||||||
def get_utxos(self, **constraints):
|
def get_utxos(self, **constraints):
|
||||||
return self.ledger.get_utxos(wallet=self.wallet, accounts=[self], **constraints)
|
return self.ledger.get_utxos(wallet=self.wallet, accounts=[self], **constraints)
|
||||||
|
|
||||||
@@ -518,13 +525,18 @@ class Account:
|
|||||||
channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes)
|
channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes)
|
||||||
self.channel_keys[channel_pubkey_hash] = private_key.to_pem().decode()
|
self.channel_keys[channel_pubkey_hash] = private_key.to_pem().decode()
|
||||||
|
|
||||||
def get_channel_private_key(self, public_key_bytes):
|
async def get_channel_private_key(self, public_key_bytes):
|
||||||
channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes)
|
channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes)
|
||||||
private_key_pem = self.channel_keys.get(channel_pubkey_hash)
|
private_key_pem = self.channel_keys.get(channel_pubkey_hash)
|
||||||
if private_key_pem:
|
if private_key_pem:
|
||||||
return ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=sha256)
|
return await asyncio.get_event_loop().run_in_executor(
|
||||||
|
None, ecdsa.SigningKey.from_pem, private_key_pem, sha256
|
||||||
|
)
|
||||||
|
|
||||||
async def maybe_migrate_certificates(self):
|
async def maybe_migrate_certificates(self):
|
||||||
|
def to_der(private_key_pem):
|
||||||
|
return ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=sha256).get_verifying_key().to_der()
|
||||||
|
|
||||||
if not self.channel_keys:
|
if not self.channel_keys:
|
||||||
return
|
return
|
||||||
channel_keys = {}
|
channel_keys = {}
|
||||||
@@ -533,8 +545,7 @@ class Account:
|
|||||||
continue
|
continue
|
||||||
if "-----BEGIN EC PRIVATE KEY-----" not in private_key_pem:
|
if "-----BEGIN EC PRIVATE KEY-----" not in private_key_pem:
|
||||||
continue
|
continue
|
||||||
private_key = ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=sha256)
|
public_key_der = await asyncio.get_event_loop().run_in_executor(None, to_der, private_key_pem)
|
||||||
public_key_der = private_key.get_verifying_key().to_der()
|
|
||||||
channel_keys[self.ledger.public_key_to_address(public_key_der)] = private_key_pem
|
channel_keys[self.ledger.public_key_to_address(public_key_der)] = private_key_pem
|
||||||
if self.channel_keys != channel_keys:
|
if self.channel_keys != channel_keys:
|
||||||
self.channel_keys = channel_keys
|
self.channel_keys = channel_keys
|
||||||
@@ -555,41 +566,24 @@ class Account:
|
|||||||
if gap_changed:
|
if gap_changed:
|
||||||
self.wallet.save()
|
self.wallet.save()
|
||||||
|
|
||||||
async def get_detailed_balance(self, confirmations=0, reserved_subtotals=False):
|
async def get_detailed_balance(self, confirmations=0, read_only=False):
|
||||||
tips_balance, supports_balance, claims_balance = 0, 0, 0
|
constraints = {}
|
||||||
get_total_balance = partial(self.get_balance, confirmations=confirmations, include_claims=True)
|
if confirmations > 0:
|
||||||
total = await get_total_balance()
|
height = self.ledger.headers.height - (confirmations-1)
|
||||||
if reserved_subtotals:
|
constraints.update({'height__lte': height, 'height__gt': 0})
|
||||||
claims_balance = await get_total_balance(txo_type__in=CLAIM_TYPES)
|
return await self.ledger.db.get_detailed_balance(
|
||||||
for amount, spent, from_me, to_me, height in await self.get_support_summary():
|
accounts=[self], read_only=read_only, **constraints
|
||||||
if confirmations > 0 and not 0 < height <= self.ledger.headers.height - (confirmations - 1):
|
)
|
||||||
continue
|
|
||||||
if not spent and to_me:
|
|
||||||
if from_me:
|
|
||||||
supports_balance += amount
|
|
||||||
else:
|
|
||||||
tips_balance += amount
|
|
||||||
reserved = claims_balance + supports_balance + tips_balance
|
|
||||||
else:
|
|
||||||
reserved = await self.get_balance(
|
|
||||||
confirmations=confirmations, include_claims=True, txo_type__gt=0
|
|
||||||
)
|
|
||||||
return {
|
|
||||||
'total': total,
|
|
||||||
'available': total - reserved,
|
|
||||||
'reserved': reserved,
|
|
||||||
'reserved_subtotals': {
|
|
||||||
'claims': claims_balance,
|
|
||||||
'supports': supports_balance,
|
|
||||||
'tips': tips_balance
|
|
||||||
} if reserved_subtotals else None
|
|
||||||
}
|
|
||||||
|
|
||||||
def get_transaction_history(self, **constraints):
|
def get_transaction_history(self, read_only=False, **constraints):
|
||||||
return self.ledger.get_transaction_history(wallet=self.wallet, accounts=[self], **constraints)
|
return self.ledger.get_transaction_history(
|
||||||
|
read_only=read_only, wallet=self.wallet, accounts=[self], **constraints
|
||||||
|
)
|
||||||
|
|
||||||
def get_transaction_history_count(self, **constraints):
|
def get_transaction_history_count(self, read_only=False, **constraints):
|
||||||
return self.ledger.get_transaction_history_count(wallet=self.wallet, accounts=[self], **constraints)
|
return self.ledger.get_transaction_history_count(
|
||||||
|
read_only=read_only, wallet=self.wallet, accounts=[self], **constraints
|
||||||
|
)
|
||||||
|
|
||||||
def get_claims(self, **constraints):
|
def get_claims(self, **constraints):
|
||||||
return self.ledger.get_claims(wallet=self.wallet, accounts=[self], **constraints)
|
return self.ledger.get_claims(wallet=self.wallet, accounts=[self], **constraints)
|
||||||
@@ -622,7 +616,7 @@ class Account:
|
|||||||
return self.ledger.get_support_count(wallet=self.wallet, accounts=[self], **constraints)
|
return self.ledger.get_support_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||||
|
|
||||||
def get_support_summary(self):
|
def get_support_summary(self):
|
||||||
return self.ledger.db.get_supports_summary(account_id=self.id)
|
return self.ledger.db.get_supports_summary(wallet=self.wallet, accounts=[self])
|
||||||
|
|
||||||
async def release_all_outputs(self):
|
async def release_all_outputs(self):
|
||||||
await self.ledger.db.release_all_outputs(self)
|
await self.ledger.db.release_all_outputs(self)
|
||||||
@@ -46,9 +46,11 @@ class _KeyBase:
|
|||||||
if len(raw_serkey) != 33:
|
if len(raw_serkey) != 33:
|
||||||
raise ValueError('raw_serkey must have length 33')
|
raise ValueError('raw_serkey must have length 33')
|
||||||
|
|
||||||
return (ver_bytes + bytes((self.depth,))
|
return (
|
||||||
+ self.parent_fingerprint() + self.n.to_bytes(4, 'big')
|
ver_bytes + bytes((self.depth,))
|
||||||
+ self.chain_code + raw_serkey)
|
+ self.parent_fingerprint() + self.n.to_bytes(4, 'big')
|
||||||
|
+ self.chain_code + raw_serkey
|
||||||
|
)
|
||||||
|
|
||||||
def identifier(self):
|
def identifier(self):
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|||||||
@@ -0,0 +1,884 @@
|
|||||||
|
HASHES = {
|
||||||
|
0: 'bf3ff54138625c56737509f080e7e7f3c55972f0f80e684f8e25d2ad83bedbe2',
|
||||||
|
1000: '4ec1f9aebc8f7f75d5d05430d1512e38598188d56a8b51510c9e47656c4ffad9',
|
||||||
|
2000: '5d5965e43d187b6f8b35b4be51099d9ec6f7b0446dac778f30040b8371b621a2',
|
||||||
|
3000: 'd429f69a9dd890f7d9827232a348fe5120371ed402baf53c383e860081f6706e',
|
||||||
|
4000: 'd18fef650f23032e4e21299b8b71127f53ff6d2114c3eac4592a71f550dc5071',
|
||||||
|
5000: '12c1dd1b9cda29eb4448e619a4099c7bb6612159e1cd9765fbbabeee50f1daff',
|
||||||
|
6000: 'ebda54cab7979bc60f46cab8d36b6fcea31a44c38d6e59bdcb582054b6885cf8',
|
||||||
|
7000: 'd2be08791053336ae9bd322ee5b97e9920764858f173a9e0e8fb54ee5d8bad0e',
|
||||||
|
8000: '94583639d9e2ce5eac78c08ef662190830263ea13468453b0a8492c3aca562fe',
|
||||||
|
9000: 'a6ebb24a3ec9bb3fbdb9d9525729513dd705fe85093c33a07bbfe34ba1f1eae0',
|
||||||
|
10000: 'a5d32c6cd43097f3725e36dd55d08ec04cbe62e40d1fb3b798167cc9c1fa1ce4',
|
||||||
|
11000: 'b5cf8df354bc26e8b0315e8038564a9a7fc1faa6ad75e67819b7eafbcfdcb145',
|
||||||
|
12000: 'ae7b0e5106299e43b78f337fe15e85c27f88d5bcf618a21c03ce43c094dde909',
|
||||||
|
13000: 'b57dfe631571b654e11fb4295aecfc11f612664d22d64a9690f64c0b3ec128bf',
|
||||||
|
14000: '76e55cc1c38d9a6a349c9a42ae0b748d0d8b27ebb30539be8bb92e71e7abca17',
|
||||||
|
15000: '5d33cbc872acb824a89ef601aa3c918e7f4dbb8d5be509a400fa41b03f7fe103',
|
||||||
|
16000: '25760f6a6bfb58104d02e60efa62ba5ae141bbac1f65d944a667ad66fbef15a9',
|
||||||
|
17000: 'ad37d98bb272aee3a950679cdd4cc203b1c129bb3a03fe74223ff858badfcc50',
|
||||||
|
18000: '9eac34fc453b97b1c62228740efcae4ef910b0bea26c322e17e3474b81713b0d',
|
||||||
|
19000: '06783429c8f50cc6495da29d38c86c3873800cf1eceffedfffe2a73f116a39d3',
|
||||||
|
20000: '0237c620f7f68d9ae9852d00551c98a938352d2e3d9336c536f1cbf5246fade6',
|
||||||
|
21000: 'c4dbb4eea7195a2cf73de2d49b32447cec9c063051acbd300b26399c9242b0ed',
|
||||||
|
22000: 'd18fea80b659d9690830c2f37c7c9d53f79fb919debb27cc2131044df26d7ff5',
|
||||||
|
23000: '0371254f4698d3c46596c1fd861e2e1e75053beb69c3c4d8ee6702f0992f0026',
|
||||||
|
24000: 'dbc97c3986bb9bb6230321d1fc8b7ca13369033f57b08ac01bb775f7241861fe',
|
||||||
|
25000: 'cfb9baa125e91d02c060940b5b51cce52b495a0b0bd081cd869a7ca45a0e0585',
|
||||||
|
26000: '6e07fecd3d0f72b8cd50a3cdc1e9f624beb3dac9f6356308a1ec295f47d68f3b',
|
||||||
|
27000: '8ad248da3aac7c1db958d0ce8a2d50fde461bbb96a628efa215116d3b74b74e5',
|
||||||
|
28000: '4e8231ede23b397c49f1a97020ab5fe45465936cb6c8cf1c39ade1b38a80547f',
|
||||||
|
29000: '1453b7a4be4076e2c77b94bc35f639c571dc05b225a5f8018a9a60337fb3ef66',
|
||||||
|
30000: '43d779c83032d75e8df09aeb6ab103ad41618777565fabc2e56bf31c3a20285a',
|
||||||
|
31000: 'ea1426f57f75ee781a6e569772944119f5e65c8b110b31a32b61d951c3a1900f',
|
||||||
|
32000: '43d1a3941b2c3033f78040830d39e657f74958d71b3cd0bc728550469cdb5fde',
|
||||||
|
33000: '3b0c0eac231476efed2b9bb9dff206a6438a14946c3b4238bf0af6d03f9a350f',
|
||||||
|
34000: 'a8136e9d6f3f5048ef8f9f56628009f5a87bb28664ba8028ecae0797e5328309',
|
||||||
|
35000: 'd8136e638995aaea03c6b50f1cc0f1891bd107290f12a6ee57e3cae36b7e3374',
|
||||||
|
36000: '06a7692cf5ce9bb71461e5427e6f55bce2904f9d3af4c28e7343fecbf0e336da',
|
||||||
|
37000: '12a1a49c237497295150f99c76d79d8affe9fd720e0cbffa62be80e2bda7f832',
|
||||||
|
38000: 'fcf3e526931b7f7cead47652f12cdf76641032bc545e22aa0dc83bef46a085fa',
|
||||||
|
39000: 'ffc34a198ab1cc13819c0c90b26a8455627ff12868b49ac1e6b5e1a086ed011f',
|
||||||
|
40000: 'ac71a9895999531350cf77a701035b0b59720a39994c990354b9cb1f6ea8eb49',
|
||||||
|
41000: '4fa753f5f2de41ad69ab32c6802eeb352f070d8684c3bd154eeb8a3c17aa6363',
|
||||||
|
42000: '6f0febbddf9248e3df2254a7196ec13f353c7a8049e098f2fcdf920580abffa4',
|
||||||
|
43000: 'e3e98488cb203be9b531e3b5ce2200f4e5655297a4f4534bcf486ad5e26687bf',
|
||||||
|
44000: 'c7c6078e30163204e49d493d506f72f71d68d2a591874328342ca3cca81c87bd',
|
||||||
|
45000: '2a65629529f69c15a6e404ba54b83e024acba9d6c786be554fb14d28951ddf9b',
|
||||||
|
46000: 'cbc148ad661e6fc390746e94cf21df71edcb222b4e8d8f0771237a8e03412f9c',
|
||||||
|
47000: 'd6977f770749b7ce1a624a61f7b2d26d817a27c50a895466ee10f8bd758c029a',
|
||||||
|
48000: 'df05fddcd5ccfab7811e5d7b6cf7b49df71c7e0f77a57ce384936948114bb93c',
|
||||||
|
49000: '9d26a67c06f066229bcdafd55cbe71e2b2518465d6deb618c418b2f229ef513f',
|
||||||
|
50000: '4ceb896d901315e0f3e10f3b1ed8472f5bb5230d4ed6377214b07e425e515f6a',
|
||||||
|
51000: '218019bd8440a8c242e84940119d4eccccd26134a208cbb92cdea3480069f482',
|
||||||
|
52000: 'c15f1538280e71abc1cf3e117c79b0b671a9f6fb46102fea7b9a1fbcad418011',
|
||||||
|
53000: '0ad4d3812a42e23fc0defe7c62dad451266ee21699ade625a5f00b91993850d3',
|
||||||
|
54000: 'f994cff7b8166b0c39a310bb686f043fe9355678ae406a7fa1a74d4bf6f9b849',
|
||||||
|
55000: '6723d30c99fb45513a42a3ffaa50d1b8350e3c8b52c6dc6894303147eba95d55',
|
||||||
|
56000: '947cbea79aabe2051522d9e957d92b86dee2c420c32a83cf6f6d655136731926',
|
||||||
|
57000: '8a153fa31a971e7e304650257bfdbaac230b36a9beebc01a85a7aab9a357a5f2',
|
||||||
|
58000: 'd19a5234ab995cb851d691b2c2c84ae665890621c52794e54c16d6a87f516c7b',
|
||||||
|
59000: 'f9c7a353e62ce57a5e3f983362b08fdbff5e2aaadce325bba7680104a6f51cb7',
|
||||||
|
60000: '9e1d535bd31525803e93e4b4c2346fcdfee7c5cdc4b9fe3032df9295aa85c754',
|
||||||
|
61000: '640387271e7eb0816b9e290dd157f58d1eb54cfc39bc7ecc4486099af2a22b76',
|
||||||
|
62000: '3d36acc4169a238657729df674277c6cf236e21c2bed4e99b8f1a6e82d00bbd6',
|
||||||
|
63000: '6d4201e45595ea160773d63bb1481f78664156de1f4564d9a8c3f61b3cc1a6c9',
|
||||||
|
64000: 'b2cadf2ba39b2d5e688029cbbc68cc6d71337b29861dee24fe941b31be1198a3',
|
||||||
|
65000: 'd8a364d385ef42e53ce88c82355235ee7a27dd3e642849dc22d08a4041695dfb',
|
||||||
|
66000: '7025d4b2d9537002ce2b9ec42c596eb864e30f41c06f02413997b5629d6dce3b',
|
||||||
|
67000: '852a3fecfb3a77bcd6d2604170cbe9646d9b889c90d234eddc8f9d112777fe41',
|
||||||
|
68000: '38956213012f66a89377f4914db838faa6738732f7a7a865ebc5777f2f126b73',
|
||||||
|
69000: '4c89b07d23727d4bd1a080bcb9a22dcdb45c904f4c9866153a000056d7ec1546',
|
||||||
|
70000: 'e4e7fbf0d17b5f933deba12e045ea23eabfa6f7dbdc633e5c3d3ef8ddd066b47',
|
||||||
|
71000: '1f07c36e82f8e7582ec9df23ddc90715df45978b6aa0a2f17c08ee14cbb23000',
|
||||||
|
72000: 'c59c660b5aca2fa3364524ef43a4fa93fc3f16df777427cf1d810f65ce42e4d2',
|
||||||
|
73000: '83a05ac2fcae8c5d50a90df308f641616fa9b9fb553763c874b764a73def2f95',
|
||||||
|
74000: '448d209c032a2c6f5330481bf34fcf0bf1ea2c47f43fa5f82cf42e0d8f4d2b20',
|
||||||
|
75000: '5627eb3491e1154ad2c90094da85dc7dbfaf89d3765848db1ed915b7e26dde58',
|
||||||
|
76000: 'b1417b1b2360e24fe42345508ba0a333f4bd76f48c959b8bb40a99bf34ec5ba0',
|
||||||
|
77000: '250aca2bc4e6da7a8e1cb2e4deb0396aee7c317177b099d7dab9731b4c0f7573',
|
||||||
|
78000: 'dbfb1dbe80a5db06c9dcc645c5279eaf40559f5e04df0e64b696e80aba5659e0',
|
||||||
|
79000: 'a7183a545925164b5a9882810ae117d0f3df9b3ed55885fe74ebea284bec484f',
|
||||||
|
80000: '5b4a7bfd9b5394e2759daecfe472ea6e92b281a7821b106314b2b2e0facbaab8',
|
||||||
|
81000: 'b7075fe530b44e4f5895521b3cf99a79fe23d8bedde0027d6e7924ca93793eab',
|
||||||
|
82000: '6a1fa75b896d799ae84046cba7cd942dad47ec04bf50a85cd1b2bb15861995e1',
|
||||||
|
83000: 'aa5f4caf970433e90e2e45f9524ab9a6e22281505ada4c206ecf4672486240e3',
|
||||||
|
84000: '103732d12ef792a1adbe8f295ca6abd008867323ee996698b2650bdc1bb8d06b',
|
||||||
|
85000: 'a214c16ffdc505d6caba7bd0c2bae766bb19f7eb4d436cfc037e93783afae7e8',
|
||||||
|
86000: 'f54226ae9a6814a345968b5c2982adf638b995ff50e27de1890b3760ead12158',
|
||||||
|
87000: '26546212a0720cab988194432f4fa7c3e47caf0fb8e31efdd1ee93c3ad056868',
|
||||||
|
88000: '48f7d47bc443bdbc6a37ffe8f1d0d91de16d4a470d03655692de8a04f84c2561',
|
||||||
|
89000: '005bac455ff093a39b907af0441897e7a9ceb19eaf51d9dd9ec2b5ea43ff6d57',
|
||||||
|
90000: 'b482f10315170e6bd280325e180ce37b0423b685baa06d33586ee659649b71a7',
|
||||||
|
91000: '56db4d5d6c460fa76d492d9900c02e8fa99bfe97c7d9fa2a75bf115128ea0da7',
|
||||||
|
92000: '72ac917c4f531f91f65fbe33e78b4d4e2bef18aab3d2ff584d59fed1ba2cf398',
|
||||||
|
93000: '1100df2fdb03d44530723d131dd7d687053db6a317ac9181cfdd51925152df02',
|
||||||
|
94000: '6d50f66efb571136501340fc00b52105e0011f0f11fd68c9b185717bd42f9307',
|
||||||
|
95000: 'bd0c996d9ec40d5e91c2b5f278a5c9c9ea0b839012a7d135e1afc73a725bd8d6',
|
||||||
|
96000: '4ebfe8a3e1a1625632896d5d2970d8b080f08c5948f0ac38e81f3fc85db5af5e',
|
||||||
|
97000: 'cbc03464aa3513aa33b540d0f026b93db7607c4bc245af8fd88c3b476ea5e394',
|
||||||
|
98000: 'c9a185ffa55d01fc73c2e60004b0c8dee60ffbac7784cd16046effd7a0a51a86',
|
||||||
|
99000: '1eba97d5ee69229f4cd697c934d60cf3262ebcdf4c79526cda71e07b52fa22f7',
|
||||||
|
100000: '0b1650207a55c64e7a6cec62ddc3cc190ed3796507bbd48358a0b7d6186bc3ce',
|
||||||
|
101000: '1f47841b0034d3cf1046660c2246902f679445b5ef621df2762bbea33b7d685a',
|
||||||
|
102000: '42f63b1b622d08e120b5405ba65d25e1a9777312cca77248b7827d7084e6d482',
|
||||||
|
103000: 'ef4861fbd4e578ff80e0d5e2afc62fb113aa203d3933f74efd3e2400d73de922',
|
||||||
|
104000: 'e97a3cb78c09eef732757e81c6a7135beb33d95398054b45df738853acedaaad',
|
||||||
|
105000: 'd12e8feec15893ccab662a1ad0754b2ccbc18b078ff3894eb2214ee4ac2f7af9',
|
||||||
|
106000: '83ae84103b37c93b2d1535fc47b053194270b3c186b1b25d69cd8a1540caaa1a',
|
||||||
|
107000: 'a102c6dce88bc4695250c24a0ebd2913b2b85c211b7d5dbbd18ecd95bea63144',
|
||||||
|
108000: 'a0c20b36140d55288dc4dfeb6c16a6ece3d43efb57b1728bc872c35d6660c704',
|
||||||
|
109000: '412fe428b2929c3cf5c6991a2657e5414a569b30e77a3f4fecef14239d89caca',
|
||||||
|
110000: '8518706e957f3ef892d1d2d7f65c5f4588620be994cda9cb7d81fb600554a456',
|
||||||
|
111000: '66f4e25f40d24360299e5358f954d8a79b245399c46dbc4e8a3db0c10fa14e18',
|
||||||
|
112000: '81c0ded5c5e30f92eec84ca293abb0165068d917bdcf436754a0e9b0ef1e325b',
|
||||||
|
113000: 'ebd5e034a45517e59e96e986b55e38051a5d2a3120a7ed92d4f3e01e18e73972',
|
||||||
|
114000: '681b85720d71ab660c7443392883ed4bdcf9aceffe202cc5ead94664e315a744',
|
||||||
|
115000: '3fbda839115bc6e6d451a05f78b27b35b137c94374fdaa42235f3035980e85e5',
|
||||||
|
116000: '7bfb88d39fe0ee7ce046648676b226997a812af1f1ed79040750045536c76067',
|
||||||
|
117000: '5853c6d48fbf99b06d3e1f474e1b667ca598712f9feaff8cda48a95935cfc498',
|
||||||
|
118000: '6a23987731379799289d2a527dd40fde8aaca8398625d2a2e3e646372ffc99e1',
|
||||||
|
119000: 'af73cbe957865b60c3c3139dd1e2bf8bf1cc504dd167545463105996609750c4',
|
||||||
|
120000: 'b309a20170421fc4ddfedfd7611716e1cf5a802a1db09b66b9ae448fdf958792',
|
||||||
|
121000: '624cc893801e4da093ca973e2135352bb428dc278e47f5308412d05acbbdeacc',
|
||||||
|
122000: '2f20c67ac3485b6e5407762a23e8a7df1bb380f1607f538cb5e2e1ffaad578ef',
|
||||||
|
123000: '89c73ed65ab8f729cdcd678dc52e876969907f1560898220702a696893744e00',
|
||||||
|
124000: '17d8155d97fa4e8705aa55969709f9cbc97f87815d40f5a2b5cf4d7b85dfe4b3',
|
||||||
|
125000: 'd3cbbdb7f51c8a1f5794bb3d1f40c413b313f91bd3f3dec0d8ebb6a02107e52a',
|
||||||
|
126000: 'bfd02d760eb5de2e792ea74215ae37d13174c3fc0aff08587281f0f39e427caf',
|
||||||
|
127000: '937dcc09a14d1de79f5eb679996225995cdce69497cff5f12c39c098c42656a0',
|
||||||
|
128000: '81d372b07e2467bfa62bb4b974d0eb75ea313e547b9d4e62f3c617f939bcf67d',
|
||||||
|
129000: '9b4b315738ab8b5266b5accada140dcf487b2f509f1d758b9b972b93ecd03c9e',
|
||||||
|
130000: '8ab3b1862cacf7f1c4f6617f7280ba31547dadd62bec9c37ecc8d074c90be2b0',
|
||||||
|
131000: '868d6520e526b8594c44a05eba67ec1de8e01b547b34b62ebc5dd09d656e9072',
|
||||||
|
132000: 'ce09471822965da31db3188cd156538c44337da2952486a34dd2ab372ea490a8',
|
||||||
|
133000: 'b0c34cd700630c9281c8657dd24baf609f72f897442538d77f4ac355199731c0',
|
||||||
|
134000: 'a76e5dc895406a3951d55b557272915810420bdc3ac076863c7efdef3b115890',
|
||||||
|
135000: '2b70fd7021d40f6df15f3f72bc8e034a8a0bcd8542847b5b7a2088253351b3a4',
|
||||||
|
136000: '6161ee84f6fe100a02de0675bc9500dc18ccaec74e7da4b92135f0ccee6fb663',
|
||||||
|
137000: 'ae5770a7158089ca9adab8bdd07f1259e6f204ed377af27fb6a772fe1c031864',
|
||||||
|
138000: 'f3b8de21370968a90cedfcb1cb6803ec9a6b7740a94d8e4c60a80407fd55a12e',
|
||||||
|
139000: 'f08f6431d7367b71b25b321809ebf48f797bf7fcc943ba366420fd3f3dc00e5b',
|
||||||
|
140000: 'e76345dbb8c4c5ed1ea37249f892f64098557700259fb360401a559c45909041',
|
||||||
|
141000: '776b333f5b221f6b443d6011bc8d0753eed5ce2446a6d3ee99a5800c7159fa92',
|
||||||
|
142000: '9453693cb846f27ba2ac39194e5b70cedfaf435b73b5f7f288dd81343ff605ea',
|
||||||
|
143000: '08aea64cc4eb0170d2704cbd3a4ee871d4f3e53a8c1395288c029eaddd1aa097',
|
||||||
|
144000: '801d60c225301481c761114d5152780d5002b77aed18845c32c9e3804a5db254',
|
||||||
|
145000: '78cc80d54933b84248aee32a18144d03237f948de7889c97e2112d17ac998009',
|
||||||
|
146000: '1e954e8f2fe59da3feef0514f6b271f52961467782b2530501c294a83181f176',
|
||||||
|
147000: 'c0ac42fe9c0e1d2b0b7e6e796c553eafbd5d7bbce7df84903dabe16dc56ba863',
|
||||||
|
148000: '75b5441fc1785dcba69b9409f24224ba3207d624bb0e7d54af24118d08c788d7',
|
||||||
|
149000: '20e53048515cc34507cf2d1dd160eabafaa97793d81626e9fce38d540142dbc4',
|
||||||
|
150000: '5ff7a08461fc3fbacf8c6261dbd34a790e271bbc39908c23d5417f0f7a71b71c',
|
||||||
|
151000: '5c14cab7670821966d5aa61f09b32d93c865764b6e258764a3c33a82a3133fe7',
|
||||||
|
152000: 'e595968e040b54ed6572862b4089e005238399014d99bf4c361cc5922dbfa6c0',
|
||||||
|
153000: '1292bac3fab11de09f46a8327511b7c37366d0f2a96a027f4bf2f16147a39af3',
|
||||||
|
154000: '3db31267722b500dd9b26f5cfc4f06f26c5c343d181464499d3bb4aaf13c3c20',
|
||||||
|
155000: 'b92962af73b42335125f28d7d3fb97e3aa6ccf47c3f4229eb7fa04b67fdb933e',
|
||||||
|
156000: 'fdbcf12415284d8cc0adbf16996b9d37a943fd1a924c5dee5f47dc8add23f278',
|
||||||
|
157000: '199781ca8e5929708f834d6f38ce3e2fe196ab519f9915f14401462273431e19',
|
||||||
|
158000: '969017fd15cf6a73683de287c5e19fbd69ebb5aaf7c13339297f7177a716de3d',
|
||||||
|
159000: '77c7995cf97218655c195fdac9010c599859a46f760d84750189114d2d2d1d9d',
|
||||||
|
160000: '00ff84f0d845b4f1099d970cc0c3e2dd1c2584c611449f2318a3f27327246d51',
|
||||||
|
161000: '08481dc5f61e776ae8be12236d595549abfa0be28b187d80dad573594d94c11f',
|
||||||
|
162000: '71497ab26b05453f3c7057f8bf57fcc8ba30920a6032ab0ae3abdd29e0677582',
|
||||||
|
163000: '08713e0b750233a3843241d24573c4800f32c894000b7815860048ef3e7a06db',
|
||||||
|
164000: '01fd806f1879285ad5234f38789074b0ea3a0b2707bc6b49aa9bd51ecd26ddcb',
|
||||||
|
165000: 'a990225a2f02a77c1c61d518d46dde1e1cff3f4df0ed24eb463b97f84c7a2616',
|
||||||
|
166000: '7772a601a6e765ee340eb666cd645503c9988430ba4f558961246e9e69349b25',
|
||||||
|
167000: '389c7a8979078b57d54936497680c7908b9f989e46631358f1c31a3094621557',
|
||||||
|
168000: '2e1df97fe3cba7add5f05734a119464ff7c22bf92701ed85642da856ec653aff',
|
||||||
|
169000: 'fe0adfe455f65cf90e1632ef39bf9ca5856020d995cb71ce8a4998a40eed5998',
|
||||||
|
170000: '87487f255873e9f6411565553fa9ccb9518e7baf71ade67492536278f1ba0feb',
|
||||||
|
171000: '37cf239bd630b7c891019adcefead4baf19a86b40e21f3dfe4a76a78f2985103',
|
||||||
|
172000: 'e2f4087b868558af51dc8f40d77c300d76eacab17e5fb8bbf3b1f99c02d995a7',
|
||||||
|
173000: '273f32717f740438e93063d259dd4f0f160077c34c937a7d6a9b1f9fbb34d87d',
|
||||||
|
174000: '807a861813ca690a6fc0715f354f36bf491c34588b5ab778c0f4e692ec3fc152',
|
||||||
|
175000: 'dd74cf3d686c59820885bab134c81b7079f2edca86a25944fe0aa36a96941550',
|
||||||
|
176000: 'aba8163b91a905aab5ad6e7599b919fba650c446938dc7b2352759203bea8957',
|
||||||
|
177000: '260bd0d053038a54565f9b25c3894cfa875c9426b9d6e9915ec1df03bcf90ac5',
|
||||||
|
178000: '7a527f71743c4dff25f13b918ad4a6d91fb0bf9c873a7282e51cb4edf545edbb',
|
||||||
|
179000: '945397818b80995ac4e23c785b6d5ed2eb01338a2d4ca6f0cf40986e87659d86',
|
||||||
|
180000: 'ba1b5263f5418844796a19db06a53c095ec0428696d8ee953790d6c86de0795a',
|
||||||
|
181000: '2b673859af43554accd8599bca53c2ca94500ecd65df7672a1e586595bdec7ff',
|
||||||
|
182000: '08bae9f096e348c8c6b42ac2762298ec3faeba896524968478fff1d40017b5b0',
|
||||||
|
183000: '211577ae9a4f93fa5dc3e33764c31048ee306b5e5ffa5081fabdf3f30e49a977',
|
||||||
|
184000: 'a4301eead637043a8751a30c13935af971a731e03e90317c5a16c1677e50d837',
|
||||||
|
185000: '1f4aa362281293e8328b6c5b32a9948ef03dcea81c6ba13f94531f0c5e42627a',
|
||||||
|
186000: '0c6b15f2164a1b55bdf0eee2b0f5ab7cb9d2dfea6fcc82686483aa7a3659f6a4',
|
||||||
|
187000: '5bfa261cad0e4d271814de1f4752352f35da10f24b8016ac0065172e1296052a',
|
||||||
|
188000: '94c2490bdafe70d65951e4034e5b9122fa6b376a5ad8c9e6276289c4a5e059e0',
|
||||||
|
189000: 'a0119c5d57523b7ccaf1e5c6671eae9f8562bbf2477f7dd8c6847c6da41707f6',
|
||||||
|
190000: '4d82802c2464053c391374eb6124f8839faa9b343bae3f34e00f87fad9f45f9b',
|
||||||
|
191000: 'd341f5b4d20894be520eeb533d56c5737ad02a7ab0c24b2463516ff04e3b8b0d',
|
||||||
|
192000: 'ddf1a72ba512786d39f7c313f710bbd452040f1eb8a0c0ee7a4b9192199fcc3c',
|
||||||
|
193000: '514e1b40e5e15dc72e75cabb8d490c5e76b968e6dcdb3acf518b47e3a2020407',
|
||||||
|
194000: 'd0bac7ca0636b662d6be169bb5af807c891523a44fceda68f08c8934d6e9e254',
|
||||||
|
195000: '00c37e81f0dfe63a8fdefd0ba91442019899d50f72b43c4de8017e767777a5af',
|
||||||
|
196000: '581a9d093458c0157464c42fa2e2d9a1f61f7edf1ccc5dd77a4fdfdd9a4b37b8',
|
||||||
|
197000: 'd7555fbf7671d2475e38f62418f6130ddfe85b8fd07fe8a9815849316165d401',
|
||||||
|
198000: 'ea388884ad5abb06ccb1d0d2202dfef5c4ce4de0f3040df089017d01f14b9530',
|
||||||
|
199000: '6a80cbb22620b7a5f99f4ea36380b7bcd22e87feaf4186bfba0b9aabcbcaab70',
|
||||||
|
200000: '7a7231293d242887c6dfa517a78b60d40b41456b6f2fe833c0217074664a61d7',
|
||||||
|
201000: '0d6bde8cdd0370964b83ee766891b3cda7b9eec098dfaa5ef05842b29bbd043c',
|
||||||
|
202000: '223fc091bddfbaedec92099f873fc1a6bc50556f3421f472a63977b3351b95c1',
|
||||||
|
203000: 'f87fb207fd397f8fb5c5c21f84f272cf33d840e20ccaf9de810af784d8459142',
|
||||||
|
204000: '6273b130d819d84492bc50fa443ca401df748e58fa84a3708a1797f116f60f1f',
|
||||||
|
205000: '498edfaaa9d5f0501aea5482ffdd9208b86af46fcea5a1cbc82d51d45eb6a256',
|
||||||
|
206000: 'c069138610493a70742918b5c624cbf9275bf55ee24649e32244238433883ea0',
|
||||||
|
207000: '5110744882ba11b98df95f11e100bc398a5aab4c8258133b518b5abb58c7fe64',
|
||||||
|
208000: 'e5690d4e0561ef015d6e2997df3b8b95163cd884d999d8c9ac212bd7e1f06ab0',
|
||||||
|
209000: '17a0a3bbf2fecbfa5c7667524cb2a452f49b8dd089af132f9d2c530ca0b677b1',
|
||||||
|
210000: '3a9489f3aca89576152949a67df6462d25bf3a78976451b5cf8bd8385a10e6ee',
|
||||||
|
211000: '45196e756c7c282c4a9e7133adb07f103d0f74ee90919cc8ad89ca3e5d38c6db',
|
||||||
|
212000: '590f21c83694c35028b9c9d632a082307b8bb27ec87e2c0c4e2881a7be9c6637',
|
||||||
|
213000: '46e56c45305c51e4d22927e449a1855064504cfb58cfceb0f293d8a4f1dca7cb',
|
||||||
|
214000: '50b3f1bb82c1b213ed2cfd2c1144b8e67e5fa731dafabfefe063f8658c5536d7',
|
||||||
|
215000: 'f53709db4ca6c12b80c64501eeb8f176211116b0fbc3a650082cf830792b206a',
|
||||||
|
216000: '6818db3f6582e71021a84fffe751fcb47db2f734d59547be47a373575b5c7d8e',
|
||||||
|
217000: '01f6f8a0699ac533ee7921f63334af07b04369d5e580c1c609c6fee64c8c6b8d',
|
||||||
|
218000: '81e9ff93cdf5686fc42663a056c74fb0f560c29a5d23f21ef92364735a865392',
|
||||||
|
219000: '80b32bb6bc3c54a19d3b604f56084b3bda13afd4af4130f791505e44fdad6c98',
|
||||||
|
220000: 'c427f24ab14449b1430c5302385f3a77f90f19715d940e8dfb7c054828270163',
|
||||||
|
221000: '4303a0e1c6f8489589290370a6843d97eb02a01be8c227d905cd0e30034feae6',
|
||||||
|
222000: 'ccfdc1768afc18cccdf2b5d90d40158d86b110d878955b6570ad9e6e40063008',
|
||||||
|
223000: '6baefb7061239c02472c4783de6d32936f7fa1da9fa6d2448b4d92f40b89e79d',
|
||||||
|
224000: 'bdc334f5d1a3bd1d19cf589e3b1ab0e8a324d44ea1c70297ff2de6a66c99399f',
|
||||||
|
225000: '82d29612f96c55dbe83655db68afe39146aff1418af929f05033fcbe612ebcb0',
|
||||||
|
226000: '179b4144f4d644f546f001273c1e5382a8de50f84ec06a2c83b02808a9167d28',
|
||||||
|
227000: 'a350e96895efe979223475151305eb8d1739121b566964033b9967031ee5581c',
|
||||||
|
228000: 'd208c66a9ccb1afde8d1540d944bfb43522015f9d2df50851e66ce1b84b06c0d',
|
||||||
|
229000: '95cfabe5d18dd8d8e4a57b7bf7052ab09b7138ba0b3c9d2efdb32352bfcca149',
|
||||||
|
230000: '3e399d08c33b8fddeb687deb276afd3ce101d19f5a71b232532951f05d46d8f6',
|
||||||
|
231000: '91c0c17d5df1c553bd7dd6db8feb58a4436d598ef4acb1ed9a06da65f3cca83f',
|
||||||
|
232000: 'eb3928a522dddde176cd4c68fa95f1e953d74f3759af4b030f7e2be4c466c45f',
|
||||||
|
233000: '3ac946df5a319d31716f50c752a5769471c0f3ba4b1002613996981c8f32fa3b',
|
||||||
|
234000: '4c9cdbce1f2a4324368130c154e2d24e885877c13a126a847059ccc289fc922c',
|
||||||
|
235000: '8701d98b254fd0b588f022ea140e936d496ca8ccc0890ce133523432f4c09e6a',
|
||||||
|
236000: '0d2d45adcdf950541c25ff5b2b87dff5faad479094b1f68804a62c3151e6d598',
|
||||||
|
237000: 'b8924ead720d0e7a304c45c209ef3fdb90cde437d138bfed2f1e8357fb3d951d',
|
||||||
|
238000: '4c9c62a3b3f6bbfb08edc23d63f9b35ed94bb38393fd1761f9b9b810bb72f68d',
|
||||||
|
239000: 'd01494b4999c2c4def2564eeb1207fd1f3d35a62fba2e61b386bfab498cb9b6f',
|
||||||
|
240000: '3021d8f0edbc0cd2ab87bad5feffac54924b51eac9f6fae8ebe289f474f11a8c',
|
||||||
|
241000: 'd2ecf0170093d3a2dfc876f673e565e3e4aa76d23896af51c96c004cb14ce67a',
|
||||||
|
242000: '2b5ef67326178702a1a35fe4a848d22b791114b59121e41a25ddc1bfac82c6bd',
|
||||||
|
243000: 'b2d546f127252c3fa5f7ca45c61268dbaef66027484a1bcdadd281fabf97d34f',
|
||||||
|
244000: '58f9b398b871dd62952899d31661c118ca4f0d1c71dd165523b889d62b154393',
|
||||||
|
245000: '504760f0e46e2de0a2c8b4dd3e93c19d0f3e0e1b36916f02cbff166655816206',
|
||||||
|
246000: '15ffb23b844e6ac648a7fd099be8e631cb51b3a2496c67ac2dbedc4f6818d7a7',
|
||||||
|
247000: 'e3e56e8575552202cf35fc883bfcfd92cacf8013a6536729ec19e53bdb0db128',
|
||||||
|
248000: '22f75d137c7fe42a80ee04a386fdea8cad56e7ce8601234b6ec676498edf9e34',
|
||||||
|
249000: 'f84709269c393cae8106ddb1fa3a48cf3226e97bcc108717e483493159921aa1',
|
||||||
|
250000: 'f3aee545330f18ec49f6f880d7f9fd31ed6633279881dd13177b0a687c51b000',
|
||||||
|
251000: 'c2e6a4cb3c6e88f523db2ac24fc7fff3b8a27d13ea7b5d614806ed96247025e4',
|
||||||
|
252000: 'efba2c1f91a13ee8d7a8658e70893ccd3014a7019c225e356c3ee67955b58756',
|
||||||
|
253000: '9097ecbbc1279bfa2c92822a6485714ca7ba35cd3dad9ccaff5a8d24645eae92',
|
||||||
|
254000: '00a41b720ec0d7a9fea3229b16ad8d065cd6bb1d14587eb3404de33e45db9454',
|
||||||
|
255000: '7523636406176d391928c638e15e959a9d0255aa71dc9f3bea8995cae02d1d51',
|
||||||
|
256000: '8d3f7c5ded24e2488e25d6030d78d351354734b8b058caf6143430e96ebd5d0c',
|
||||||
|
257000: '48fbbc3737a718163a501e52964eb58aaf4163aab7ceaa4f75f928b25c376ec2',
|
||||||
|
258000: 'e1052e7ebf83246770a740c0c6ce8f011fc7007448777c533d67fec810fe5e64',
|
||||||
|
259000: '043f7d98b500fafc248549c1f5a9727fb0cec4e3e5f13071eeba459fb9179d13',
|
||||||
|
260000: 'ed2f3960b077651ac554b277dd271e12e6cd05942fb54b9464f18432adbe5ca2',
|
||||||
|
261000: '234f17bf2054ad4d3efeae165c3359a14e08d2e53506c839cbed6ba3d2a48ab5',
|
||||||
|
262000: '4342d662c4dab581bf1c123e057e4413bbf74b7a2a331e80f7099201539be54c',
|
||||||
|
263000: 'a7b1380371fbfe64bfb94bed5ae256f7db59a4661e461221cae99dcd54c72a8a',
|
||||||
|
264000: '4f2a606cdc5f346e0855d962cbef7c2e4993b2b07b86ce658a796b1baf6179d8',
|
||||||
|
265000: 'cc09cde8c101eb17b5457bf976525940904cb839e70ef3335780eb0261976826',
|
||||||
|
266000: '3c5f24533723afb124b2bc8c2f8d373eb5640dc4ea4dc76b841e81706dd6b29d',
|
||||||
|
267000: '467c5ca63621be011a07129d0c3ea785e2a0853cc3c0d1e9cace122831d42507',
|
||||||
|
268000: '07df13a794f6c0fc77142fbd0afa23121ee2a2a23bc9abca35c164edb5f6ad1c',
|
||||||
|
269000: 'e2be277e83a997a111fb45948e9a7afdd7925feb1708a84c62968cf6a42da1c1',
|
||||||
|
270000: '463425dddeeac2f27ee717364073229c333a7ba357888623c3b15a1db52542f4',
|
||||||
|
271000: 'd62e0bd8584fae7e4893c18252ae8d1c245e2e4c8e2f2d73703c681c6259f19f',
|
||||||
|
272000: '85cabf5254b6b6292c105e68d44f78d6effeec582467caf9d30a5deeb43b6a6a',
|
||||||
|
273000: 'f5509cee847a757c84adea8e40f3c7c0cc6a4da33a8cd3019cf9ca850d0bc93b',
|
||||||
|
274000: '2cfdb3477de2fe1c81c7133094094c79e933cfe6034c158de07ffd0c4c08b564',
|
||||||
|
275000: '1c1b8e447e95ec3c306fc68528631682bd85e1c876d86223fea28b7ff383c16a',
|
||||||
|
276000: 'b37e5e4f6d29590c9a58773a4d4384cf7c51d04fede4a602ba8095e14bf65993',
|
||||||
|
277000: 'b92d6ffdecdca014c946c2fe8c6c404ccf60d8401e7631918da8885bd360daca',
|
||||||
|
278000: 'c1f82e49c8f66aa580a216c8e08262080ff9fc4f4df1c482ce8b2695b90b7240',
|
||||||
|
279000: '41c48f5965e8939d1df69127798b46af17646ea75e545a7b47049596d9bfb3a2',
|
||||||
|
280000: '48da55fb8ce259432417ffacba5e064df4df2a8293cdd484ee0ad8ce4b0bc38f',
|
||||||
|
281000: '0e892fd450a8ca018567e1a6e23bcdc5b4b872686f816a415991f59b3abd76f3',
|
||||||
|
282000: 'b90fb78edf920d0bfb4bce1f39327c6777b511fd05d715729d7c4cdbf449d59e',
|
||||||
|
283000: '80965fd3817b9509ddbe6a919aa73adb11af64cda5d8352ad2bd9b7824e95c42',
|
||||||
|
284000: '43447043aea93b162d6f615e8cb7d992d02c176a41b0518256f7e0d8fb3162b1',
|
||||||
|
285000: '33bc92c77ba88251822698a7bd02860ef9c12420e4c27f4020370acaab0a2f37',
|
||||||
|
286000: 'eb664123736754c19a47e2afbbc8645c598d7b70c3d42132c334cfeea8fd6bf3',
|
||||||
|
287000: '6af8f2fb3ea1f0fd0d6432599c3a681be7d384cdc553903bcaac8de2d472725f',
|
||||||
|
288000: 'b5fcfef1b5f6373bfc9322164160fe98fc7fb4f3117bf91c8fe61324db62723a',
|
||||||
|
289000: 'f6f1c6f8a2a2905b2ca18845b8bc771bad6e9cc32fd2c8e20a565cbd7781bac9',
|
||||||
|
290000: 'cded625fb7b923d7471ee28276b6983bb4fb4f6281ded9a8637b6974710feab5',
|
||||||
|
291000: '49af69a4855a36d00cfe2408f7d090fb5001913aa06615eca5af78ecb92b1fb9',
|
||||||
|
292000: '2dfe91f7f1870539e6b51ce58f9cd6415ea8fd50dad376d74941a9ff1bf0f8f4',
|
||||||
|
293000: 'b84f14b784cf6eca78d13f4b0db4714be5bcfa59b73974b7106a0206a8993767',
|
||||||
|
294000: '07e95fc3c141dc64ae192cd233f97ac16b1adc9dfbbd685475bf693930b6f604',
|
||||||
|
295000: 'c39cca37abe0b8839f8e797287385c73031496c632a064a89b9dc0730a3fffd2',
|
||||||
|
296000: 'e69afa519ac761328358f5b3ac8b4395d5aa4b955bf1b4c5f3bc0af49cbc3756',
|
||||||
|
297000: '20c93e2d1d9bce1c861eb16febe50e534acc3e2d142cd2d0767bfe076545d1f8',
|
||||||
|
298000: '75970b654ac884cbd1328b68b59963bfa5babacca4c7d48368a1d938feb8af6c',
|
||||||
|
299000: '05448c7f1230870e814858ac9a7858fc72a045dd9e0f1f14b8862d2fdd524b07',
|
||||||
|
300000: 'e254223f85c3ba65a0295a5538387c08ae94e6070f5c989d27349978fef077c7',
|
||||||
|
301000: '4a8bd2357b4ce05422d51e052018fb9f75ce3c07c53b5d1623ac19afff388e63',
|
||||||
|
302000: '18f4a960b810034f5ea02458bbcb9bf3d38d681555be4e40d132f867444306c8',
|
||||||
|
303000: '667cd927ba1e7ab162c17a711bb0ba65f838e543f46b093c86c14e39988da156',
|
||||||
|
304000: '55280e92a3c3b9ab9e1032490aff10c1a2fe9e96cf4f88e49b7a756c0491701e',
|
||||||
|
305000: '8af5b503f1819b23da7b3663314a15b10e39e072377d1b08518a373c85515c1a',
|
||||||
|
306000: '439796b2b4add7dd0fbe14bb3d04f3917e92de2afa07da2a384b389ad9228888',
|
||||||
|
307000: '07fd4cbacddb504e23880dc454fb70659386afa21dab3e862cbe7688218a43fb',
|
||||||
|
308000: '2734a4205be5f2c8ce5850bee8d4ad0829c208911bf4ce833349a43af2db8fe1',
|
||||||
|
309000: '261b84349406a913a4cb700fd218112920910c1c57922e289114c7b903fe2d9f',
|
||||||
|
310000: 'bc652e239edd3476df7bd1e0563595145fd4f2b37dc2ac3bc28ec089a7550812',
|
||||||
|
311000: 'fd7c1b32015a2418efc6e1a73470ea3205de05662c767b0d4198fcaf804c3b5a',
|
||||||
|
312000: '4f017d27b81da7eaf62760cb7fe6613a6ec28fdc18859b6e727b216240a86315',
|
||||||
|
313000: '2d09525e2364e7cd7fe45191fee11f1ce16a2fbd16b56e41f90a3c64ce2f944e',
|
||||||
|
314000: 'c05770490a47280c5b318709c264b2ab951a5bf8c4575cbf1b1e3dbf35323f45',
|
||||||
|
315000: 'ac5da2f5abcee0f92f29479c6355ddafd1f3b4b3cca6659117c17c9fc10fc743',
|
||||||
|
316000: '4496d1a0e4c22a2521595bd4fae60abdb9404f79135311606a9e6713be373271',
|
||||||
|
317000: '40ba7c87490677af9e8b033a19ee5ffe51ba192f5912f491738bb6485cfb76d4',
|
||||||
|
318000: 'f42045295ce30bd4ba9fd9c845bc1228949b4152517185d3dd0598d8bae38481',
|
||||||
|
319000: 'f488695e46b53e4734f4eb9fced93f067f037af7099d0f9510856a898a0c438d',
|
||||||
|
320000: '83e42370911c70bfa2ad89a82ecd02bbc9cb13d24c2a900a0030771898ed618b',
|
||||||
|
321000: '74eabfbb64eb89488b59393ed529b4ee97e0f88a7fc14f798f22715bcee3fd19',
|
||||||
|
322000: '54f42438cf96fe8efa0180a47350002b34ccfe82123a85a9a6dc6f8b0894f0a2',
|
||||||
|
323000: '5509357b42476732435a1d5a6a850e27f62cb3ca81219a94f08dded06dfcbbf8',
|
||||||
|
324000: '1cd477361dc4c72b4f157469b6f3616e060f5d058b44ba864225210432b35540',
|
||||||
|
325000: '060ae9c541eee90a80a11fe300583e47c0ad4d12093d0651310cc6c87ecf2b75',
|
||||||
|
326000: '13ce59868c947ecf2d6aa986501c9f8411a59b4e0b8b6113f4f45dff4074fe59',
|
||||||
|
327000: '16a1532ff983f5cdab6c2d465d58f0369665a926c9d877212d95f9387e8732ca',
|
||||||
|
328000: 'c9608a16fc43e7e94f04b3309c067b0a4f68a66705009dabb39898f41fdc862e',
|
||||||
|
329000: '029b02acbe0fc8610894f0b89d6bd274362195a443fb9e5e4d4485be7786d461',
|
||||||
|
330000: '01181b9529da87eb39510619285c08b05cfa2ca2c7a2fb3ecadba474c1db3aa5',
|
||||||
|
331000: '2371752c35da119184e53dcb593a193fb1c81d6edcff921599328a6142a57e3d',
|
||||||
|
332000: 'fc025a8f319725faa9903eca86935116a2aad8fec6193055d5d5f0f431ba84b2',
|
||||||
|
333000: 'b602651afd9988075887a3f48455d977357032185b7d482262efff9cc97a45a6',
|
||||||
|
334000: 'a9a2bfb6b0f8db298b5ba732f4a23242fa89e1c5038916ccfba3751f7dd1b4a0',
|
||||||
|
335000: 'eeb3e35b6ea94119bf058e6b02a2ec2a706089bc1c360ca28d1b4e2ce1b547ac',
|
||||||
|
336000: 'e834f7ac6bd8e5b46ecebda4997ae73bccf353fa024fbaba7ad4d1d70d08d14b',
|
||||||
|
337000: '229a7017b6d0ffa08374e297b07f12620bcdbaaf5977dd39f3f207ecb03c7adc',
|
||||||
|
338000: '83d86500e8b2ef46a0f1505e29f7401b123ea5ed5feacb49864ab1de284eb574',
|
||||||
|
339000: 'b08e9dcef3e38d631414b4d3a8d8b3dfa8a07d9e5829c515e7db2fec64c60827',
|
||||||
|
340000: 'aa36ce6494df6905f4516530ddca77d88341c0202bf052347a86e090c03ab0ad',
|
||||||
|
341000: '527b50125d5479117ca8bfe84fd5507b2312360908d4cafe8588d40c4ef5622d',
|
||||||
|
342000: 'd1397e8b0f847b6d46c409659e9e10f3182ad9bdbca563cc2598af37cdc080bf',
|
||||||
|
343000: '896576aef0e0dbdb2df8291f2832716029f69718ac79f355425d475a1b2b7b4a',
|
||||||
|
344000: 'cf4b3c75279b0f7d71cbba934228709ff03e3254fdd83a47fde66e83ff393eb3',
|
||||||
|
345000: 'e68c760edf370e758e815d9363ada456fa2568b06abf8626f28a32cc62944770',
|
||||||
|
346000: 'ab19fe52e780e45a716ed97e93084bb3f7916e0a4479f131c7f5f7a2e7d69d9c',
|
||||||
|
347000: 'bc77d338a10fd71daa6c68fa9df2fdccf44b6d9b1d7c6456bb281bf8547b622b',
|
||||||
|
348000: 'f84469ebb26540cb3de0ae03f1f47452f9a9bc6b7195c3b5260d48623507ed86',
|
||||||
|
349000: '5bafc3e6dbe4bf88257c4381cb4f2fea0a4cde1f272e1f1006600b507cb0f670',
|
||||||
|
350000: 'e53ff96761b9c98bed635457acd1856a2ffe65b84e339cea94ad30f13a31e5e9',
|
||||||
|
351000: '5e0eb50d32e90e5ca73639313e5d3a850a113fbe60ac494ca6cdb799e9604d7e',
|
||||||
|
352000: '1a1a04aaef110493bf8fbdb8d1784a7706c6e303773ad0bad6511dd3c1da6c54',
|
||||||
|
353000: 'd72f3b615e244584f131f2f8e7154534aa9d2d9c4a710bb4de44de556b5d1232',
|
||||||
|
354000: 'e81b823c64229c555a50e90a4c3c4654b6673cbff4c62c6e897d09aac00d95f8',
|
||||||
|
355000: '11eed14cac2024588b389ddcd54315923ffff03d6c4c5f32657ba551fd308516',
|
||||||
|
356000: '465f28c63e564296f75a4a22c3ad16714cda29d7a2797e51e512eb1ed5f6a49e',
|
||||||
|
357000: '2b544c79c6f74ae1c05c268669e57ad41d6807f1d0ce1a5024b7b08646df6861',
|
||||||
|
358000: '8a76eccf113819aad8d7c17a459a614274bfdaeacfb4659d945f13dbf3907ccc',
|
||||||
|
359000: 'e63743f458f1479593d73952bfb882c72e5889331add576a716761eff2cf13dd',
|
||||||
|
360000: '0ab0bef439debf08dc3e1df9010c4e119d6862de6620de5f725cac0013c06301',
|
||||||
|
361000: '50f649f0c8bc0d77ffddd53fd5b896705a8858d3684a3ec876cb320ad26d2c29',
|
||||||
|
362000: 'e058c43c366c9c307ca3ce5a732f0ab2b4e09ff291a067366af25a82fb3cddf1',
|
||||||
|
363000: 'f99d7f0ec93ec7fd74273bb6f8bc4ad8ec0fcda9f6406001668dcf62a088a867',
|
||||||
|
364000: 'e5a69ea8763650542b1f90d6217af00da137a9bb72cd0dbd5379570d1fa696f0',
|
||||||
|
365000: '5d0eb3063474c6bbf9d9581898fcf8c4cd097fe5c6e3875d7cd701a5a0302507',
|
||||||
|
366000: '14da2978f4d4eeff5eb0a6bb663e514095acfe2210039df8dda9b76f522f74ba',
|
||||||
|
367000: '4ad3ebe23fe9b0f407c0765adb69ee973de5b84db29c97167c7a358fe8696bfa',
|
||||||
|
368000: '22db57ccf8c8193ae9fc08abe11f1ea8e869bfd6ab2c0b918820acdaafc19136',
|
||||||
|
369000: '530052d61878d20282a69e0bace41857f7edea6d3e9e92fd0a7781c8b60d208e',
|
||||||
|
370000: 'cd057d0c889c78e06d0eb71287c46deea4340b8c7a4c470dd69020e1a9029c43',
|
||||||
|
371000: '41f32ab47863f0c44b8755ecf9b201aaa92999051ccbf8a5acdee0e12dee4110',
|
||||||
|
372000: '37de29bd97d75a41dc3eb52bed9c09baf069c23120a393888e15e61ac7f81ea2',
|
||||||
|
373000: 'd6b26bb791fdf6d79f59d5c37ef2af0a35149737029157724224d614477527f4',
|
||||||
|
374000: 'de18c6a921838385950b10ed9429579e95a0bdc574f448f457ee486a59a04260',
|
||||||
|
375000: '1f86edf4ae47f409f8de6a16c17c2da063937478a0ccc800633e4d33b531709f',
|
||||||
|
376000: 'a0511050ef086052fe5671d3333c255507b03edbd365cd7ce7cecddb4000f391',
|
||||||
|
377000: '7b8f0060f51fd170296b9e000bb2970097f77ebf0b5872a9ac01be5aa03d7ffa',
|
||||||
|
378000: 'fc5669d194d822c55e141ddc9aa229059e4663169456dc927d43b44a2c8d3c0d',
|
||||||
|
379000: '1c0bf45f052e0123d1a6806fc4a93ac8261ed26772f324c1b3e7c3fa44d7f442',
|
||||||
|
380000: '73b69dcfa0a916a4c495a14d4d43e270b123ba37d267e3eb0d65890336fd3e32',
|
||||||
|
381000: 'e89e3d548193df667c637f774c76f508ec79b7bf1f2ac5d9adf7468006e34454',
|
||||||
|
382000: '6b835973c2efcd564fbd58d96226cb984d5c32dbf63d5b6044a0a11f77c2ccc7',
|
||||||
|
383000: '50b7011656c5903941303c7a19e4ea0fc6bc10456ba85638b0919658556e47cb',
|
||||||
|
384000: '74c509b238c1418ae527d370e8fe922f7639c88ef8d78fc3a45894deeb2252a2',
|
||||||
|
385000: '6f9f6147b06d8fea8c440cdebb3869b7bface7cb2355d66639060056adc9bcc2',
|
||||||
|
386000: '5b59e68fd62751b8be184a5621f1c63f1e9cbd22051487999816df392dca7837',
|
||||||
|
387000: '86d8b9628b83ac63637c86c364001067ee1c3050f395758fd3f0bcab20cef703',
|
||||||
|
388000: '5e570f7b69738285848bd8b7f2fbb82d05e99a72cd4a0faf20a9e3491d2d041c',
|
||||||
|
389000: '78d31bda70842388c1cfb2b83e9e95ded37a0304aaed3292e2687a96c07be9b4',
|
||||||
|
390000: '316da5e2ce79bbc4f4500ea6417271f0ae268f85f86f81223d0af38d1e0e0a67',
|
||||||
|
391000: '45cebf8a25c25aaaac06e915fb3fcc99a84be61a266c2bd8997bd1fe00a7321c',
|
||||||
|
392000: 'c5ff6e24f524bfb1f629245345052c0a0df33ab881ee9d11dcaf64915b2acad3',
|
||||||
|
393000: 'a54b5d4824bae06f91566b73c99a07b812095d7c133d1042ded80211c0402b3f',
|
||||||
|
394000: '0ae76082f8347ddf838ce0ad486f5decf6223e8b8cfc16dfe178b15830807c60',
|
||||||
|
395000: 'dc436c97ee6ed427687e6d48e609d271b125495afdebbcf611c0fd5102ec900e',
|
||||||
|
396000: 'a87871c13ca3119f309a4ed98e4224d4eff01387ea67e86148007ac0d759f2b9',
|
||||||
|
397000: '58c143b302132ad4b38718ebe1c5fa81c1177a30045f1e4a52dfa749e84dde1d',
|
||||||
|
398000: '33089400401a190c9375abb63c6ef489123f7369e5a35766b59e69edaf4e3b93',
|
||||||
|
399000: '656305fae087b86758bc7a01e13051b1aaf4a5c53d9b71e56614217a82b20eea',
|
||||||
|
400000: '7ea274642eac11d02aef589428b181e1c8aed913e248b357917604b00b23c752',
|
||||||
|
401000: '1969a9f64e316e0b4fc864fd7162379821510a944652b8dea747e425c6222807',
|
||||||
|
402000: '3700e1703e7803a4577de7e364a34c3be94d3f933e05c5dfbf6ff68349b6fa31',
|
||||||
|
403000: '8b622599bf4b2d82bc11c897f35536036e6798ac101a6256e14477624147efe2',
|
||||||
|
404000: '5f6f6dcea393ddccdec69b30881ab08e01e1a7e6a50d681883c759b742c12d6e',
|
||||||
|
405000: 'c3fb0507a57da69042e246dc8ecac101f33cf2b1fb53fe2f507ce28a0d82e4b5',
|
||||||
|
406000: '4f034663bb7b84439fa256f7b4ee2a5cf93e0048c1eb476aa2152453ee7be544',
|
||||||
|
407000: 'ff99ff38df6f928242571fed5b03fe0c3cb481b510d3c943e47e26f26440cbda',
|
||||||
|
408000: '89865af5e99e64cef61dd931061bbca7fe355218668f487f06474e46ebf892d8',
|
||||||
|
409000: '1b1d050c0b98d55d9ecb0d1a5b03e75caa352b4e0ce6a57136551000e150c480',
|
||||||
|
410000: 'fe52790595db35c7685e5261de78d139c830af0a5ae73a7e33da21be9ffcc9e6',
|
||||||
|
411000: '1afe55e66e6e16abff282c9d6bcfbc6a3a6e40db3c60323cd44e3b3a5d4ba924',
|
||||||
|
412000: 'b59a4057d73e8037f3e0e9b92310def7900c19917892ebe7689d5f8d77912388',
|
||||||
|
413000: '603f3ca16a6a795d5aa807eac5d6ee7f80a1e2596cd91cf41cf90ee309b2871a',
|
||||||
|
414000: '10e255807a43bf5b95f9af648f1724de1e953a5540819822be5de184ead24ae0',
|
||||||
|
415000: '9faf6b1c33006e3da3ae2c7599c7fdeb5078a8427edd6666290e32aea67f5a23',
|
||||||
|
416000: '973b77e4d491ce0c163ca64f5419b05822eebc2f5d85810f51370f56e0d812b6',
|
||||||
|
417000: 'bf99710d77fe84ae3ef5a371887cacc2ca1fbc5549b04ef9f61cb9d911b06a9f',
|
||||||
|
418000: 'f032acfc2d629fecfa180067b4531f9d72924c5f204cfe9053c6556a334373e8',
|
||||||
|
419000: 'bc9f085612af6a21411fcf51259387d97e6ae1169e4c6f92766a168302a98bac',
|
||||||
|
420000: '9136bd1fa7e8850c52e3cda39a96f754cdc605a69ea49b7d6d959d0cce81efba',
|
||||||
|
421000: 'ec2a78bad1b4b7325d574122ad8c3072d27fe6854a6965e6917a28e0c868a229',
|
||||||
|
422000: '536f4cc5d6ab082d32ccdba1caccb5d4f3145b60ee8538b1e18c6231dcfa23df',
|
||||||
|
423000: 'ac7ad1f97da0e55661bb25d53c27ea72f705d54050f020aad9fd5856a27117cd',
|
||||||
|
424000: 'f840eed1c0fdfba43965bfc175e4d92a5ddd0fd6022dfef988d70a5e7f363116',
|
||||||
|
425000: 'c1a8f9b79a34b76c9ae58e005501dbb1893cfbda8bb25b0d54854657b3ff1a54',
|
||||||
|
426000: 'e8895d8f797026a2f19ade2b548bd2cb655e2d756dcc0a50850fd37148d24ae5',
|
||||||
|
427000: '0ca56cf4fd882583854fca00e339534346d53178c6679f8eb1e7a5dbb7b9e7b0',
|
||||||
|
428000: 'd3ce901843f952df4b63e755a0f58924ae11324aad47afece9a93502dcae7c13',
|
||||||
|
429000: '774cf2d1ba62f20f5e1ce50692f65f0ce35c7456d3988cfbb2e1b753e0ab0f51',
|
||||||
|
430000: 'a07fc310f6201c30f1fa70a082348942c65743d51e3c19492596f6aab4bc9bb5',
|
||||||
|
431000: '7bc7f405bdfc91dd01877ccfd6142c973d81dab12fa188f468abe6c9a30fe0d8',
|
||||||
|
432000: '7fe40e145078c1952f87547156e2147144449971f108164d27bcfb93f1c840d7',
|
||||||
|
433000: 'dbe52d2772d6c0461e53357b71a79096140039ef643876ec2d27e1bdfee7e81f',
|
||||||
|
434000: '94b379e842d4979bbec171bb08881227ac4e9477186ab63f7eee9b4261b92365',
|
||||||
|
435000: 'bc9cd265de563cfe80090ce12085539bc67d406c23e4d41751f191d6b6c63168',
|
||||||
|
436000: '3ff03098d2c20b5cc080363a84d8295510db937272801454d429c4d17a1dbe10',
|
||||||
|
437000: '5b8ebd40170e217eb66990af3574b80f4e66bedd092e44836aa5101ded50c13d',
|
||||||
|
438000: 'f725813181d50c0b20552635ecc578685ab0218619d53ed23493c46fb060ecec',
|
||||||
|
439000: 'cc7cb296ae79850ba2f6f7532c445189452831957f62cb823bc83aa6b51daee9',
|
||||||
|
440000: '757b816392227c9d1a14f7350cab6e77a46e465e7853b1291abdaebcee2bba94',
|
||||||
|
441000: '834515861eb72aa472843a6924fa9d6350630ed56e35181d9fe66e3a46b6952a',
|
||||||
|
442000: 'cac31ac937ebb6f0cf4f3c33f19d66a05bdc11105a98ef12c32b494e19e772a0',
|
||||||
|
443000: '552b03680b5495c7bd6ec58db0b6767e6bde5c5961a02b7b2130ea6116bddbad',
|
||||||
|
444000: '2bea6ef419d9fe81de8bf5cfe21b2603f202e2b9a959d2cfbdcbccbb7a7406c8',
|
||||||
|
445000: 'a4106f2115b356eca92e5e3f45b8ae0e13b82af94bae391942200d1c59d34d17',
|
||||||
|
446000: 'ba49924afd8a40260e546e5e4633e945877ac9d5754c277b77bb154c0c9d0e40',
|
||||||
|
447000: '83ea56135c2b052df0fb09c68cb7a085d876854f34338abaf3cf51760c83c3eb',
|
||||||
|
448000: '806d77789673fa09d24c188ee024e71425df7baf623fda073076f3a9b87ddbce',
|
||||||
|
449000: 'c41d60b93fca1d2adbbfcfa99475a8eda152630d815eb1f6c58cbd15a0c2eb65',
|
||||||
|
450000: '732d315feeb485fcc5055916df17827d80e7f831c43df81ea79123882c4fb356',
|
||||||
|
451000: '202156d29e57c51941cdb7256918128589f5b8dbae5762bbbeacff6eefbbfc71',
|
||||||
|
452000: '0194e8482252f2492e1e2696fa5761f50875d0188310a0cf1d967edffb1b79f5',
|
||||||
|
453000: '9e63570821d8b5cd7de0c5f9fda67300f531d87cd8f778fdc6bee9519b77e01e',
|
||||||
|
454000: '632353fe048e83ebef942d64d550721d29b3fc8083a247b2443f2aa2b706c6fb',
|
||||||
|
455000: '65863feebdbd60d16330000cbae496667a67dd5fa8b8e3b6dd3732843f6222a5',
|
||||||
|
456000: 'bbc554cb3c0281d6853ff2f6c25b1251eb108c09e5a6c4a69a56e8f79c9206e4',
|
||||||
|
457000: 'fe6fdabb84ad1ffa807fd55db29dc95dafeb0e9a5eeb5ce75be8a4d16f9f116c',
|
||||||
|
458000: '949bd40d861e4b297352722b6f01efe9574711fe64b10c2e8dd769ad99cf9655',
|
||||||
|
459000: '9c139c9ccb41fbb3ecee878cf47bb37e28c76816c257f99dd7b23ec1f3eb3ab8',
|
||||||
|
460000: '61c3126d504fb38ea21948d2837368441389155e9675c7f1629eefcd03198f8a',
|
||||||
|
461000: '28ff145fe2abc7cacfb1613d40b45e13580c643ad68c44db81d15125324f4f33',
|
||||||
|
462000: 'e6b532924d07f1ff3976a53397327ccb232d3b2ac6ab090874cfa6ae47b8b315',
|
||||||
|
463000: '15ca651db9b159978ba6b2adfae1669e3042f4311ed683c72838bf337d71274d',
|
||||||
|
464000: '6903539d4f38f728e075441efa96b8013295ac166db50ddd4103fb2bc9937f0f',
|
||||||
|
465000: '6b6c2640411d900c13f5b53a9a993d10fbbd7f03f6fd477e120ba06a7cd67778',
|
||||||
|
466000: 'a05d282fcb9c091a0a96c2ddb3728c1e98eb767fd5311bcc08c9c10b5a0eb24f',
|
||||||
|
467000: 'afb81fdfd562f2cd2d53434e6458c38462f85b65958ecade001bcf2b49f9b28a',
|
||||||
|
468000: '97374586191b3a30654483722f7c6486ef9d9f71aaca44b72146c11d4f4b80c7',
|
||||||
|
469000: '90a829ff58be9ad30024ba1851515cd96d567be1d99154cb966067316d1902af',
|
||||||
|
470000: 'e9fbf2d2291e1b5af5dded3c223611d9f2e31fa6e1d8c075741aa155538bbddc',
|
||||||
|
471000: '3b4513a6a6c05ae9d2ab880a2d1aed3b189e0fca9cfa2bbc7c31ebcbffaed9a4',
|
||||||
|
472000: '32a57820226f672e695823620fccd42a6fe624dd2d88f7e7c76e0e9128883b27',
|
||||||
|
473000: '15cb7e920799939eb01d06b1b713cba3ed31ef92ca037ee98eb8101b6331e5a6',
|
||||||
|
474000: 'cc23e76e68dd726f31e592a4ea5d6aa3cacd827d068317a0a476f16ce468d9f9',
|
||||||
|
475000: '9e7e33a8ee53a4bd94a26b35bb43054b42a238a85a859ed9e4a33db978a128ff',
|
||||||
|
476000: '23c85e4d8edc7939fe0e5a47fa250f0921b366711c34910c394af00a4efd90c6',
|
||||||
|
477000: 'fd9d489fba4175c1fb3b8a0f9bb947287e64da4bee9f8a7fe73411632c820368',
|
||||||
|
478000: 'e287b69696013e70b276843e83af6cb5cd076000cb280e43de29cad7f706dec9',
|
||||||
|
479000: 'e86c7a06d96887e404c052b0fcedf10d40423b02d6b9fa7ccb9827551e5d089f',
|
||||||
|
480000: '74e7ad7d78dc585a5cd1054c0288aefcec0e031cd40f0ebaa0fefc7a325f212a',
|
||||||
|
481000: 'a01f98758cd3e561d02bef30c054ef70405ebdce9e670a7d6165efe3a9030886',
|
||||||
|
482000: 'cc9e22f04e4e2e4f52b8382281092ba50ec99722c9172a0e1c9ad654d26bd4d9',
|
||||||
|
483000: 'c92e278d54111ab2f3906eafab87fdb932307e60836df209856a97cb9e1ae3a8',
|
||||||
|
484000: '18fc89d07631aa01891aa44231dae6cfa43fd4733c193468b033532a4574eb80',
|
||||||
|
485000: 'ab546bb694363e07469dec48b559479d34d724a43865fa8f3a178ddc7b83e8a2',
|
||||||
|
486000: '59926fddc89abcfb55775a58e5e6e453e388f530f720dc61001f1ac59b59cd43',
|
||||||
|
487000: 'b41bcbd91984a3dfe72c87bb7c2ef32ba3eb3cf4318f5d37654459143dceca1b',
|
||||||
|
488000: '94e859b8f01ccbe9fc5bea5c274f252e23599cd444d5eb0962ece97110b86eda',
|
||||||
|
489000: 'fd99b951714162e55d8944e8c3ce9b098c4b74ea24017e4b204f95359202df08',
|
||||||
|
490000: '80b84d18f4bc6d624d4190db94920976926bbe48f2e3e0f4c08106b9f99ac5df',
|
||||||
|
491000: 'b0c5b14b01225b1ed8272dbe9b2def41756cda0a3fe6a1a8e75b0795b72e6a7b',
|
||||||
|
492000: '572dfe57507384278def462488416248a3039c1a76a8c4aa23d1ec706de1e3f6',
|
||||||
|
493000: '75f22a8d4d202914c0f3bffaf95e47eeb82eb00dcd2913af5dd67acf6f8ed8f0',
|
||||||
|
494000: '626e26524f9427582d61f340ee9181d8f238ad8acf96c1862724fab62ede8c8b',
|
||||||
|
495000: '69a11346ba65c79b5fd566768a8b2880a5a29e2552ee8e343bc88bcf43e9ca54',
|
||||||
|
496000: '314c88db22753d317c7b65f872d54d3ff672d04b3781293d59a1ea902811336e',
|
||||||
|
497000: '7bfcf2614b54ac3808894694e8b15940d6740760a2252ba4b47b2926bdd4bf51',
|
||||||
|
498000: 'cc02f90d2fd4e0f378d2ae3bf1011026ca3d7f6642dab316f5fbc2f57aa840d0',
|
||||||
|
499000: '841182d8519cd7b2d8fd01ddd087de892ecfafaacf9d6aa029ce1ed71ee0d538',
|
||||||
|
500000: 'e2742854018ecbfef71d626ab797b7c6137e84ff1e27a05b5da84b2fc7c43145',
|
||||||
|
501000: '07334852d0b542f897164e4d74e699678c850af4e46111d495701a312be21105',
|
||||||
|
502000: '6150771ac25f4185eb94ef6aa15e7f409b6beb411358ba7d982755655d431492',
|
||||||
|
503000: '156d3165eb11d021c46ae5968aeac20f65eb05e68d841bdc98dde156d42fb500',
|
||||||
|
504000: 'e9d474e59bf527d84f031ba41cf471f49e5bf0472ae1b3f48f036813a0b195e6',
|
||||||
|
505000: '261cda40eed5ec5b2a44edcd995b24887a45a417fec3f68507a4d600ef2abc33',
|
||||||
|
506000: 'ffbeca24a8f2b7cd6d64207911e153a2f0bb70d34bb4f073c33197e3a2276558',
|
||||||
|
507000: '14e2664bed708d57478f6fb5a4228e4c26843bd6533056fb38dbf1e7595028a9',
|
||||||
|
508000: '2750e25c3dde233447266f7b35f78d8bff18bbde4ae094d06e38fbf6cd2c6902',
|
||||||
|
509000: '5fc210a72d142dd413bc3b76d5ac772564a5917aef13890e7cee190c61b059d0',
|
||||||
|
510000: '7184fa5628bfe59481bb3542bb4212b535c3ede3c24425f0bf7b1b453c238c6a',
|
||||||
|
511000: '2b4bec04d73f0e524cf12d04bbf7b3859b489275edd047f41b139ee6220074de',
|
||||||
|
512000: 'ebf81767f4d26c98bce62a5f5d1a289da5bba770e88f346e2fbb1d676c155e09',
|
||||||
|
513000: '8bd8a99c65a258ebec809ca76e2cc91b0b4fec5dfe0970736d2fcd706aa660ff',
|
||||||
|
514000: '2dd7cba17366c483819ea090d3196950ee35a4bd9f28acd94575525c851234d5',
|
||||||
|
515000: '33ccf5f5230b9a79cd89eace91cd1c2858f6d040a11494ffb8d51ee934753d57',
|
||||||
|
516000: '30eafb000d199c8d9d1a55c5b09a382a3b10bcbca837e584db6647699ce6a3c0',
|
||||||
|
517000: 'e7ed622512e2483fb8537579ce2d2519053f9a6c8d19c42d665007632d5f74d3',
|
||||||
|
518000: '5180d11f98a25b8698b969c3c9a4b736522397859e7dc00665658f93c6cb8be4',
|
||||||
|
519000: '17ca81b3a3c942c80384c4d2c176b5274c52569b90eef686e077f53a884130c7',
|
||||||
|
520000: 'fe1f552778cd5250ba6556712217b86fd6f1d574143f8ad6238e1d6b67481f97',
|
||||||
|
521000: 'd7ac10bcb8662ab9940c4212dbf0c4b9cf9750e3c04a905efdffbce63f065de7',
|
||||||
|
522000: 'fa7625300aa890ccda58268eaf2daef14539c195f93311099754f955f367413f',
|
||||||
|
523000: 'a55ef67a0e75bea12003609359443a28b42b9037290872d6b1672433cf5c0d34',
|
||||||
|
524000: '08d1d556393bbddf6eda11df852a792515ab9c1d519bd91c48711a0de8faffcc',
|
||||||
|
525000: '76b8691b16f7dafcf980ec3e7c69c2f3caee32057ac8d61b3d3aef0fd394642c',
|
||||||
|
526000: '6d3ecbd7132a725e330f0a566d8d602a9cd96326a36387c1c1fe52f7f6ad975a',
|
||||||
|
527000: '71d08ce7aec6ff50fa1ddc37f6b1574db6e71004e8e111f04a9b715103f68e01',
|
||||||
|
528000: 'e9b103c11db50a121c9e3a1fb4cbbd4fb124b07a67167c5a461a539dac3f932c',
|
||||||
|
529000: 'd2f784907a019c1eb5830a74799562cd6d21570c4591bb038f379897f3296d75',
|
||||||
|
530000: '996bca5a7dfc4a8c7f9f8dfb38b1881fb97893a950f4c8f57462506782ee8ca7',
|
||||||
|
531000: 'c84dea2e7fb0c9c884f8dce525b8b65bb7deb33ab7fdbcf72a5382e9bac2ade5',
|
||||||
|
532000: '3e200d76a92dd1f6fcc420d44ebe446829fa5e0162177acd75a13c98f1477cd1',
|
||||||
|
533000: '37fa7cd81c658ada13d04dcf0e674017c6b7430bb0cd5a080b1b924396377b52',
|
||||||
|
534000: '9483c344293c376d2bd247f2846a5ff5038b27af48ea5badcde101fcd0113dfd',
|
||||||
|
535000: 'c9d4f9982955c07960da1e4f92803ba135f4584416e0767a3567e4d2a21e62dc',
|
||||||
|
536000: 'bc9abe4e4be06aa8f61b599d64923ba88d36886b638f413b12bf87ae852e4010',
|
||||||
|
537000: 'df8307b0097b4882740c18432ab2d7e64486cb7bb70b503923bd630ea277e9e3',
|
||||||
|
538000: '6e2accc5e65a85e0aad098dbc7021bba3cac37c70f6a0ff3135f86b90954bd2c',
|
||||||
|
539000: 'b7d9f50727cb512dbaa1ba073d227805e7adc70f9aee46b9721a65ca3d34a8fb',
|
||||||
|
540000: '19c695a6b8bd256fddfd00c53e6401829ae59ebdc1e71ac234f82ffb2991ed0f',
|
||||||
|
541000: 'cc07c3740f96ec8929c7c8714892a19815fe64f4bcdc4d1bf12cd0a9e1e82125',
|
||||||
|
542000: '23ffec3127ec3e093ec8221fa760510e63fcab31f9f0fb2b48727649e4308980',
|
||||||
|
543000: '6c73b242bb05af125ce4ea324d921ece54c9510980128605aec23cc465dbeb00',
|
||||||
|
544000: '51f215e5f11ae31853c4caaae1d01aac2dbafd00302b70e7198e213d485d5bfe',
|
||||||
|
545000: '2c36150f62c86d050e4c64d83aa0071ef4af0bf8e1e17623c740ea4ee7f3275e',
|
||||||
|
546000: '322e6d77f529412c4b7a1c7778253900f1976f238f96b771845b56aa080f4d23',
|
||||||
|
547000: '1d3413530079389231bdba0a0a7de7c85a4ca991569b2f26a1e48ad2f34a995f',
|
||||||
|
548000: '3576ec0d3ed469c5498229af531fdda290777fa1ab7cd83c49450f730c5ac0d6',
|
||||||
|
549000: 'd2a5ed3d920f273bf891ac40ece534a56f8774c8809ed7d8482ce4b78bdf016e',
|
||||||
|
550000: 'e1f75701ff829339e03c019eef2da1ee4239f38879173a833f81295200e96d29',
|
||||||
|
551000: 'fe8339fb53a33e3b5c4a39cdd64d869f41b4fc88c73fa018eb5245d2a7a01e94',
|
||||||
|
552000: '8c2180cc834d5c14c9801772caa65e82b44cde33da7aac586c81b2bfe77412dc',
|
||||||
|
553000: 'db9b4ea2cbdb21fd48a69da3bac2dbe8cea2a16d45f9252776633661e28fd1d2',
|
||||||
|
554000: 'd09c32f70dd94d737205922618b789204a39668b59fd3ef405ba6d4ba3bd8aea',
|
||||||
|
555000: '6e8efd0190d866972b469cc827330f84f515d6466d8748fc33520e8e0505bf3e',
|
||||||
|
556000: '8ce7cc8aa6ad30971014b332c64ce87303098e64391a9be03d1c621df851c00c',
|
||||||
|
557000: '9959211d82372edae69cfac47cc5c937fecbb6de9e235a5a3954c653f3b1fda6',
|
||||||
|
558000: '4f1d620babaa28a03d02c2ba8b7d1526ae47d45bac1220a1a794cf4a489536be',
|
||||||
|
559000: '3a0f1617c6056a27b1da33f0e4ca80b5b6648341551c19622f7f4b4f016ce8ce',
|
||||||
|
560000: '54877adf522c9abaec851a358ec6fba9957227e0e306d3a425167685380cf6ff',
|
||||||
|
561000: '55db8f8ad8869221ee891244767a7698d48ec61264729cae19008779dadba422',
|
||||||
|
562000: '4ac9c41b29253ed03c45222412c53ddde7b9da63441d06ea91b448d89dc220bb',
|
||||||
|
563000: 'cce11479fe1fcda6af734a03be8ea044df7e2726df67927f5bc76c33728133a2',
|
||||||
|
564000: 'e2e5c52e028ab1be7f5d182a217e16694ffd370930cdafa9443b627e8da256c5',
|
||||||
|
565000: '81488a98ee93b0cf5704cb1f46383456bf5238fdae7bcde4d0826276b5a48235',
|
||||||
|
566000: 'c9afdc804f7b0df600c51b9d596d6107a0cb71a4f8ed3ea764585ffc5f70644d',
|
||||||
|
567000: 'efd3ac4baf1c6e1fb2f8bac40f1416f189c9eb1135330ae4c25f296f012907fa',
|
||||||
|
568000: '225c3db3b83b78fbf9459c7e1e53ae35b01752608d329eefa99247d8a2b9b83e',
|
||||||
|
569000: 'ac98f3fb9084261c19a63fc8f10baeb5b03aaf9ca432d7f763a3a6706f81eefc',
|
||||||
|
570000: '4ee2eae15c9019466bb50d103947bdf60a5397f8fff56154ef782fd1b8579dd9',
|
||||||
|
571000: 'a89f29bfc76988560ccd59a61f8a323fde7c4020abafd577f88d1b0b4c289260',
|
||||||
|
572000: 'eb8380e096c79806443535f2f3b007378e43d2cf3ff56f3b7e6a1b2639ff3da2',
|
||||||
|
573000: '3182ec1555aae0df8d76dd5f37bebac6936e0e0c5609263263eef2e6e87863ef',
|
||||||
|
574000: 'fc45bbadf22417d49e7df89f02df358b85a4cc07adb4010d3bc0334f397b01d6',
|
||||||
|
575000: '6faadf2b0c3a98952bb047a58b83466a223dd323a6e32d68be5e8d064482fa84',
|
||||||
|
576000: '68465ee1d64b65ca71bbeeb611a8b4657d3ce765857d049536047e2ea0490044',
|
||||||
|
577000: 'a019fabcf35e7d4fe15e4e3a499dea2bedc61e4c9cea2803731995def15a9794',
|
||||||
|
578000: 'cc2fb6111fa1a338eeade31574b77a1012ec5948f88e7e5276f7ba4443df3db0',
|
||||||
|
579000: 'c5e7c605c03e56f221397f55bcd164dff21856f7fa48e745e61f2d05d7de5f54',
|
||||||
|
580000: '110fe7150988c930f9c324e00e108050b8be5ca2eaf200f54477073c72feef2a',
|
||||||
|
581000: '0c3adb09dca9a9924fcfe30f16bc70734ec62fd4a306ce65a38b74c5d817ee40',
|
||||||
|
582000: 'fc47433896c47765651d8cb31c88cbf80d0887bf071eaab9d2963af9f8bdfb56',
|
||||||
|
583000: 'c7c518f8634bc7c5cc536c9b77fe119cd6cdce4fafb25d4c0aba53c5bd970e35',
|
||||||
|
584000: '43b2357dd053608b30488aca98f5d9c151a68730fb8d008f1bd1488bfab0c391',
|
||||||
|
585000: 'cdfdb4ebd1c5fc3689b4f013254c832da7bfb12f3440fd327c6b065afd093dce',
|
||||||
|
586000: '48979b814cb6964b20ad8b322c4936c7b241c24fe6d610181c43d1cd003e5d90',
|
||||||
|
587000: 'c262a418368aab465facb8f1a82a392eb74e3fd56760779033a97a3f930630ff',
|
||||||
|
588000: 'b20dc4e0c17a223408a51b958b6b481ddeadad513e046c86c7418a8e96e35992',
|
||||||
|
589000: '0411c5ce49578218dc8e225c11276ed9a9802aac47d8f5837c4eadb3bd72e1dd',
|
||||||
|
590000: 'be8fe8b058722f4919af2834c076e2bfb602654a0343b57eb568d0ed4833fb39',
|
||||||
|
591000: 'affcbbd2a9fa43a64332412d649c2c0bd4b85e90e5e67576f09ee0c207a66cdc',
|
||||||
|
592000: '14a2d414220d240286ec6eddab0b0540ff4d1794b39f7913c00ab005e2ca2a3d',
|
||||||
|
593000: 'a1e2744404626b5e8e101402f50eb751c282ce692d58493ac6fa2efa43254526',
|
||||||
|
594000: '679da754e8f4cfd017ed5c8111361c0f9624104a87a50d077dd7bb5bb97c7ddb',
|
||||||
|
595000: '0c31eeb25d718f96c68de13704a0904c3a524b5936690ddab8b4b38fbabad5f5',
|
||||||
|
596000: 'f930b6516bb01d3365bd0e28aed56a580b4efbb45b0abed5ba941f7a56b545de',
|
||||||
|
597000: '56287fa7bcb53be966ba75c15b92ddf94a7bb51f4ef51037e56ccca936df4d41',
|
||||||
|
598000: '1a6303e68038921fe23644dd10f5cbad4af3648f7cbe8432860f439c84fb226b',
|
||||||
|
599000: 'add270bb4fcb8dc9c254c9130fa037dcedb8ca922c2e35ed403661a896addf5e',
|
||||||
|
600000: '786ea387bc7e1665655f1f683d56c67baeb9bc482bbe4401acd167793d34a508',
|
||||||
|
601000: 'a4c5f398f28cfcdc6e0b6fd467582ff5abfcb4f003cc59ae7703e9467cdd9ee4',
|
||||||
|
602000: '21b19a5a23344325581e7daf0a398b37cac5b4c335237bb8e6c14e270bdc1567',
|
||||||
|
603000: '000eef5b4300cfd804554e199ca6ffcfe73ec8446e82ad32abbc51f45594a368',
|
||||||
|
604000: 'fbf77437dfba9caa4d11b03a3673f2274dbd3d92e704bc5e8ed3f90d99d2c988',
|
||||||
|
605000: '0bdba251f44baa12e2b33b1ba89eb8a5d31ecaf6193b836e6356845149ea3f33',
|
||||||
|
606000: '2dcadcae1cdc68ab6404606937aa9981400e12ae77474462764b8077d391d2d4',
|
||||||
|
607000: 'eff9dfe7fb3e31536dc12c78603f023f8f1f59b2b44a59359e8bfe50b2e22299',
|
||||||
|
608000: '50fdd1223efaaa261f6fe592d7b9e1e32b5a50f78dfee7bc8af19250f8a82776',
|
||||||
|
609000: '5d88ff14c7c36602251061379179ed87c5f7d60d746d8d10fe678a0303ce6596',
|
||||||
|
610000: '5e56249989c623bd7f818ac1e64c5d1b1f3afec3678559d446e356ee5bb3394a',
|
||||||
|
611000: '5a9b15a38dd35b9ef0a028b12ab69b0993eb4bc5577d50c486fdfd032f243a14',
|
||||||
|
612000: '258213d8b7141418f22e303ec43005bae0508a3597fa99c4d59d656d7d768af2',
|
||||||
|
613000: '5ee8b46ebdec5576550540f7fe01c5a91c3ba7bfab26db605ade4e620430cc21',
|
||||||
|
614000: 'c41a592e2042d201bf028bbfef484d264a391db9dc9ef302708e7a39f723cf5c',
|
||||||
|
615000: '70fa6591c0b7bb63665f040b87750668be47058942c79c863bc00232f536d725',
|
||||||
|
616000: 'f4932c1c8fc84e3f4b97336424e27eeb09a65d7347097e983e1ab698a38dcc0a',
|
||||||
|
617000: 'd82bd9b58adf54a50bfd471c66144f2969ab443648fb7bab8299408afc021544',
|
||||||
|
618000: '72f7dfda9bd67b900106e632cf50204fe27cecdeea51d9b0cfde9ba6da8dcd5c',
|
||||||
|
619000: 'ac434b8d723ba498bee878cf589c7d61b74c7a00ea5b7abe70149ba878522332',
|
||||||
|
620000: '519930116d8a8f3b01ba443bbe4fe5346cebc783c632cdd5f62d305f61b6f446',
|
||||||
|
621000: 'fed977a0174f8cdc3e979e7435ffde2f7dbac2d4e1c63593117cfe81f320bfec',
|
||||||
|
622000: '1a6daef00425e6ad136d7bda36cad99baa762f6d788b92f630ed1af33995211e',
|
||||||
|
623000: 'b9f1e2836bd2acae3a9d21516052007005f3f67ae1fc133c1c4c336abe0dce51',
|
||||||
|
624000: '5d4a19f7e7f99b5c39bc844b0a90ebcbd97d88f817f8f2edafca78ae499c3167',
|
||||||
|
625000: 'c1afa80aebb856746da25503eb786f247ab098a94ddc58fc96ca46caee7019c3',
|
||||||
|
626000: '222c2ec50130a658ece226c420c76a8f7a15640033fd89ccaa29bd7c639cb94b',
|
||||||
|
627000: 'de9830e9cc7537c42133153cccf891dcd422203827b1377df748ec27dbfd04a9',
|
||||||
|
628000: '35d44ad84a3d9d548bbc8c91ca734979759e3a03e37153b650f794a0aed1b742',
|
||||||
|
629000: 'c119f3acbd761598673b48eaf0d223e1729b8c8aaef9a715f1bc3dde561a4568',
|
||||||
|
630000: 'ebfda8a47983ced56d40e95ce3be793305899cb92359ee2f9a83f7f6a59a4fe7',
|
||||||
|
631000: '1479c2e6217b97e98f44bbb315ca99d1ab02aad14f52d60f7e6670f5c6d0fbe4',
|
||||||
|
632000: '6ed7f9c90ea95825b1aae91644a68e36b6452e61f05a2821e94399823ad669e3',
|
||||||
|
633000: 'e00eb0bffcf784f9f8fef21b92b9cd98e44db5d790b6eab309d0b11adf910820',
|
||||||
|
634000: '3c71674dab2c2644d910acff4336fb02c2dd8281e30e5c97b9378d50f3d6eac0',
|
||||||
|
635000: 'ee3fbedd791f31de2f820cf37cf5a7bcf1fddff354c1ab5736384f41846894f5',
|
||||||
|
636000: '551b7598f5576c577b48ac7530e56422a5fd31a3860b37994f8489308830e9f6',
|
||||||
|
637000: '531b1a767734f1750077ec9d1ab57fd4430e0154deca2fdb9dc5cfcbe980726c',
|
||||||
|
638000: '3ae3a009b3822211b3bbe9745f006eaf3aeae365e81890205e0223925461af81',
|
||||||
|
639000: '2aa1de8c824fcc1a8f34269010405a0a0fb921fa0f8b998010e4076fd722948c',
|
||||||
|
640000: 'badc624e2ffabc438b7b2c9894e2134e36c2e80af700cebdf0980169c1cfe49a',
|
||||||
|
641000: '0b550e8f9b66ac3a3d0e5a911819643600dec121959fe2b5128cc6c68503e655',
|
||||||
|
642000: 'bfccbe9a0a127d4cf606adb5eb42f3f0ee2ebc780b719d0ec612e590aa7ec47f',
|
||||||
|
643000: '4d44b13390410caf86d145103471469d0b10450166f5b96b8886a440aab5b1cf',
|
||||||
|
644000: '2670950e311d8230e8d3447e97652cb5592248b59968bedb6a98a98c12b8d81d',
|
||||||
|
645000: '6703a25f54614ff79c954438ab9fae5baff32c81fa47306b337f0f87bb11f949',
|
||||||
|
646000: 'c93876f02c788041b400aa42149c176e624eef01bc66b49adf8fcbe255dd1833',
|
||||||
|
647000: '276521b16f6a98f5399a9754621fbd7999b6b94e3cff17cd88d4a1176f431353',
|
||||||
|
648000: '65b6238c589baec511c240cc87cb6ce758ad6cf119f465838a976a853c7f5be6',
|
||||||
|
649000: 'c94f6f7fa7a8afdbecc7c291eab2a3cf218ff1af5117ed1f3f0e60214ec7041d',
|
||||||
|
650000: 'b823008cdb0176fa0522c26b35962b4325fd526217f7eb19464cffcb3c24b4d9',
|
||||||
|
651000: '77024b0bfd3cc7c8d52eff6db4dcec041f8675e6f3c413b7d581bedb1b53bb11',
|
||||||
|
652000: '22a677eef4cfcee12756492eab7e184bda054481d4fc19cc46bc7147c097d118',
|
||||||
|
653000: '66779d92b9fc28fddc46ed14b79b26b3ad7672f0dbc7cf4462d5a49769a89470',
|
||||||
|
654000: '097eb77aede84c35822b3700673efd0129de247cb8ba5ef6b8e6dd7e85bb908b',
|
||||||
|
655000: '83b5b4d4aeef3782c82b9a2fd53937e4523ea15867780e4264688e860fd99593',
|
||||||
|
656000: '99646218016eda7976b45fbb6e891fc4bc130bfb3b5e6dcac8b11e0a4b2f59bd',
|
||||||
|
657000: '9ad1995af3f7f6839a11bf5fcd856f471b200911e1e5a647390a7aa26abc6825',
|
||||||
|
658000: 'fd5a4462737448dcfa9d81802d16f4621ea0b177f538144d836829a536cdd451',
|
||||||
|
659000: '7f6baf86cafbdc820437d63260fcc434694dcf0ab8000307f134fc9c50f437b6',
|
||||||
|
660000: '54a5f2c5a1f1534f23f54c5ac6e3f794ebb62f298dc9ed1aba4112e10cc778cb',
|
||||||
|
661000: '91b46c2d247a43ea916adad7edcc37e18d149fe3ab970eaf39cd060a9c856ce2',
|
||||||
|
662000: 'b49e0df59bfc9dcd32cd1a4b903a12accda70054fbd5f4bc480de0367f254c21',
|
||||||
|
663000: '29d2d3e06f744ec2225659199bd43e446308fe1e1ff16c26e0e89caa468450f1',
|
||||||
|
664000: '50b0665a8c7a9fd1ffb165fbb0148115f83fb7a5f6e9b8ad16c9dff9175d11c8',
|
||||||
|
665000: '35754b82653110c6cb0f9a0e7011ed1ff8b360301e4280e6aa2f6faa5950a369',
|
||||||
|
666000: 'fbc64efa658b00bbb70f5db8265e99465a07627d663c9a716ec8b42bed82338e',
|
||||||
|
667000: 'a046d6957f24ec9f66c73612d3eb483c65e4216fc2761260d9818486cbebe56e',
|
||||||
|
668000: '5d78ef63ea8ce0f66032368effcd56fa443b0f5c488ef8c243d3622113087fbe',
|
||||||
|
669000: '0b1b551500e2c8617150a18deec1e3b9594dd98fc9b97e20dacf2f059f966692',
|
||||||
|
670000: '5fa21e8edbdbb2ccccc2e3aca8e15b0965cd307ea54b6621760e654b53488d33',
|
||||||
|
671000: '90affcb451a786aa42ff93dd1f39961a15cc8c332613f8dc591d9f9c34d359e6',
|
||||||
|
672000: '1a8996b27fb79d43627caf0166f5c8ec72e1e0d09e6c0e17a0d9418d37719afc',
|
||||||
|
673000: '491ec5299fa141a82b438ef5d2429eb5560a0f04412878165c86f9fbb94198af',
|
||||||
|
674000: 'dd680d4c1101a98b6465ac85a63ef12f3ebb1eb88277d37aa341a5b387e2f1b6',
|
||||||
|
675000: 'af8acc98b0fd9ff347acb01801f246c63c6698aff0a0cd69b0f91e17da3742df',
|
||||||
|
676000: 'af24a5759888314e5ef719ee23b9f948b3962ac0b949231a540d45eedaa614f8',
|
||||||
|
677000: '7ec568d4e5ceff99c0e9cefa6407784aa0246845420812bcfd8a5d9c03cb01a8',
|
||||||
|
678000: '845df5d7e1d6f22242ffc4c976937da67ce0880a94c5bf189195d5728f208976',
|
||||||
|
679000: '41f895ce11bd91e09ea8b16102c8f2192cdd2c754e2cb04d33edea688fa7c3b8',
|
||||||
|
680000: 'cbf1e3a313abbe05d90ff1de6d861cf58da50b7f8f7ba43a593a3b6192d65f8a',
|
||||||
|
681000: '1ac184be2b5edfdcc0a73a6a93d3f59a5197bd5bba2ddb1ce737251526998bc9',
|
||||||
|
682000: '0e793523ba0a679dce1e9cf1d1189c5311e27ae8d88c35bce36ddb3c16dc34c4',
|
||||||
|
683000: '08e07172a6b6c3d70c4379b206ed796ea8d916d314d0b6f02539becbb90077d5',
|
||||||
|
684000: '0b67663fc37940b4d40ba88b8b7610776e35f070e4995c5ab09ccd4b86dc1143',
|
||||||
|
685000: '8ae92af68bcc012326889b5ef89b899aa38f65dfd3d9f7dd5b29dba0c5fbebbd',
|
||||||
|
686000: '5223ea0cffd3c59fa6597208425ab55b3a76366fbd27d22508cd208ebf2a2eec',
|
||||||
|
687000: 'e160e55af7fd110905364f980543fca62a123c84c81e9fd38a0aeeebc30a501a',
|
||||||
|
688000: 'fcf3b5e0afae8d665cd6f63dacd2a861aecac20f8b73a682f79cad9523c5c4f3',
|
||||||
|
689000: '772ffab484f07ad91ed45cf6f569eff7440f4243e16e07698e4b7bd4a109e6e3',
|
||||||
|
690000: '5b2a4fc69617ccf1787ee40a0f6d7e0af783bcd856c2c8e0ab747a91a7c68d19',
|
||||||
|
691000: 'e4ba352a759e2425d508a4d5b58595e6ff5eb912d8bfece5a0bc646f61e77084',
|
||||||
|
692000: 'fca2c5b6a721db85278ab55b5e2e39a445e5b1ee5ba69a17044623a6b945d0b2',
|
||||||
|
693000: '99f4365eee70f86499ec26c373922d389cbc5e2a198e96af5d5823ad241748a1',
|
||||||
|
694000: '0e19e76398e65cb01517dcd4ee702c9c04c0fd53cf9468ee539094aa6248e1c8',
|
||||||
|
695000: '5addd92022007d81ced43595458e2eee2903227063af8e9edd75cccbe559930d',
|
||||||
|
696000: 'ba25dfe3467cc9999eb8593d955032475b777cefc8006851f692266ebb83d140',
|
||||||
|
697000: 'e10c1c734038198a99bf970ae89e3a56b2058612e35eac8141d48e147d2d47e0',
|
||||||
|
698000: '472c4de8e57737ca48c3bdbf3c35c37f24a179f5dfa48af89efda3c3d33c131f',
|
||||||
|
699000: 'b359f2e61a3ca3cf8b613045617e38b06767ccff72129f13971faf26a4d08234',
|
||||||
|
700000: 'd0555d53358978ae0abcb09f911c1b3e7b5a282b4ff6d455ca9ad04299666dc5',
|
||||||
|
701000: '5c763b8b4329809553fb58ab279ebbc6639695c45760f0c181168d41da95e6d7',
|
||||||
|
702000: '8a6abdca484fbdac5fd6fb279a435377d964d43b62393b0d5b0cd3baaab1d2c3',
|
||||||
|
703000: '50cf159af75b28c7a95ca990e480ebcd534c4601b03d7c0a979f26f4e33cecd7',
|
||||||
|
704000: '0daf7f9c7eca5c6d91de9ebadb3ef1287c290700bab0aa9f6a2ac4bf42a8a98c',
|
||||||
|
705000: 'a359e7584112eadd7fad5d961f1dee180481852d7706701c2211a98a009c128d',
|
||||||
|
706000: '139ab9432b0c7f62739f818b114d57a59cbf20584230f1198c35f1da62c2ed62',
|
||||||
|
707000: '43e1f4c83d26d419a35adc4dc5bd85c6253ec0faf7e70196bc5d5fac18c51746',
|
||||||
|
708000: '1a05dba8d5d2ef4984c79cc939148ad71535043640fd5644141cdbc512623514',
|
||||||
|
709000: 'fe7a8bd03f742de5227e5d0c8dcfd0b5cc13582703b74b1898c506fc6ffdab04',
|
||||||
|
710000: '26217b4ecadd39e9a14ffb6cbcaab6ee7bf0954efcbbbec4de9bddb461819084',
|
||||||
|
711000: '94aee5cc00e862e4952e573908449f4a48b630285f2f2fb20765d01dfcb68ac6',
|
||||||
|
712000: '21fb256c0f5133634b94f6522cb8d2b0a9b982f1672ba561033be689410e7860',
|
||||||
|
713000: '8d7d7ee4cb0598fecd85c8004b59074bedc3219f2376966e01a7faa92377cdcf',
|
||||||
|
714000: '56a033531fc8dbb7ffe07d7365cce20281e522c3ff9fe7a19d0188af351a5799',
|
||||||
|
715000: '4a6671846cccdc26d6fd1d77db50772315f932c24dc706004e11de68ad1ac387',
|
||||||
|
716000: '4222447b25305bc063b668c0e010b16f9c0802fff9371e9d096ba2174926f073',
|
||||||
|
717000: 'c231fcf5238a34cd15ac735d27b3dd0ee025714e48c408a798f0ba74be0aef77',
|
||||||
|
718000: '0f96aaa30d20fc572d9db26871143a657b706c35a9668d31e3b64280a049787a',
|
||||||
|
719000: '8557c088c4fc4745674c795d4a58aeb3df0dc91bb5ff93cc6268c4a88c64db5a',
|
||||||
|
720000: 'd699e06ee7835c9d687853187125f27145ec91daf394fdee37218b8c34fee9f4',
|
||||||
|
721000: 'c675e891a36425627e20de36f1fbdd5baba7114661f1c45f81f66a3fc55da902',
|
||||||
|
722000: '0c6fedfb6d6c1a77254904fdd2400dedce7d45bdc2271beb77e2087a0ba30d1a',
|
||||||
|
723000: 'a442c320886beccb3d7ea13276dbef8e98e1a47686cba2cdbeb6a2d2883af928',
|
||||||
|
724000: '1592e2d6ac3be7535b44db6cc99080d00b19c6663f52eef3c28eae3dac27ba49',
|
||||||
|
725000: '45b2a800f17b8571172a2658577dbe95b91ae88e611a91ac0c92609b3600f693',
|
||||||
|
726000: 'ed6257a6567665747aa354e93ab7d3e6539d6dd41fced8a2f62cf848e2b30ce0',
|
||||||
|
727000: '4b1a577c6c2358b0344bb1befd9b8e5572b787ec2bdc0bdcd4a150f26b2e2ab7',
|
||||||
|
728000: '448765fbdf6261c376120ff9401db8a8841fcbed466365f982d3bc53775b93ca',
|
||||||
|
729000: '99c2acea0af193d2e10498acd1c6d162d2a804a69157af46817b5ece5ea86491',
|
||||||
|
730000: '94cec967e44f850f512d4240cb8a52ffaf953d0364b0a1dd7604b4a01406e669',
|
||||||
|
731000: '6e63f5019439bc7e27a17a189baad0da8f5724883af3ca35efa0d4e5aaa75b97',
|
||||||
|
732000: '53e1b373805f3236c7725415e872d5635b8679894c4fb630c62b6b75b4ec9d9c',
|
||||||
|
733000: '43e9ab6cf54fde5dcdc4c473af26b256435f4af4254d96fa728f2af9b078d630',
|
||||||
|
734000: 'a3ef7f9257d591c7dcc0f82346cb162a768ee5fe1228353ec485e69be1bf585f',
|
||||||
|
735000: '9bc81abb6c9294463d7fa12b9ceea4f929a5491cf4b6ff8e47e0a95b02c6d355',
|
||||||
|
736000: 'a3b391ecba546ebbbe6e05c5222beca269e5dce6e508028ea41725fef138b687',
|
||||||
|
737000: '0f2e4e43c76b3bf6fc6db9b87adb9a17a05e85110dcb923442746a00446e513a',
|
||||||
|
738000: 'aebdf15b23eb7a37600f67d45bf6586b1d5bff3d5f3459adc2f6211ab3dd0bcb',
|
||||||
|
739000: '3f5a894ac42f95f7d54ce25c42ea0baf1a05b2da0e9406978de0dc53484d8b04',
|
||||||
|
740000: '55debc22f995d844eafa0a90296c9f4f433e2b7f38456fff45dd3c66cef04e37',
|
||||||
|
741000: '927b47fc909b4b55c067bbd75d8638af1400fac076cb642e9500a747d849e458',
|
||||||
|
742000: '97fa3d83eb94114496e418c118f549ebfb8f6d123d0b40a12ecb093239557646',
|
||||||
|
743000: '482b66d8d5084703079c28e3ae69e5dee735f762d6fcf9743e75f04e139fd181',
|
||||||
|
744000: 'f406890d5c70808a58fb14429bad812a3185bdb9dace1aa57de76663f92b5013',
|
||||||
|
745000: '2bd0802cbb8aa4441a159104d39515a4ff6fc8dfe616bc83e88197847c78bcff',
|
||||||
|
746000: '24d090a7b6359db3d5d714a69ddc9a6f2e8ff8f044b723220a8ba32df785fd54',
|
||||||
|
747000: '07c4ce9ce5310ee472cf753ddb03c39c5fee6c910d491daffd38615205411633',
|
||||||
|
748000: 'ea913798c0f09d0a27eae7c852954c2c88b8c3b7f23f8fba26b68a3952d0ffde',
|
||||||
|
749000: '23f256adebfe35d49ba84ad49f3f71fc67f7745091c91f22e65f1cc2e23b8f2c',
|
||||||
|
750000: '96db12ee3a295f3d5c56d244e6e7493f58c08d3427e379940e5d4f891a41ec26',
|
||||||
|
751000: 'cedaf12415dac1314942e58ced80830b92fbfabc41f42a0b0f054f0672ef9822',
|
||||||
|
752000: '293606bcd9fbbee5584724301b2cf86bb69204820023e1fb46c238ddfbc660ab',
|
||||||
|
753000: 'f4d43cbb38b7d97919dedc0f5a6dc8007896c4f443b76f3e5693e25bc46760cf',
|
||||||
|
754000: 'fcaad22fd815311280fe451086516375d1d9d92b2990c7c351407df5aa19011e',
|
||||||
|
755000: 'b9276f10d1844cb5b0308766c8db960490ac34a73c4653d0a91202789a6ccb9b',
|
||||||
|
756000: '2fe5581f1110c1c8dcea46cad647551bd6bd640cb37738d863e189bd8f368347',
|
||||||
|
757000: 'b9d915f366f0b010429a52245b0fb02774157eb9fd8f66bce32dcd3acc71c2a1',
|
||||||
|
758000: '62d1854fc15db56b5d0e05ceeb54c1297966bf9dc7f7a0a14b42c059fc485d1b',
|
||||||
|
759000: 'f4ca9f69d16d092f4a0ea5102e6343b21204c4ea9cd9b22cddd77dbb5d68ade3',
|
||||||
|
760000: 'df3bb86641330d8cc7f55a2fd0da28251219e95babe960a308b18e08a7d88fc8',
|
||||||
|
761000: 'a93029475de4bc7569b6ae802d658cd91c84cc253772712a279f140a6c3b91b1',
|
||||||
|
762000: '307e289dc6ec8bcd62ca8831e4159d5edd780f2fae55ba55dd446225450f46f8',
|
||||||
|
763000: '293f73514abca24f374473bd0394179812952a04ea13dc60ef5ada5331fa274f',
|
||||||
|
764000: 'dd8b082db9281e3d9bacf15d6b352fda186d2d2923c7731844d0d4764dd71db8',
|
||||||
|
765000: '201239e562d2571bf47347b3522fff89632aecea3b2d8cef05151f88b2b0bcdb',
|
||||||
|
766000: '4a55a538b51b5650979e64521998cd5c5ad055ba9f3ac0e3e2a28febc6cc2798',
|
||||||
|
767000: '3916666f2adbb05ea98ec1961f9546b9afa0f6910ec95e42ce37267f2ae4f79c',
|
||||||
|
768000: 'dc0ad881eedcb5fd4954238f462080d6e7636b058d481698ed1c077e0ce2207e',
|
||||||
|
769000: 'eaf10a1e1ec6e129289b8479a05df03e0808f1f0946f1995de6524e9ebe7a461',
|
||||||
|
770000: '7200c64f22e32de7f999583361c933680fc9a2ffcb9a5ab73d3076fd49ec7537',
|
||||||
|
771000: 'd883111a2eeacff80ce31df35ab6c943805b9e48877b413fccf371e5dbfa7fb2',
|
||||||
|
772000: '3977d3c60edb9c80c97bb2b759b1659cbb650ad2d3a6f61d2caec83f1b2ae84c',
|
||||||
|
773000: '9c7175fb8646a1a82383b4c534fd01bcf92d65c43d87ae854d51a784b04dc77e',
|
||||||
|
774000: 'e0e92485f86e5fffa87b3497424e43b02a37710517d9d3f272392e8cdc56e5e9',
|
||||||
|
775000: '6395229113d3aa2105afbaeb8b59621a536fc61fe272314b2fc3bdda98dd66cc',
|
||||||
|
776000: 'b4b00207328b5f032bd4f0b634f91323ff520ada8c8bfec241b23c8e4bfd5a4e',
|
||||||
|
777000: '14cdc6f5f7b4bd5bad745dfe6fcd114e9194026412a2e1b3f345be2eef433d16',
|
||||||
|
778000: 'd3cd7b68be504c32117b670d38d59d44b02dcf3d65811efc2ca5531d902623cc',
|
||||||
|
779000: 'afcd220e4040cb5f92d4b38fc204e59822df2218f767f2c4b33597b238a35f77',
|
||||||
|
780000: '78252a9cfc289a70192ed8dd3dddeb1b9a4f9b8eff9a5d0ac259b3254472cf68',
|
||||||
|
781000: '02ebc3f17d947481a311b4771c254f1e002b6a9198d4a5258ce6c13165aadddc',
|
||||||
|
782000: '8dd9f1f372ee6d688a0bcdc3b342c77804ba5a646a218be4bc2aa02d846206c0',
|
||||||
|
783000: 'e46b0d02ec2ef488fae455665e107520e1bd2b4f35ca52af7ad8addd2f72fa73',
|
||||||
|
784000: '9ee8a8de94231e3ae3a610b82fdbca48dc14d9b80791d20af6c365a31822df6f',
|
||||||
|
785000: '21e1cc12def8173a50158b2833bd91a62140c61646f5e08aecaee3e6da20735e',
|
||||||
|
786000: 'b3e659f84d73de42888cc0f2b69bae71dd5fa6756a437a4b21958b182faa316e',
|
||||||
|
787000: 'a9be7ba00ea6a9ea6bd03d8412ec014ca7e8cda6bdc33382f165e702811b8836',
|
||||||
|
788000: 'a4c14729f8a68c03f5a0ccd890ac6a92b39c143f1f752fe81ad051eb52d8dce0',
|
||||||
|
789000: '5cf66d224e5645097efc9c3c0392b51c8ca8ea1295151921a7912a2f04ee1274',
|
||||||
|
790000: '676769ade71c33bc102bce416e66eb2c6794b03d7b8f5a590c87c380da463775',
|
||||||
|
791000: '0228e074451797bf6bfbc941bcafcbadc972d32e4e1e0c5da015513f65714217',
|
||||||
|
792000: '0fa3d00a1f19c5ac060e10a410cf7cea18eac5f89018d79ce51ac3fc66bbb365',
|
||||||
|
793000: '5f68d0868b424e32f5ce3d8e7d9f18979da7b831b8ef4e3974d62fb20ff53a97',
|
||||||
|
794000: '34508c56423739c00a837801b654b07decb274d02b383eff396d23c4d64bc0e9',
|
||||||
|
795000: '7f70910c855d1fd88cd7f9be8a3b94314ee408a31a2da6301404bf8deb07c12c',
|
||||||
|
796000: 'b74ab8813b1d2a0967fea0e66597572e5f0b5a285e21f5150fcc9d5f757de130',
|
||||||
|
797000: 'bba27b1491d907ab1baa456cb651dc5b071231b1b6ad27b62d351ca12c25dbfd',
|
||||||
|
798000: 'e75dcb15b2fc91f02e75e600dde9f6f46c09672533bc82a5d6916c4a2cd8613a',
|
||||||
|
799000: 'adf62c826a3e0b33af439a7881918ae4ce19c5fb2ca37d21243415f7d716aa65',
|
||||||
|
800000: 'd8f0ca13a8c8a19c254a3a6ba15150a34711dca96f2d877162cc44aa2acfb268',
|
||||||
|
801000: '2a8c7104c4040a2bc31913ae25e9361df5bac9477368c708f86c1ca640480887',
|
||||||
|
802000: '1f3b09d3561c4a8a056b263289bd492dc6c0d604c3fa195935e735d1c0ddc40e',
|
||||||
|
803000: '037769628c40a701fdb4b16d79084b8fbb319fde79770a7ac842f3cdc813099e',
|
||||||
|
804000: 'a0c6a089e5fa1e3589ca282085fe7201a5705776d81b257ffd252b2947fa6428',
|
||||||
|
805000: 'b2ac99bfc4a488e7b7624b31ee061991a6dd0881bb005cd13f3dd2e66a08fe19',
|
||||||
|
806000: 'ffe63cb999a278280b80a667d2dcb60c40e43a53f733914d8bec808b694ebf83',
|
||||||
|
807000: 'eddb09fc6c4869a59b520d0befb1fb6ac952333f3cc5de086539c85ea8558778',
|
||||||
|
808000: '0f4fb3f9172e52897ea992d9f3a2024126c4d2e63e9888739f11fb1f5e4c1f46',
|
||||||
|
809000: '9641dd720d23ced2f1cb6e5cf46ac4e547afb9f56263c4cf58e3b19d407cf401',
|
||||||
|
810000: 'de6dc953acd7e5ef213b3aaf1c4a9ee1d5b756bfce5525ee105214647e243a85',
|
||||||
|
811000: 'c52c83712ca12b24b2db1b4a575e7f352b1d560cbf702e121a03bdca9e8be23d',
|
||||||
|
812000: '83143734bb965318a53a38a7e403dcdb3e3fadedb01ab12c370417fc2a0655c0',
|
||||||
|
813000: 'e480deff10c5a84fc957e3aed936690e24b74dd08fa8858a8a953c2f7383b914',
|
||||||
|
814000: '810d33afcee07b9abe16c6cdc3a041038daa131c476b0daf48a080007f08b490',
|
||||||
|
815000: 'b4aeb9e16fddd27844b2d56bc2b221134039bb5642c9e9ba88372afbdeac3972',
|
||||||
|
816000: '86e73b67aae3d248011b8f66ed414cb8a9ba4b2a3cf7e32773cfbff055d719b7',
|
||||||
|
817000: '3ebb8b83752b48242016cb682f0f6bd14e15371bf1163a5933193eaa0edeb351',
|
||||||
|
818000: '4d925e17f642f220bbf317d3d5355d2f41fbce325f190f8c3b32dc0b337d24d6',
|
||||||
|
819000: 'b9cc126d620f6b99d90a00d35957b0e428aaaa7c986bc9e816a60e4334572961',
|
||||||
|
820000: '9c2f8c142bed1f94dca29276f7c83958be8cfe11773bb9b56c808fbcf7d3b1f8',
|
||||||
|
821000: 'e5509eb98895cfa12a8da5d54c1df3f52472ffcbdf707adbf84a4a9c5d356203',
|
||||||
|
822000: '764aada4802ebfe4ef935ab50af06a4f83aa556c49fdde3d9e12e1abd230c16b',
|
||||||
|
823000: '1dbd745c2e96a365d865f990d109137d32d42977f503af55d8c00b109d31d3c3',
|
||||||
|
824000: '954304a0b0c8f549c3bffd5ff46b5b8f05b0f0fde2a36f24fd5af9d774fb3079',
|
||||||
|
825000: '17808b14f2056c1a5d46cb7617e9de9be6a1a6084edbc1bdb778586467a72297',
|
||||||
|
826000: '3ca1167d4cac8b187829b23001b438617c43704b42462c4eb001b0d434cb9651',
|
||||||
|
827000: '246d1607245e4a202f420393ac2e30e9cbf5eb5570dc997073b897f6d8643023',
|
||||||
|
828000: '1764730a8dc3e89d02d168ff6bb54e8c903820b74711af6ff27bd0c8545577e7',
|
||||||
|
829000: 'd9f3ab0cd823c6305bd8b95a96188bb4f2ca90b4d66c5d12293e8b6192bac0f2',
|
||||||
|
830000: 'd4ff51f0092b04aedf8d39937680d8e8309b1be21d36e7833ed36f8e30aad6ea',
|
||||||
|
831000: '3e92e76721b962396dce52993fa7606552f0907b38f7b2bd7b21ada98c145f47',
|
||||||
|
832000: 'df12fcdb4cbe53ba627ace6de898298de175f8671d3d90170732d110fcdc34b8',
|
||||||
|
833000: '25167ff38ae4a5964b618cabe0a12d4de62ac7a4c47448cdb4499e09e108d5b9',
|
||||||
|
834000: 'd31f5309ea179a1e386e835fc372e47dcda6871a3a239abfba50c4f368994f13',
|
||||||
|
835000: 'aff7e8dd3e55ea807fcbe284014075f420b3a23f1b0eb47bacdc1c91d2899813',
|
||||||
|
836000: '3b5ac6d64c470739bb17d1544a285affb40f2d33e92687e5ba7c5ac602e0d72a',
|
||||||
|
837000: 'd5619cbfe4f27c55f2bf9351b4891636cf64fef88212a5eeeae7bd3de47fe0bd',
|
||||||
|
838000: '1f9102a49c6ac470cb5d0050e5300b1443840d6d65719b835e3bea484aafb2ec',
|
||||||
|
839000: '3f63e391f0fbc5787fbe4ace3bada3816261294ea1c6ee435001801023682f90',
|
||||||
|
840000: '777894fd12bd0d6dee7bcde2995c68e55e7094e3122da38571e4b6c4304b75e0',
|
||||||
|
841000: 'ceb0c598c788e25e43e25aa4beff5c7377035824844cf1675eaea537074df028',
|
||||||
|
842000: '8661cf2065dc713d2ba043f0b81f0effcc940eeb3e91906a21ff22c210561dcd',
|
||||||
|
843000: '0dc2766f90415009d0c86bedffee6ebcf58042eb08262c0c67c4e9ed86b2aec8',
|
||||||
|
844000: '26d072da864cab268a12794977b04ec44fb69ef3978e2342e82225974dac54dd',
|
||||||
|
845000: '95e93bb60be8d5f07a1f4d26290c914957a82fc9d26ae8a3f20082eda27406ff',
|
||||||
|
846000: 'f1bdc39af7705e58ab8b6c31dc70dce1e115db1cfd8cc9b037949dfbec82a59a',
|
||||||
|
847000: 'f5f10f06396ecf2765d8a081141d489737c1d8d57c281f28f57c4cb2f90db883',
|
||||||
|
848000: '331b8ef08605bae8d749893af9ed54f0df4f07a5a002108a2a0aea82d0360979',
|
||||||
|
849000: '75b5f6233ab9a1bbc3c8b2893e5b22a0aa98e7ea635261255dc3c281f67d2260',
|
||||||
|
850000: '5d7e6fe83e0ea1910a54a00090704737671d6f44df4228e21440ad1fc15e595f',
|
||||||
|
851000: '7822db25d3ff0f6695ee38bad91edf317b5c6611673d28f1d22053110bb558be',
|
||||||
|
852000: '2f0effad83a3561fc1a2806a562786a641d9ddb18d16bb9308006e7d324a21e9',
|
||||||
|
853000: 'f603b2eaff11d5296377d990651317d40a1b2599ad2c5250eab131090f4b9458',
|
||||||
|
854000: '34d59b26a50f18a9f250736d0f2e69d28b7e196fbef9b8a26c6b0b75c16aa194',
|
||||||
|
855000: '76dd1ffff3946c0878969886fcf177ce5ab5560df19ddf006f9bcb02ae3e4e4f',
|
||||||
|
856000: '74ff0b6f64e9dd5802fec2aac1d3ae194d28b9264114adaf0a882b46c8c918fe',
|
||||||
|
857000: '7b5badfa2e4f40aa597a504d7ebe83c3705a2c6169a8c168ce293db223bc2d32',
|
||||||
|
858000: '2bb0767a0f72b20d45ecfc3e34517dbda16d85758e040cf0e147f4cbd0cc57ac',
|
||||||
|
859000: '3d741b9c365a91ed76f85824b94d19ec19b608d232660840ba59c7aa4b2cb67f',
|
||||||
|
860000: 'd481a5a117878c0e3acd1f5844e150fb30e617577947d9846b1d214d703b71b0',
|
||||||
|
861000: '54033424e488a3f1ad6946d4a6d9acb48465d6b1dbe8e1c2504a54cc84d7cad4',
|
||||||
|
862000: '464bc3820a8cc8844dc9e26c388009e9982c656d46ef4b4fd0a2cb0e4eea0aaa',
|
||||||
|
863000: 'd1aa94be2174f66780c4f226b9da3f6712b0f37af8dec33360bea83ca261b342',
|
||||||
|
864000: '8c16008f11de5bc395d88cd802514ff647450f1bc136724b9aaf2ccce10a494f',
|
||||||
|
865000: '3dae86012e97a201e2e1a47c899001ac00f78dc108026ed7c4194858c6c6dd5a',
|
||||||
|
866000: 'afe5b0ccab995e1a1fa25fbc24c1d4b1a92c43042d03395f8743dcd806e72fd8',
|
||||||
|
867000: 'c83716ac171aa9ab0d414833db340fa30e82bfda6cc616d3038529caab9b5600',
|
||||||
|
868000: '8c409fe03cd35ef2d8e366818788b40eaeb4c8f6ae91450d75f4a66ca5f69cad',
|
||||||
|
869000: '1d47909ceba790b8e1ce2e9902ee2775ea99e58efdb95668f9803a8ccf95f286',
|
||||||
|
870000: '9adf5da1476388f053aa42de636da169d1cf1c9652cdf7cd9ad4fb18a0eb3388',
|
||||||
|
871000: '8ad57fb1e74bcba0b5614fbac003be2bb32275dd85b38f2d28a0585005a99cfc',
|
||||||
|
872000: '84a32e92012a356106e9657da8dab1a5491ea588fc29d411c69b20680c666420',
|
||||||
|
873000: 'adf5921bbbfaa43929f67e6a070975313b77b456e262c700a27be611fceb17ae',
|
||||||
|
874000: '09eaa7c4b18c79a46a2895190333f72336826223d5c986849a06f5153f49f2a5',
|
||||||
|
875000: '235d7e4f31966507312149ea4c5e294aa84c695cf840117f0ef5963be7a0bda1',
|
||||||
|
876000: '9aa9cb806ccbec0475ac330b496c5b2edeba38ba3f1e13ddd54a01457634a288',
|
||||||
|
877000: 'c1e7f9b2b20bb1c4c0deadbc786d31fdf36f262325342aa23d1a66e2846b22bc',
|
||||||
|
878000: 'ee0d2b20ac28ce23ab38698a57c6beff14f12b7af9d027c05cc92f652695f46b',
|
||||||
|
879000: '0eb0810f4b81d1845b0a88f05449408df2e45715c9210a656f45278c5fdf7956',
|
||||||
|
880000: 'e7d613027e3b4ca38d09bbef07998b57db237c6d67f1e8ea50024d2e0d9a1a72',
|
||||||
|
881000: '21af4d355d8756b8bf0369b2d79b5c824148ae069026ba5c14f9dd6b7555e1db',
|
||||||
|
}
|
||||||
@@ -5,7 +5,7 @@ from lbry.wallet.transaction import OutputEffectiveAmountEstimator
|
|||||||
|
|
||||||
MAXIMUM_TRIES = 100000
|
MAXIMUM_TRIES = 100000
|
||||||
|
|
||||||
STRATEGIES = []
|
STRATEGIES = ['sqlite'] # sqlite coin chooser is in database.py
|
||||||
|
|
||||||
|
|
||||||
def strategy(method):
|
def strategy(method):
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ COIN = 100*CENT
|
|||||||
TIMEOUT = 30.0
|
TIMEOUT = 30.0
|
||||||
|
|
||||||
TXO_TYPES = {
|
TXO_TYPES = {
|
||||||
|
"other": 0,
|
||||||
"stream": 1,
|
"stream": 1,
|
||||||
"channel": 2,
|
"channel": 2,
|
||||||
"support": 3,
|
"support": 3,
|
||||||
@@ -14,9 +15,13 @@ TXO_TYPES = {
|
|||||||
"repost": 6,
|
"repost": 6,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
CLAIM_TYPE_NAMES = [
|
||||||
|
'stream',
|
||||||
|
'channel',
|
||||||
|
'collection',
|
||||||
|
'repost',
|
||||||
|
]
|
||||||
|
|
||||||
CLAIM_TYPES = [
|
CLAIM_TYPES = [
|
||||||
TXO_TYPES['stream'],
|
TXO_TYPES[name] for name in CLAIM_TYPE_NAMES
|
||||||
TXO_TYPES['channel'],
|
|
||||||
TXO_TYPES['collection'],
|
|
||||||
TXO_TYPES['repost'],
|
|
||||||
]
|
]
|
||||||
+709
-183
File diff suppressed because it is too large.
Load diff
+138
-68
@@ -1,16 +1,18 @@
|
|||||||
|
import base64
|
||||||
import os
|
import os
|
||||||
import struct
|
import struct
|
||||||
import asyncio
|
import asyncio
|
||||||
import hashlib
|
|
||||||
import logging
|
import logging
|
||||||
|
import zlib
|
||||||
|
from datetime import date
|
||||||
|
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
from contextlib import asynccontextmanager
|
from typing import Optional, Iterator, Tuple, Callable
|
||||||
from typing import Optional, Iterator, Tuple
|
|
||||||
from binascii import hexlify, unhexlify
|
from binascii import hexlify, unhexlify
|
||||||
|
|
||||||
from lbry.crypto.hash import sha512, double_sha256, ripemd160
|
from lbry.crypto.hash import sha512, double_sha256, ripemd160
|
||||||
from lbry.wallet.util import ArithUint256
|
from lbry.wallet.util import ArithUint256, date_to_julian_day
|
||||||
|
from .checkpoints import HASHES
|
||||||
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
@@ -32,25 +34,50 @@ class Headers:
|
|||||||
max_target = 0x0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
|
max_target = 0x0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
|
||||||
genesis_hash = b'9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463'
|
genesis_hash = b'9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463'
|
||||||
target_timespan = 150
|
target_timespan = 150
|
||||||
checkpoint = (600_000, b'100b33ca3d0b86a48f0d6d6f30458a130ecb89d5affefe4afccb134d5a40f4c2')
|
checkpoints = HASHES
|
||||||
|
first_block_timestamp = 1466646588 # block 1, as 0 is off by a lot
|
||||||
|
timestamp_average_offset = 160.6855883050695 # calculated at 733447
|
||||||
|
|
||||||
validate_difficulty: bool = True
|
validate_difficulty: bool = True
|
||||||
|
|
||||||
def __init__(self, path) -> None:
|
def __init__(self, path) -> None:
|
||||||
if path == ':memory:':
|
self.io = None
|
||||||
self.io = BytesIO()
|
|
||||||
self.path = path
|
self.path = path
|
||||||
self._size: Optional[int] = None
|
self._size: Optional[int] = None
|
||||||
|
self.chunk_getter: Optional[Callable] = None
|
||||||
|
self.known_missing_checkpointed_chunks = set()
|
||||||
|
self.check_chunk_lock = asyncio.Lock()
|
||||||
|
|
||||||
async def open(self):
|
async def open(self):
|
||||||
|
self.io = BytesIO()
|
||||||
if self.path != ':memory:':
|
if self.path != ':memory:':
|
||||||
if not os.path.exists(self.path):
|
def _readit():
|
||||||
self.io = open(self.path, 'w+b')
|
if os.path.exists(self.path):
|
||||||
else:
|
with open(self.path, 'r+b') as header_file:
|
||||||
self.io = open(self.path, 'r+b')
|
self.io.seek(0)
|
||||||
|
self.io.write(header_file.read())
|
||||||
|
await asyncio.get_event_loop().run_in_executor(None, _readit)
|
||||||
|
bytes_size = self.io.seek(0, os.SEEK_END)
|
||||||
|
self._size = bytes_size // self.header_size
|
||||||
|
max_checkpointed_height = max(self.checkpoints.keys() or [-1]) + 1000
|
||||||
|
if bytes_size % self.header_size:
|
||||||
|
log.warning("Reader file size doesnt match header size. Repairing, might take a while.")
|
||||||
|
await self.repair()
|
||||||
|
else:
|
||||||
|
# try repairing any incomplete write on tip from previous runs (outside of checkpoints, that are ok)
|
||||||
|
await self.repair(start_height=max_checkpointed_height)
|
||||||
|
await self.ensure_checkpointed_size()
|
||||||
|
await self.get_all_missing_headers()
|
||||||
|
|
||||||
async def close(self):
|
async def close(self):
|
||||||
self.io.close()
|
if self.io is not None:
|
||||||
|
def _close():
|
||||||
|
flags = 'r+b' if os.path.exists(self.path) else 'w+b'
|
||||||
|
with open(self.path, flags) as header_file:
|
||||||
|
header_file.write(self.io.getbuffer())
|
||||||
|
await asyncio.get_event_loop().run_in_executor(None, _close)
|
||||||
|
self.io.close()
|
||||||
|
self.io = None
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def serialize(header):
|
def serialize(header):
|
||||||
@@ -97,23 +124,92 @@ class Headers:
|
|||||||
return new_target
|
return new_target
|
||||||
|
|
||||||
def __len__(self) -> int:
|
def __len__(self) -> int:
|
||||||
if self._size is None:
|
|
||||||
self._size = self.io.seek(0, os.SEEK_END) // self.header_size
|
|
||||||
return self._size
|
return self._size
|
||||||
|
|
||||||
def __bool__(self):
|
def __bool__(self):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def __getitem__(self, height) -> dict:
|
async def get(self, height) -> dict:
|
||||||
if isinstance(height, slice):
|
if isinstance(height, slice):
|
||||||
raise NotImplementedError("Slicing of header chain has not been implemented yet.")
|
raise NotImplementedError("Slicing of header chain has not been implemented yet.")
|
||||||
|
try:
|
||||||
|
return self.deserialize(height, await self.get_raw_header(height))
|
||||||
|
except struct.error:
|
||||||
|
raise IndexError(f"failed to get {height}, at {len(self)}")
|
||||||
|
|
||||||
|
def estimated_timestamp(self, height, try_real_headers=True):
|
||||||
|
if height <= 0:
|
||||||
|
return
|
||||||
|
if try_real_headers and self.has_header(height):
|
||||||
|
offset = height * self.header_size
|
||||||
|
return struct.unpack('<I', self.io.getbuffer()[offset + 100: offset + 104])[0]
|
||||||
|
return int(self.first_block_timestamp + (height * self.timestamp_average_offset))
|
||||||
|
|
||||||
|
def estimated_julian_day(self, height):
|
||||||
|
return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height, False)))
|
||||||
|
|
||||||
|
async def get_raw_header(self, height) -> bytes:
|
||||||
|
if self.chunk_getter:
|
||||||
|
await self.ensure_chunk_at(height)
|
||||||
if not 0 <= height <= self.height:
|
if not 0 <= height <= self.height:
|
||||||
raise IndexError(f"{height} is out of bounds, current height: {self.height}")
|
raise IndexError(f"{height} is out of bounds, current height: {self.height}")
|
||||||
return self.deserialize(height, self.get_raw_header(height))
|
return self._read(height)
|
||||||
|
|
||||||
def get_raw_header(self, height) -> bytes:
|
def _read(self, height, count=1):
|
||||||
self.io.seek(height * self.header_size, os.SEEK_SET)
|
offset = height * self.header_size
|
||||||
return self.io.read(self.header_size)
|
return bytes(self.io.getbuffer()[offset: offset + self.header_size * count])
|
||||||
|
|
||||||
|
def chunk_hash(self, start, count):
|
||||||
|
return self.hash_header(self._read(start, count)).decode()
|
||||||
|
|
||||||
|
async def ensure_checkpointed_size(self):
|
||||||
|
max_checkpointed_height = max(self.checkpoints.keys() or [-1])
|
||||||
|
if self.height < max_checkpointed_height:
|
||||||
|
self._write(max_checkpointed_height, bytes([0] * self.header_size * 1000))
|
||||||
|
|
||||||
|
async def ensure_chunk_at(self, height):
|
||||||
|
async with self.check_chunk_lock:
|
||||||
|
if self.has_header(height):
|
||||||
|
log.debug("has header %s", height)
|
||||||
|
return
|
||||||
|
return await self.fetch_chunk(height)
|
||||||
|
|
||||||
|
async def fetch_chunk(self, height):
|
||||||
|
log.info("on-demand fetching height %s", height)
|
||||||
|
start = (height // 1000) * 1000
|
||||||
|
headers = await self.chunk_getter(start) # pylint: disable=not-callable
|
||||||
|
chunk = (
|
||||||
|
zlib.decompress(base64.b64decode(headers['base64']), wbits=-15, bufsize=600_000)
|
||||||
|
)
|
||||||
|
chunk_hash = self.hash_header(chunk).decode()
|
||||||
|
if self.checkpoints.get(start) == chunk_hash:
|
||||||
|
self._write(start, chunk)
|
||||||
|
if start in self.known_missing_checkpointed_chunks:
|
||||||
|
self.known_missing_checkpointed_chunks.remove(start)
|
||||||
|
return
|
||||||
|
elif start not in self.checkpoints:
|
||||||
|
return # todo: fixme
|
||||||
|
raise Exception(
|
||||||
|
f"Checkpoint mismatch at height {start}. Expected {self.checkpoints[start]}, but got {chunk_hash} instead."
|
||||||
|
)
|
||||||
|
|
||||||
|
def has_header(self, height):
|
||||||
|
normalized_height = (height // 1000) * 1000
|
||||||
|
if normalized_height in self.checkpoints:
|
||||||
|
return normalized_height not in self.known_missing_checkpointed_chunks
|
||||||
|
|
||||||
|
empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d'
|
||||||
|
all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b'
|
||||||
|
return self.chunk_hash(height, 1) not in (empty, all_zeroes)
|
||||||
|
|
||||||
|
async def get_all_missing_headers(self):
|
||||||
|
# Heavy operation done in one optimized shot
|
||||||
|
for chunk_height, expected_hash in reversed(list(self.checkpoints.items())):
|
||||||
|
if chunk_height in self.known_missing_checkpointed_chunks:
|
||||||
|
continue
|
||||||
|
if self.chunk_hash(chunk_height, 1000) != expected_hash:
|
||||||
|
self.known_missing_checkpointed_chunks.add(chunk_height)
|
||||||
|
return self.known_missing_checkpointed_chunks
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def height(self) -> int:
|
def height(self) -> int:
|
||||||
@@ -123,9 +219,9 @@ class Headers:
|
|||||||
def bytes_size(self):
|
def bytes_size(self):
|
||||||
return len(self) * self.header_size
|
return len(self) * self.header_size
|
||||||
|
|
||||||
def hash(self, height=None) -> bytes:
|
async def hash(self, height=None) -> bytes:
|
||||||
return self.hash_header(
|
return self.hash_header(
|
||||||
self.get_raw_header(height if height is not None else self.height)
|
await self.get_raw_header(height if height is not None else self.height)
|
||||||
)
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -134,44 +230,18 @@ class Headers:
|
|||||||
return b'0' * 64
|
return b'0' * 64
|
||||||
return hexlify(double_sha256(header)[::-1])
|
return hexlify(double_sha256(header)[::-1])
|
||||||
|
|
||||||
@asynccontextmanager
|
|
||||||
async def checkpointed_connector(self):
|
|
||||||
buf = BytesIO()
|
|
||||||
try:
|
|
||||||
yield buf
|
|
||||||
finally:
|
|
||||||
await asyncio.sleep(0)
|
|
||||||
final_height = len(self) + buf.tell() // self.header_size
|
|
||||||
verifiable_bytes = (self.checkpoint[0] - len(self)) * self.header_size if self.checkpoint else 0
|
|
||||||
if verifiable_bytes > 0 and final_height >= self.checkpoint[0]:
|
|
||||||
buf.seek(0)
|
|
||||||
self.io.seek(0)
|
|
||||||
h = hashlib.sha256()
|
|
||||||
h.update(self.io.read())
|
|
||||||
h.update(buf.read(verifiable_bytes))
|
|
||||||
if h.hexdigest().encode() == self.checkpoint[1]:
|
|
||||||
buf.seek(0)
|
|
||||||
self._write(len(self), buf.read(verifiable_bytes))
|
|
||||||
remaining = buf.read()
|
|
||||||
buf.seek(0)
|
|
||||||
buf.write(remaining)
|
|
||||||
buf.truncate()
|
|
||||||
else:
|
|
||||||
log.warning("Checkpoint mismatch, connecting headers through slow method.")
|
|
||||||
if buf.tell() > 0:
|
|
||||||
await self.connect(len(self), buf.getvalue())
|
|
||||||
|
|
||||||
async def connect(self, start: int, headers: bytes) -> int:
|
async def connect(self, start: int, headers: bytes) -> int:
|
||||||
added = 0
|
added = 0
|
||||||
bail = False
|
bail = False
|
||||||
for height, chunk in self._iterate_chunks(start, headers):
|
for height, chunk in self._iterate_chunks(start, headers):
|
||||||
try:
|
try:
|
||||||
# validate_chunk() is CPU bound and reads previous chunks from file system
|
# validate_chunk() is CPU bound and reads previous chunks from file system
|
||||||
self.validate_chunk(height, chunk)
|
await self.validate_chunk(height, chunk)
|
||||||
except InvalidHeader as e:
|
except InvalidHeader as e:
|
||||||
bail = True
|
bail = True
|
||||||
chunk = chunk[:(height-e.height)*self.header_size]
|
chunk = chunk[:(height-e.height)*self.header_size]
|
||||||
added += self._write(height, chunk) if chunk else 0
|
if chunk:
|
||||||
|
added += self._write(height, chunk)
|
||||||
if bail:
|
if bail:
|
||||||
break
|
break
|
||||||
return added
|
return added
|
||||||
@@ -179,20 +249,21 @@ class Headers:
|
|||||||
def _write(self, height, verified_chunk):
|
def _write(self, height, verified_chunk):
|
||||||
self.io.seek(height * self.header_size, os.SEEK_SET)
|
self.io.seek(height * self.header_size, os.SEEK_SET)
|
||||||
written = self.io.write(verified_chunk) // self.header_size
|
written = self.io.write(verified_chunk) // self.header_size
|
||||||
self.io.truncate()
|
# self.io.truncate()
|
||||||
# .seek()/.write()/.truncate() might also .flush() when needed
|
# .seek()/.write()/.truncate() might also .flush() when needed
|
||||||
# the goal here is mainly to ensure we're definitely flush()'ing
|
# the goal here is mainly to ensure we're definitely flush()'ing
|
||||||
self.io.flush()
|
self.io.flush()
|
||||||
self._size = self.io.tell() // self.header_size
|
self._size = max(self._size or 0, self.io.tell() // self.header_size)
|
||||||
return written
|
return written
|
||||||
|
|
||||||
def validate_chunk(self, height, chunk):
|
async def validate_chunk(self, height, chunk):
|
||||||
previous_hash, previous_header, previous_previous_header = None, None, None
|
previous_hash, previous_header, previous_previous_header = None, None, None
|
||||||
if height > 0:
|
if height > 0:
|
||||||
previous_header = self[height-1]
|
raw = await self.get_raw_header(height-1)
|
||||||
previous_hash = self.hash(height-1)
|
previous_header = self.deserialize(height-1, raw)
|
||||||
|
previous_hash = self.hash_header(raw)
|
||||||
if height > 1:
|
if height > 1:
|
||||||
previous_previous_header = self[height-2]
|
previous_previous_header = await self.get(height-2)
|
||||||
chunk_target = self.get_next_chunk_target(height // 2016 - 1)
|
chunk_target = self.get_next_chunk_target(height // 2016 - 1)
|
||||||
for current_hash, current_header in self._iterate_headers(height, chunk):
|
for current_hash, current_header in self._iterate_headers(height, chunk):
|
||||||
block_target = self.get_next_block_target(chunk_target, previous_previous_header, previous_header)
|
block_target = self.get_next_block_target(chunk_target, previous_previous_header, previous_header)
|
||||||
@@ -231,28 +302,30 @@ class Headers:
|
|||||||
height, f"insufficient proof of work: {proof_of_work.value} vs target {target.value}"
|
height, f"insufficient proof of work: {proof_of_work.value} vs target {target.value}"
|
||||||
)
|
)
|
||||||
|
|
||||||
async def repair(self):
|
async def repair(self, start_height=0):
|
||||||
previous_header_hash = fail = None
|
previous_header_hash = fail = None
|
||||||
batch_size = 36
|
batch_size = 36
|
||||||
for start_height in range(0, self.height, batch_size):
|
for height in range(start_height, self.height, batch_size):
|
||||||
self.io.seek(self.header_size * start_height)
|
headers = self._read(height, batch_size)
|
||||||
headers = self.io.read(self.header_size*batch_size)
|
|
||||||
if len(headers) % self.header_size != 0:
|
if len(headers) % self.header_size != 0:
|
||||||
headers = headers[:(len(headers) // self.header_size) * self.header_size]
|
headers = headers[:(len(headers) // self.header_size) * self.header_size]
|
||||||
for header_hash, header in self._iterate_headers(start_height, headers):
|
for header_hash, header in self._iterate_headers(height, headers):
|
||||||
height = header['block_height']
|
height = header['block_height']
|
||||||
if height:
|
if previous_header_hash:
|
||||||
if header['prev_block_hash'] != previous_header_hash:
|
if header['prev_block_hash'] != previous_header_hash:
|
||||||
fail = True
|
fail = True
|
||||||
else:
|
elif height == 0:
|
||||||
if header_hash != self.genesis_hash:
|
if header_hash != self.genesis_hash:
|
||||||
fail = True
|
fail = True
|
||||||
|
else:
|
||||||
|
# for sanity and clarity, since it is the only way we can end up here
|
||||||
|
assert start_height > 0 and height == start_height
|
||||||
if fail:
|
if fail:
|
||||||
log.warning("Header file corrupted at height %s, truncating it.", height - 1)
|
log.warning("Header file corrupted at height %s, truncating it.", height - 1)
|
||||||
self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET)
|
self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET)
|
||||||
self.io.truncate()
|
self.io.truncate()
|
||||||
self.io.flush()
|
self.io.flush()
|
||||||
self._size = None
|
self._size = self.io.seek(0, os.SEEK_END) // self.header_size
|
||||||
return
|
return
|
||||||
previous_header_hash = header_hash
|
previous_header_hash = header_hash
|
||||||
|
|
||||||
@@ -276,10 +349,6 @@ class Headers:
|
|||||||
header = headers[start:end]
|
header = headers[start:end]
|
||||||
yield self.hash_header(header), self.deserialize(height+idx, header)
|
yield self.hash_header(header), self.deserialize(height+idx, header)
|
||||||
|
|
||||||
@property
|
|
||||||
def claim_trie_root(self):
|
|
||||||
return self[self.height]['claim_trie_root']
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def header_hash_to_pow_hash(header_hash: bytes):
|
def header_hash_to_pow_hash(header_hash: bytes):
|
||||||
header_hash_bytes = unhexlify(header_hash)[::-1]
|
header_hash_bytes = unhexlify(header_hash)[::-1]
|
||||||
@@ -295,3 +364,4 @@ class UnvalidatedHeaders(Headers):
|
|||||||
validate_difficulty = False
|
validate_difficulty = False
|
||||||
max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
|
max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
|
||||||
genesis_hash = b'6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556'
|
genesis_hash = b'6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556'
|
||||||
|
checkpoints = {}
|
||||||
+444
-186
@@ -1,21 +1,20 @@
|
|||||||
import os
|
import os
|
||||||
import zlib
|
import copy
|
||||||
import base64
|
import time
|
||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
from io import StringIO
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from operator import itemgetter
|
from operator import itemgetter
|
||||||
from collections import namedtuple, defaultdict
|
from collections import defaultdict
|
||||||
from binascii import hexlify, unhexlify
|
from binascii import hexlify, unhexlify
|
||||||
from typing import Dict, Tuple, Type, Iterable, List, Optional, DefaultDict
|
from typing import Dict, Tuple, Type, Iterable, List, Optional, DefaultDict, NamedTuple
|
||||||
|
|
||||||
import pylru
|
|
||||||
from lbry.schema.result import Outputs, INVALID, NOT_FOUND
|
from lbry.schema.result import Outputs, INVALID, NOT_FOUND
|
||||||
from lbry.schema.url import URL
|
from lbry.schema.url import URL
|
||||||
from lbry.crypto.hash import hash160, double_sha256, sha256
|
from lbry.crypto.hash import hash160, double_sha256, sha256
|
||||||
from lbry.crypto.base58 import Base58
|
from lbry.crypto.base58 import Base58
|
||||||
|
from lbry.utils import LRUCacheWithMetrics
|
||||||
|
|
||||||
from .tasks import TaskGroup
|
from .tasks import TaskGroup
|
||||||
from .database import Database
|
from .database import Database
|
||||||
@@ -25,11 +24,11 @@ from .account import Account, AddressManager, SingleKey
|
|||||||
from .network import Network
|
from .network import Network
|
||||||
from .transaction import Transaction, Output
|
from .transaction import Transaction, Output
|
||||||
from .header import Headers, UnvalidatedHeaders
|
from .header import Headers, UnvalidatedHeaders
|
||||||
from .constants import TXO_TYPES, COIN, NULL_HASH32
|
from .checkpoints import HASHES
|
||||||
|
from .constants import TXO_TYPES, CLAIM_TYPES, COIN, NULL_HASH32
|
||||||
from .bip32 import PubKey, PrivateKey
|
from .bip32 import PubKey, PrivateKey
|
||||||
from .coinselection import CoinSelector
|
from .coinselection import CoinSelector
|
||||||
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
LedgerType = Type['BaseLedger']
|
LedgerType = Type['BaseLedger']
|
||||||
@@ -53,25 +52,29 @@ class LedgerRegistry(type):
|
|||||||
return mcs.ledgers[ledger_id]
|
return mcs.ledgers[ledger_id]
|
||||||
|
|
||||||
|
|
||||||
class TransactionEvent(namedtuple('TransactionEvent', ('address', 'tx'))):
|
class TransactionEvent(NamedTuple):
|
||||||
pass
|
address: str
|
||||||
|
tx: Transaction
|
||||||
|
|
||||||
|
|
||||||
class AddressesGeneratedEvent(namedtuple('AddressesGeneratedEvent', ('address_manager', 'addresses'))):
|
class AddressesGeneratedEvent(NamedTuple):
|
||||||
pass
|
address_manager: AddressManager
|
||||||
|
addresses: List[str]
|
||||||
|
|
||||||
|
|
||||||
class BlockHeightEvent(namedtuple('BlockHeightEvent', ('height', 'change'))):
|
class BlockHeightEvent(NamedTuple):
|
||||||
pass
|
height: int
|
||||||
|
change: int
|
||||||
|
|
||||||
|
|
||||||
class TransactionCacheItem:
|
class TransactionCacheItem:
|
||||||
__slots__ = '_tx', 'lock', 'has_tx'
|
__slots__ = '_tx', 'lock', 'has_tx', 'pending_verifications'
|
||||||
|
|
||||||
def __init__(self, tx: Optional[Transaction] = None, lock: Optional[asyncio.Lock] = None):
|
def __init__(self, tx: Optional[Transaction] = None, lock: Optional[asyncio.Lock] = None):
|
||||||
self.has_tx = asyncio.Event()
|
self.has_tx = asyncio.Event()
|
||||||
self.lock = lock or asyncio.Lock()
|
self.lock = lock or asyncio.Lock()
|
||||||
self._tx = self.tx = tx
|
self._tx = self.tx = tx
|
||||||
|
self.pending_verifications = 0
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def tx(self) -> Optional[Transaction]:
|
def tx(self) -> Optional[Transaction]:
|
||||||
@@ -105,6 +108,8 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
default_fee_per_byte = 50
|
default_fee_per_byte = 50
|
||||||
default_fee_per_name_char = 200000
|
default_fee_per_name_char = 200000
|
||||||
|
|
||||||
|
checkpoints = HASHES
|
||||||
|
|
||||||
def __init__(self, config=None):
|
def __init__(self, config=None):
|
||||||
self.config = config or {}
|
self.config = config or {}
|
||||||
self.db: Database = self.config.get('db') or Database(
|
self.db: Database = self.config.get('db') or Database(
|
||||||
@@ -114,10 +119,10 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
self.headers: Headers = self.config.get('headers') or self.headers_class(
|
self.headers: Headers = self.config.get('headers') or self.headers_class(
|
||||||
os.path.join(self.path, "headers")
|
os.path.join(self.path, "headers")
|
||||||
)
|
)
|
||||||
|
self.headers.checkpoints = self.checkpoints
|
||||||
self.network: Network = self.config.get('network') or Network(self)
|
self.network: Network = self.config.get('network') or Network(self)
|
||||||
self.network.on_header.listen(self.receive_header)
|
self.network.on_header.listen(self.receive_header)
|
||||||
self.network.on_status.listen(self.process_status_update)
|
self.network.on_status.listen(self.process_status_update)
|
||||||
self.network.on_connected.listen(self.join_network)
|
|
||||||
|
|
||||||
self.accounts = []
|
self.accounts = []
|
||||||
self.fee_per_byte: int = self.config.get('fee_per_byte', self.default_fee_per_byte)
|
self.fee_per_byte: int = self.config.get('fee_per_byte', self.default_fee_per_byte)
|
||||||
@@ -150,17 +155,19 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
self._on_ready_controller = StreamController()
|
self._on_ready_controller = StreamController()
|
||||||
self.on_ready = self._on_ready_controller.stream
|
self.on_ready = self._on_ready_controller.stream
|
||||||
|
|
||||||
self._tx_cache = pylru.lrucache(100000)
|
self._tx_cache = LRUCacheWithMetrics(self.config.get("tx_cache_size", 1024), metric_name='tx')
|
||||||
self._update_tasks = TaskGroup()
|
self._update_tasks = TaskGroup()
|
||||||
|
self._other_tasks = TaskGroup() # that we dont need to start
|
||||||
self._utxo_reservation_lock = asyncio.Lock()
|
self._utxo_reservation_lock = asyncio.Lock()
|
||||||
self._header_processing_lock = asyncio.Lock()
|
self._header_processing_lock = asyncio.Lock()
|
||||||
self._address_update_locks: DefaultDict[str, asyncio.Lock] = defaultdict(asyncio.Lock)
|
self._address_update_locks: DefaultDict[str, asyncio.Lock] = defaultdict(asyncio.Lock)
|
||||||
|
self._history_lock = asyncio.Lock()
|
||||||
|
|
||||||
self.coin_selection_strategy = None
|
self.coin_selection_strategy = None
|
||||||
self._known_addresses_out_of_sync = set()
|
self._known_addresses_out_of_sync = set()
|
||||||
|
|
||||||
self.fee_per_name_char = self.config.get('fee_per_name_char', self.default_fee_per_name_char)
|
self.fee_per_name_char = self.config.get('fee_per_name_char', self.default_fee_per_name_char)
|
||||||
self._balance_cache = pylru.lrucache(100000)
|
self._balance_cache = LRUCacheWithMetrics(2 ** 15)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_id(cls):
|
def get_id(cls):
|
||||||
@@ -171,15 +178,25 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
raw_address = cls.pubkey_address_prefix + h160
|
raw_address = cls.pubkey_address_prefix + h160
|
||||||
return Base58.encode(bytearray(raw_address + double_sha256(raw_address)[0:4]))
|
return Base58.encode(bytearray(raw_address + double_sha256(raw_address)[0:4]))
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def hash160_to_script_address(cls, h160):
|
||||||
|
raw_address = cls.script_address_prefix + h160
|
||||||
|
return Base58.encode(bytearray(raw_address + double_sha256(raw_address)[0:4]))
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def address_to_hash160(address):
|
def address_to_hash160(address):
|
||||||
return Base58.decode(address)[1:21]
|
return Base58.decode(address)[1:21]
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def is_valid_address(cls, address):
|
def is_pubkey_address(cls, address):
|
||||||
decoded = Base58.decode_check(address)
|
decoded = Base58.decode_check(address)
|
||||||
return decoded[0] == cls.pubkey_address_prefix[0]
|
return decoded[0] == cls.pubkey_address_prefix[0]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def is_script_address(cls, address):
|
||||||
|
decoded = Base58.decode_check(address)
|
||||||
|
return decoded[0] == cls.script_address_prefix[0]
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def public_key_to_address(cls, public_key):
|
def public_key_to_address(cls, public_key):
|
||||||
return cls.hash160_to_address(hash160(public_key))
|
return cls.hash160_to_address(hash160(public_key))
|
||||||
@@ -224,7 +241,7 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
async def get_effective_amount_estimators(self, funding_accounts: Iterable[Account]):
|
async def get_effective_amount_estimators(self, funding_accounts: Iterable[Account]):
|
||||||
estimators = []
|
estimators = []
|
||||||
for account in funding_accounts:
|
for account in funding_accounts:
|
||||||
utxos = await account.get_utxos()
|
utxos = await account.get_utxos(no_tx=True, no_channel_info=True)
|
||||||
for utxo in utxos:
|
for utxo in utxos:
|
||||||
estimators.append(utxo.get_estimator(self))
|
estimators.append(utxo.get_estimator(self))
|
||||||
return estimators
|
return estimators
|
||||||
@@ -235,11 +252,15 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
def get_address_count(self, **constraints):
|
def get_address_count(self, **constraints):
|
||||||
return self.db.get_address_count(**constraints)
|
return self.db.get_address_count(**constraints)
|
||||||
|
|
||||||
async def get_spendable_utxos(self, amount: int, funding_accounts):
|
async def get_spendable_utxos(self, amount: int, funding_accounts: Optional[Iterable['Account']], min_amount=1):
|
||||||
|
min_amount = min(amount // 10, min_amount)
|
||||||
|
fee = Output.pay_pubkey_hash(COIN, NULL_HASH32).get_fee(self)
|
||||||
|
selector = CoinSelector(amount, fee)
|
||||||
async with self._utxo_reservation_lock:
|
async with self._utxo_reservation_lock:
|
||||||
|
if self.coin_selection_strategy == 'sqlite':
|
||||||
|
return await self.db.get_spendable_utxos(self, amount + fee, funding_accounts, min_amount=min_amount,
|
||||||
|
fee_per_byte=self.fee_per_byte)
|
||||||
txos = await self.get_effective_amount_estimators(funding_accounts)
|
txos = await self.get_effective_amount_estimators(funding_accounts)
|
||||||
fee = Output.pay_pubkey_hash(COIN, NULL_HASH32).get_fee(self)
|
|
||||||
selector = CoinSelector(amount, fee)
|
|
||||||
spendables = selector.select(txos, self.coin_selection_strategy)
|
spendables = selector.select(txos, self.coin_selection_strategy)
|
||||||
if spendables:
|
if spendables:
|
||||||
await self.reserve_outputs(s.txo for s in spendables)
|
await self.reserve_outputs(s.txo for s in spendables)
|
||||||
@@ -262,6 +283,21 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
self.constraint_spending_utxos(constraints)
|
self.constraint_spending_utxos(constraints)
|
||||||
return self.db.get_utxo_count(**constraints)
|
return self.db.get_utxo_count(**constraints)
|
||||||
|
|
||||||
|
async def get_txos(self, resolve=False, **constraints) -> List[Output]:
|
||||||
|
txos = await self.db.get_txos(**constraints)
|
||||||
|
if resolve:
|
||||||
|
return await self._resolve_for_local_results(constraints.get('accounts', []), txos)
|
||||||
|
return txos
|
||||||
|
|
||||||
|
def get_txo_count(self, **constraints):
|
||||||
|
return self.db.get_txo_count(**constraints)
|
||||||
|
|
||||||
|
def get_txo_sum(self, **constraints):
|
||||||
|
return self.db.get_txo_sum(**constraints)
|
||||||
|
|
||||||
|
def get_txo_plot(self, **constraints):
|
||||||
|
return self.db.get_txo_plot(**constraints)
|
||||||
|
|
||||||
def get_transactions(self, **constraints):
|
def get_transactions(self, **constraints):
|
||||||
return self.db.get_transactions(**constraints)
|
return self.db.get_transactions(**constraints)
|
||||||
|
|
||||||
@@ -297,12 +333,15 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
self.db.open(),
|
self.db.open(),
|
||||||
self.headers.open()
|
self.headers.open()
|
||||||
])
|
])
|
||||||
first_connection = self.network.on_connected.first
|
fully_synced = self.on_ready.first
|
||||||
asyncio.ensure_future(self.network.start())
|
asyncio.create_task(self.network.start())
|
||||||
await first_connection
|
await self.network.on_connected.first
|
||||||
async with self._header_processing_lock:
|
async with self._header_processing_lock:
|
||||||
await self._update_tasks.add(self.initial_headers_sync())
|
await self._update_tasks.add(self.initial_headers_sync())
|
||||||
await self._on_ready_controller.stream.first
|
self.network.on_connected.listen(self.join_network)
|
||||||
|
asyncio.ensure_future(self.join_network())
|
||||||
|
await fully_synced
|
||||||
|
await self.db.release_all_outputs()
|
||||||
await asyncio.gather(*(a.maybe_migrate_certificates() for a in self.accounts))
|
await asyncio.gather(*(a.maybe_migrate_certificates() for a in self.accounts))
|
||||||
await asyncio.gather(*(a.save_max_gap() for a in self.accounts))
|
await asyncio.gather(*(a.save_max_gap() for a in self.accounts))
|
||||||
if len(self.accounts) > 10:
|
if len(self.accounts) > 10:
|
||||||
@@ -313,15 +352,15 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
|
|
||||||
async def join_network(self, *_):
|
async def join_network(self, *_):
|
||||||
log.info("Subscribing and updating accounts.")
|
log.info("Subscribing and updating accounts.")
|
||||||
async with self._header_processing_lock:
|
await self._update_tasks.add(self.subscribe_accounts())
|
||||||
await self.update_headers()
|
|
||||||
await self.subscribe_accounts()
|
|
||||||
await self._update_tasks.done.wait()
|
await self._update_tasks.done.wait()
|
||||||
self._on_ready_controller.add(True)
|
self._on_ready_controller.add(True)
|
||||||
|
|
||||||
async def stop(self):
|
async def stop(self):
|
||||||
self._update_tasks.cancel()
|
self._update_tasks.cancel()
|
||||||
|
self._other_tasks.cancel()
|
||||||
await self._update_tasks.done.wait()
|
await self._update_tasks.done.wait()
|
||||||
|
await self._other_tasks.done.wait()
|
||||||
await self.network.stop()
|
await self.network.stop()
|
||||||
await self.db.close()
|
await self.db.close()
|
||||||
await self.headers.close()
|
await self.headers.close()
|
||||||
@@ -331,19 +370,15 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
return max(self.headers.height, self._download_height)
|
return max(self.headers.height, self._download_height)
|
||||||
|
|
||||||
async def initial_headers_sync(self):
|
async def initial_headers_sync(self):
|
||||||
target = self.network.remote_height + 1
|
get_chunk = partial(self.network.retriable_call, self.network.get_headers, count=1000, b64=True)
|
||||||
current = len(self.headers)
|
self.headers.chunk_getter = get_chunk
|
||||||
get_chunk = partial(self.network.retriable_call, self.network.get_headers, count=4096, b64=True)
|
|
||||||
chunks = [asyncio.create_task(get_chunk(height)) for height in range(current, target, 4096)]
|
async def doit():
|
||||||
total = 0
|
for height in reversed(sorted(self.headers.known_missing_checkpointed_chunks)):
|
||||||
async with self.headers.checkpointed_connector() as buffer:
|
async with self._header_processing_lock:
|
||||||
for chunk in chunks:
|
await self.headers.ensure_chunk_at(height)
|
||||||
headers = await chunk
|
self._other_tasks.add(doit())
|
||||||
total += buffer.write(
|
await self.update_headers()
|
||||||
zlib.decompress(base64.b64decode(headers['base64']), wbits=-15, bufsize=600_000)
|
|
||||||
)
|
|
||||||
self._download_height = current + total // self.headers.header_size
|
|
||||||
log.info("Headers sync: %s / %s", self._download_height, target)
|
|
||||||
|
|
||||||
async def update_headers(self, height=None, headers=None, subscription_update=False):
|
async def update_headers(self, height=None, headers=None, subscription_update=False):
|
||||||
rewound = 0
|
rewound = 0
|
||||||
@@ -390,6 +425,7 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
"Blockchain Reorganization: attempting rewind to height %s from starting height %s",
|
"Blockchain Reorganization: attempting rewind to height %s from starting height %s",
|
||||||
height, height+rewound
|
height, height+rewound
|
||||||
)
|
)
|
||||||
|
self._tx_cache.clear()
|
||||||
|
|
||||||
else:
|
else:
|
||||||
raise IndexError(f"headers.connect() returned negative number ({added})")
|
raise IndexError(f"headers.connect() returned negative number ({added})")
|
||||||
@@ -454,21 +490,23 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
for address, remote_status in zip(batch, results):
|
for address, remote_status in zip(batch, results):
|
||||||
self._update_tasks.add(self.update_history(address, remote_status, address_manager))
|
self._update_tasks.add(self.update_history(address, remote_status, address_manager))
|
||||||
addresses_remaining = addresses_remaining[batch_size:]
|
addresses_remaining = addresses_remaining[batch_size:]
|
||||||
log.info("subscribed to %i/%i addresses on %s:%i", len(addresses) - len(addresses_remaining),
|
if self.network.client and self.network.client.server_address_and_port:
|
||||||
len(addresses), *self.network.client.server_address_and_port)
|
log.info("subscribed to %i/%i addresses on %s:%i", len(addresses) - len(addresses_remaining),
|
||||||
log.info(
|
len(addresses), *self.network.client.server_address_and_port)
|
||||||
"finished subscribing to %i addresses on %s:%i", len(addresses),
|
if self.network.client and self.network.client.server_address_and_port:
|
||||||
*self.network.client.server_address_and_port
|
log.info(
|
||||||
)
|
"finished subscribing to %i addresses on %s:%i", len(addresses),
|
||||||
|
*self.network.client.server_address_and_port
|
||||||
|
)
|
||||||
|
|
||||||
def process_status_update(self, update):
|
def process_status_update(self, update):
|
||||||
address, remote_status = update
|
address, remote_status = update
|
||||||
self._update_tasks.add(self.update_history(address, remote_status))
|
self._update_tasks.add(self.update_history(address, remote_status))
|
||||||
|
|
||||||
async def update_history(self, address, remote_status, address_manager: AddressManager = None):
|
async def update_history(self, address, remote_status, address_manager: AddressManager = None,
|
||||||
|
reattempt_update: bool = True):
|
||||||
async with self._address_update_locks[address]:
|
async with self._address_update_locks[address]:
|
||||||
self._known_addresses_out_of_sync.discard(address)
|
self._known_addresses_out_of_sync.discard(address)
|
||||||
|
|
||||||
local_status, local_history = await self.get_local_status_and_history(address)
|
local_status, local_history = await self.get_local_status_and_history(address)
|
||||||
|
|
||||||
if local_status == remote_status:
|
if local_status == remote_status:
|
||||||
@@ -478,57 +516,55 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
remote_history = list(map(itemgetter('tx_hash', 'height'), remote_history))
|
remote_history = list(map(itemgetter('tx_hash', 'height'), remote_history))
|
||||||
we_need = set(remote_history) - set(local_history)
|
we_need = set(remote_history) - set(local_history)
|
||||||
if not we_need:
|
if not we_need:
|
||||||
|
remote_missing = set(local_history) - set(remote_history)
|
||||||
|
if remote_missing:
|
||||||
|
log.warning(
|
||||||
|
"%i transactions we have for %s are not in the remote address history",
|
||||||
|
len(remote_missing), address
|
||||||
|
)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
cache_tasks: List[asyncio.Future[Transaction]] = []
|
to_request = {}
|
||||||
synced_history = StringIO()
|
pending_synced_history = {}
|
||||||
|
already_synced = set()
|
||||||
|
|
||||||
|
already_synced_offset = 0
|
||||||
for i, (txid, remote_height) in enumerate(remote_history):
|
for i, (txid, remote_height) in enumerate(remote_history):
|
||||||
if i < len(local_history) and local_history[i] == (txid, remote_height) and not cache_tasks:
|
if i == already_synced_offset and i < len(local_history) and local_history[i] == (txid, remote_height):
|
||||||
synced_history.write(f'{txid}:{remote_height}:')
|
pending_synced_history[i] = f'{txid}:{remote_height}:'
|
||||||
else:
|
already_synced.add((txid, remote_height))
|
||||||
check_local = (txid, remote_height) not in we_need
|
already_synced_offset += 1
|
||||||
cache_tasks.append(asyncio.ensure_future(
|
continue
|
||||||
self.cache_transaction(txid, remote_height, check_local=check_local)
|
|
||||||
))
|
|
||||||
|
|
||||||
synced_txs = []
|
tx_indexes = {}
|
||||||
for task in cache_tasks:
|
|
||||||
tx = await task
|
|
||||||
|
|
||||||
check_db_for_txos = []
|
for i, (txid, remote_height) in enumerate(remote_history):
|
||||||
for txi in tx.inputs:
|
tx_indexes[txid] = i
|
||||||
if txi.txo_ref.txo is not None:
|
if (txid, remote_height) in already_synced:
|
||||||
continue
|
continue
|
||||||
cache_item = self._tx_cache.get(txi.txo_ref.tx_ref.id)
|
to_request[i] = (txid, remote_height)
|
||||||
if cache_item is not None:
|
|
||||||
if cache_item.tx is None:
|
|
||||||
await cache_item.has_tx.wait()
|
|
||||||
assert cache_item.tx is not None
|
|
||||||
txi.txo_ref = cache_item.tx.outputs[txi.txo_ref.position].ref
|
|
||||||
else:
|
|
||||||
check_db_for_txos.append(txi.txo_ref.id)
|
|
||||||
|
|
||||||
referenced_txos = {} if not check_db_for_txos else {
|
log.debug(
|
||||||
txo.id: txo for txo in await self.db.get_txos(txoid__in=check_db_for_txos, no_tx=True)
|
"request %i transactions, %i/%i for %s are already synced", len(to_request), len(already_synced),
|
||||||
}
|
len(remote_history), address
|
||||||
|
|
||||||
for txi in tx.inputs:
|
|
||||||
if txi.txo_ref.txo is not None:
|
|
||||||
continue
|
|
||||||
referenced_txo = referenced_txos.get(txi.txo_ref.id)
|
|
||||||
if referenced_txo is not None:
|
|
||||||
txi.txo_ref = referenced_txo.ref
|
|
||||||
|
|
||||||
synced_history.write(f'{tx.id}:{tx.height}:')
|
|
||||||
synced_txs.append(tx)
|
|
||||||
|
|
||||||
await self.db.save_transaction_io_batch(
|
|
||||||
synced_txs, address, self.address_to_hash160(address), synced_history.getvalue()
|
|
||||||
)
|
)
|
||||||
await asyncio.wait([
|
remote_history_txids = {txid for txid, _ in remote_history}
|
||||||
self._on_transaction_controller.add(TransactionEvent(address, tx))
|
async for tx in self.request_synced_transactions(to_request, remote_history_txids, address):
|
||||||
for tx in synced_txs
|
pending_synced_history[tx_indexes[tx.id]] = f"{tx.id}:{tx.height}:"
|
||||||
])
|
if len(pending_synced_history) % 100 == 0:
|
||||||
|
log.info("Syncing address %s: %d/%d", address, len(pending_synced_history), len(to_request))
|
||||||
|
log.info("Sync finished for address %s: %d/%d", address, len(pending_synced_history), len(to_request))
|
||||||
|
|
||||||
|
assert len(pending_synced_history) == len(remote_history), \
|
||||||
|
f"{len(pending_synced_history)} vs {len(remote_history)}"
|
||||||
|
synced_history = ""
|
||||||
|
for remote_i, i in zip(range(len(remote_history)), sorted(pending_synced_history.keys())):
|
||||||
|
assert i == remote_i, f"{i} vs {remote_i}"
|
||||||
|
txid, height = remote_history[remote_i]
|
||||||
|
if f"{txid}:{height}:" != pending_synced_history[i]:
|
||||||
|
log.warning("history mismatch: %s vs %s", remote_history[remote_i], pending_synced_history[i])
|
||||||
|
synced_history += pending_synced_history[i]
|
||||||
|
await self.db.set_address_history(address, synced_history)
|
||||||
|
|
||||||
if address_manager is None:
|
if address_manager is None:
|
||||||
address_manager = await self.get_address_manager_for_address(address)
|
address_manager = await self.get_address_manager_for_address(address)
|
||||||
@@ -537,55 +573,135 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
await address_manager.ensure_address_gap()
|
await address_manager.ensure_address_gap()
|
||||||
|
|
||||||
local_status, local_history = \
|
local_status, local_history = \
|
||||||
await self.get_local_status_and_history(address, synced_history.getvalue())
|
await self.get_local_status_and_history(address, synced_history)
|
||||||
|
|
||||||
if local_status != remote_status:
|
if local_status != remote_status:
|
||||||
if local_history == remote_history:
|
if local_history == remote_history:
|
||||||
|
log.warning(
|
||||||
|
"%s has a synced history but a mismatched status", address
|
||||||
|
)
|
||||||
return True
|
return True
|
||||||
|
remote_set = set(remote_history)
|
||||||
|
local_set = set(local_history)
|
||||||
log.warning(
|
log.warning(
|
||||||
"Wallet is out of sync after syncing. Remote: %s with %d items, local: %s with %d items",
|
"%s is out of sync after syncing.\n"
|
||||||
remote_status, len(remote_history), local_status, len(local_history)
|
"Remote: %s with %d items (%i unique), local: %s with %d items (%i unique).\n"
|
||||||
|
"Histories are mismatched on %i items.\n"
|
||||||
|
"Local is missing\n"
|
||||||
|
"%s\n"
|
||||||
|
"Remote is missing\n"
|
||||||
|
"%s\n"
|
||||||
|
"******",
|
||||||
|
address, remote_status, len(remote_history), len(remote_set),
|
||||||
|
local_status, len(local_history), len(local_set), len(remote_set.symmetric_difference(local_set)),
|
||||||
|
"\n".join([f"{txid} - {height}" for txid, height in local_set.difference(remote_set)]),
|
||||||
|
"\n".join([f"{txid} - {height}" for txid, height in remote_set.difference(local_set)])
|
||||||
)
|
)
|
||||||
log.warning("local: %s", local_history)
|
|
||||||
log.warning("remote: %s", remote_history)
|
|
||||||
self._known_addresses_out_of_sync.add(address)
|
self._known_addresses_out_of_sync.add(address)
|
||||||
return False
|
return False
|
||||||
else:
|
else:
|
||||||
|
log.debug("finished syncing transaction history for %s, %i known txs", address, len(local_history))
|
||||||
return True
|
return True
|
||||||
|
|
||||||
async def cache_transaction(self, txid, remote_height, check_local=True):
|
async def maybe_verify_transaction(self, tx, remote_height, merkle=None):
|
||||||
cache_item = self._tx_cache.get(txid)
|
|
||||||
if cache_item is None:
|
|
||||||
cache_item = self._tx_cache[txid] = TransactionCacheItem()
|
|
||||||
elif cache_item.tx is not None and \
|
|
||||||
cache_item.tx.height >= remote_height and \
|
|
||||||
(cache_item.tx.is_verified or remote_height < 1):
|
|
||||||
return cache_item.tx # cached tx is already up-to-date
|
|
||||||
|
|
||||||
async with cache_item.lock:
|
|
||||||
|
|
||||||
tx = cache_item.tx
|
|
||||||
|
|
||||||
if tx is None and check_local:
|
|
||||||
# check local db
|
|
||||||
tx = cache_item.tx = await self.db.get_transaction(txid=txid)
|
|
||||||
|
|
||||||
if tx is None:
|
|
||||||
# fetch from network
|
|
||||||
_raw = await self.network.retriable_call(self.network.get_transaction, txid, remote_height)
|
|
||||||
tx = Transaction(unhexlify(_raw))
|
|
||||||
cache_item.tx = tx # make sure it's saved before caching it
|
|
||||||
|
|
||||||
await self.maybe_verify_transaction(tx, remote_height)
|
|
||||||
return tx
|
|
||||||
|
|
||||||
async def maybe_verify_transaction(self, tx, remote_height):
|
|
||||||
tx.height = remote_height
|
tx.height = remote_height
|
||||||
if 0 < remote_height < len(self.headers):
|
if 0 < remote_height < len(self.headers):
|
||||||
merkle = await self.network.retriable_call(self.network.get_merkle, tx.id, remote_height)
|
# can't be tx.pending_verifications == 1 because we have to handle the transaction_show case
|
||||||
|
if not merkle:
|
||||||
|
merkle = await self.network.retriable_call(self.network.get_merkle, tx.id, remote_height)
|
||||||
|
if 'merkle' not in merkle:
|
||||||
|
return
|
||||||
merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash)
|
merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash)
|
||||||
header = self.headers[remote_height]
|
header = await self.headers.get(remote_height)
|
||||||
tx.position = merkle['pos']
|
tx.position = merkle['pos']
|
||||||
tx.is_verified = merkle_root == header['merkle_root']
|
tx.is_verified = merkle_root == header['merkle_root']
|
||||||
|
return tx
|
||||||
|
|
||||||
|
async def request_transactions(self, to_request: Tuple[Tuple[str, int], ...], cached=False):
|
||||||
|
batches = [[]]
|
||||||
|
remote_heights = {}
|
||||||
|
cache_hits = set()
|
||||||
|
|
||||||
|
for txid, height in sorted(to_request, key=lambda x: x[1]):
|
||||||
|
if cached:
|
||||||
|
cached_tx = self._tx_cache.get(txid)
|
||||||
|
if cached_tx is not None:
|
||||||
|
if cached_tx.tx is not None and cached_tx.tx.is_verified:
|
||||||
|
cache_hits.add(txid)
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
self._tx_cache[txid] = TransactionCacheItem()
|
||||||
|
remote_heights[txid] = height
|
||||||
|
if len(batches[-1]) == 100:
|
||||||
|
batches.append([])
|
||||||
|
batches[-1].append(txid)
|
||||||
|
if not batches[-1]:
|
||||||
|
batches.pop()
|
||||||
|
if cached and cache_hits:
|
||||||
|
yield {txid: self._tx_cache[txid].tx for txid in cache_hits}
|
||||||
|
|
||||||
|
for batch in batches:
|
||||||
|
txs = await self._single_batch(batch, remote_heights)
|
||||||
|
if cached:
|
||||||
|
for txid, tx in txs.items():
|
||||||
|
self._tx_cache[txid].tx = tx
|
||||||
|
yield txs
|
||||||
|
|
||||||
|
async def request_synced_transactions(self, to_request, remote_history, address):
|
||||||
|
async for txs in self.request_transactions(((txid, height) for txid, height in to_request.values())):
|
||||||
|
for tx in txs.values():
|
||||||
|
yield tx
|
||||||
|
await self._sync_and_save_batch(address, remote_history, txs)
|
||||||
|
|
||||||
|
async def _single_batch(self, batch, remote_heights):
|
||||||
|
heights = {remote_heights[txid] for txid in batch}
|
||||||
|
unrestriced = 0 < min(heights) < max(heights) < max(self.headers.checkpoints or [0])
|
||||||
|
batch_result = await self.network.retriable_call(self.network.get_transaction_batch, batch, not unrestriced)
|
||||||
|
txs = {}
|
||||||
|
for txid, (raw, merkle) in batch_result.items():
|
||||||
|
remote_height = remote_heights[txid]
|
||||||
|
tx = Transaction(unhexlify(raw), height=remote_height)
|
||||||
|
txs[tx.id] = tx
|
||||||
|
await self.maybe_verify_transaction(tx, remote_height, merkle)
|
||||||
|
return txs
|
||||||
|
|
||||||
|
async def _sync_and_save_batch(self, address, remote_history, pending_txs):
|
||||||
|
await asyncio.gather(*(self._sync(tx, remote_history, pending_txs) for tx in pending_txs.values()))
|
||||||
|
await self.db.save_transaction_io_batch(
|
||||||
|
pending_txs.values(), address, self.address_to_hash160(address), ""
|
||||||
|
)
|
||||||
|
while pending_txs:
|
||||||
|
self._on_transaction_controller.add(TransactionEvent(address, pending_txs.popitem()[1]))
|
||||||
|
|
||||||
|
async def _sync(self, tx, remote_history, pending_txs):
|
||||||
|
check_db_for_txos = {}
|
||||||
|
for txi in tx.inputs:
|
||||||
|
if txi.txo_ref.txo is not None:
|
||||||
|
continue
|
||||||
|
wanted_txid = txi.txo_ref.tx_ref.id
|
||||||
|
if wanted_txid not in remote_history:
|
||||||
|
continue
|
||||||
|
if wanted_txid in pending_txs:
|
||||||
|
txi.txo_ref = pending_txs[wanted_txid].outputs[txi.txo_ref.position].ref
|
||||||
|
else:
|
||||||
|
check_db_for_txos[txi] = txi.txo_ref.id
|
||||||
|
|
||||||
|
referenced_txos = {} if not check_db_for_txos else {
|
||||||
|
txo.id: txo for txo in await self.db.get_txos(
|
||||||
|
txoid__in=list(check_db_for_txos.values()), order_by='txo.txoid', no_tx=True
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
for txi in check_db_for_txos:
|
||||||
|
if txi.txo_ref.id in referenced_txos:
|
||||||
|
txi.txo_ref = referenced_txos[txi.txo_ref.id].ref
|
||||||
|
else:
|
||||||
|
tx_from_db = await self.db.get_transaction(txid=txi.txo_ref.tx_ref.id)
|
||||||
|
if tx_from_db is None:
|
||||||
|
log.warning("%s not on db, not on cache, but on remote history!", txi.txo_ref.id)
|
||||||
|
else:
|
||||||
|
txi.txo_ref = tx_from_db.outputs[txi.txo_ref.position].ref
|
||||||
|
return tx
|
||||||
|
|
||||||
async def get_address_manager_for_address(self, address) -> Optional[AddressManager]:
|
async def get_address_manager_for_address(self, address) -> Optional[AddressManager]:
|
||||||
details = await self.db.get_address(address=address)
|
details = await self.db.get_address(address=address)
|
||||||
@@ -599,6 +715,7 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
return self.network.broadcast(hexlify(tx.raw).decode())
|
return self.network.broadcast(hexlify(tx.raw).decode())
|
||||||
|
|
||||||
async def wait(self, tx: Transaction, height=-1, timeout=1):
|
async def wait(self, tx: Transaction, height=-1, timeout=1):
|
||||||
|
timeout = timeout or 600 # after 10 minutes there is almost 0 hope
|
||||||
addresses = set()
|
addresses = set()
|
||||||
for txi in tx.inputs:
|
for txi in tx.inputs:
|
||||||
if txi.txo_ref.txo is not None:
|
if txi.txo_ref.txo is not None:
|
||||||
@@ -606,43 +723,89 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
self.hash160_to_address(txi.txo_ref.txo.pubkey_hash)
|
self.hash160_to_address(txi.txo_ref.txo.pubkey_hash)
|
||||||
)
|
)
|
||||||
for txo in tx.outputs:
|
for txo in tx.outputs:
|
||||||
if txo.has_address:
|
if txo.is_pubkey_hash:
|
||||||
addresses.add(self.hash160_to_address(txo.pubkey_hash))
|
addresses.add(self.hash160_to_address(txo.pubkey_hash))
|
||||||
|
elif txo.is_script_hash:
|
||||||
|
addresses.add(self.hash160_to_script_address(txo.script_hash))
|
||||||
|
start = int(time.perf_counter())
|
||||||
|
while timeout and (int(time.perf_counter()) - start) <= timeout:
|
||||||
|
if await self._wait_round(tx, height, addresses):
|
||||||
|
return
|
||||||
|
raise asyncio.TimeoutError('Timed out waiting for transaction.')
|
||||||
|
|
||||||
|
async def _wait_round(self, tx: Transaction, height: int, addresses: Iterable[str]):
|
||||||
records = await self.db.get_addresses(address__in=addresses)
|
records = await self.db.get_addresses(address__in=addresses)
|
||||||
_, pending = await asyncio.wait([
|
_, pending = await asyncio.wait([
|
||||||
self.on_transaction.where(partial(
|
self.on_transaction.where(partial(
|
||||||
lambda a, e: a == e.address and e.tx.height >= height and e.tx.id == tx.id,
|
lambda a, e: a == e.address and e.tx.height >= height and e.tx.id == tx.id,
|
||||||
address_record['address']
|
address_record['address']
|
||||||
)) for address_record in records
|
)) for address_record in records
|
||||||
], timeout=timeout)
|
], timeout=1)
|
||||||
if pending:
|
if not pending:
|
||||||
records = await self.db.get_addresses(address__in=addresses)
|
return True
|
||||||
for record in records:
|
records = await self.db.get_addresses(address__in=addresses)
|
||||||
found = False
|
for record in records:
|
||||||
local_history = (await self.get_local_status_and_history(
|
local_history = (await self.get_local_status_and_history(
|
||||||
record['address'], history=record['history']
|
record['address'], history=record['history']
|
||||||
))[1] if record['history'] else []
|
))[1] if record['history'] else []
|
||||||
for txid, local_height in local_history:
|
for txid, local_height in local_history:
|
||||||
if txid == tx.id and local_height >= height:
|
if txid == tx.id:
|
||||||
found = True
|
if local_height >= height or (local_height == 0 and height > local_height):
|
||||||
if not found:
|
return True
|
||||||
print(record['history'], addresses, tx.id)
|
log.warning(
|
||||||
raise asyncio.TimeoutError('Timed out waiting for transaction.')
|
"local history has higher height than remote for %s (%i vs %i)", txid,
|
||||||
|
local_height, height
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
log.warning(
|
||||||
|
"local history does not contain %s, requested height %i", tx.id, height
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
async def _inflate_outputs(self, query, accounts) -> Tuple[List[Output], dict, int, int]:
|
async def _inflate_outputs(
|
||||||
|
self, query, accounts,
|
||||||
|
include_purchase_receipt=False,
|
||||||
|
include_is_my_output=False,
|
||||||
|
include_sent_supports=False,
|
||||||
|
include_sent_tips=False,
|
||||||
|
include_received_tips=False,
|
||||||
|
hub_server=False) -> Tuple[List[Output], dict, int, int]:
|
||||||
encoded_outputs = await query
|
encoded_outputs = await query
|
||||||
outputs = Outputs.from_base64(encoded_outputs or b'') # TODO: why is the server returning None?
|
if hub_server:
|
||||||
txs = []
|
outputs = Outputs.from_grpc(encoded_outputs)
|
||||||
|
else:
|
||||||
|
outputs = Outputs.from_base64(encoded_outputs or b'') # TODO: why is the server returning None?
|
||||||
|
txs: List[Transaction] = []
|
||||||
if len(outputs.txs) > 0:
|
if len(outputs.txs) > 0:
|
||||||
txs: List[Transaction] = await asyncio.gather(*(
|
async for tx in self.request_transactions(tuple(outputs.txs), cached=True):
|
||||||
self.cache_transaction(*tx) for tx in outputs.txs
|
txs.extend(tx.values())
|
||||||
))
|
|
||||||
if accounts:
|
_txos, blocked = outputs.inflate(txs)
|
||||||
|
|
||||||
|
txos = []
|
||||||
|
for txo in _txos:
|
||||||
|
if isinstance(txo, Output):
|
||||||
|
# transactions and outputs are cached and shared between wallets
|
||||||
|
# we don't want to leak informaion between wallet so we add the
|
||||||
|
# wallet specific metadata on throw away copies of the txos
|
||||||
|
txo = copy.copy(txo)
|
||||||
|
channel = txo.channel
|
||||||
|
txo.purchase_receipt = None
|
||||||
|
txo.update_annotations(None)
|
||||||
|
txo.channel = channel
|
||||||
|
txos.append(txo)
|
||||||
|
|
||||||
|
includes = (
|
||||||
|
include_purchase_receipt, include_is_my_output,
|
||||||
|
include_sent_supports, include_sent_tips
|
||||||
|
)
|
||||||
|
if accounts and any(includes):
|
||||||
|
receipts = {}
|
||||||
|
if include_purchase_receipt:
|
||||||
priced_claims = []
|
priced_claims = []
|
||||||
for tx in txs:
|
for txo in txos:
|
||||||
for txo in tx.outputs:
|
if isinstance(txo, Output) and txo.has_price:
|
||||||
if txo.has_price:
|
priced_claims.append(txo)
|
||||||
priced_claims.append(txo)
|
|
||||||
if priced_claims:
|
if priced_claims:
|
||||||
receipts = {
|
receipts = {
|
||||||
txo.purchased_claim_id: txo for txo in
|
txo.purchased_claim_id: txo for txo in
|
||||||
@@ -651,14 +814,57 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
purchased_claim_id__in=[c.claim_id for c in priced_claims]
|
purchased_claim_id__in=[c.claim_id for c in priced_claims]
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
for txo in priced_claims:
|
for txo in txos:
|
||||||
|
if isinstance(txo, Output) and txo.can_decode_claim:
|
||||||
|
if include_purchase_receipt:
|
||||||
txo.purchase_receipt = receipts.get(txo.claim_id)
|
txo.purchase_receipt = receipts.get(txo.claim_id)
|
||||||
txos, blocked = outputs.inflate(txs)
|
if include_is_my_output:
|
||||||
|
mine = await self.db.get_txo_count(
|
||||||
|
claim_id=txo.claim_id, txo_type__in=CLAIM_TYPES, is_my_output=True,
|
||||||
|
is_spent=False, accounts=accounts
|
||||||
|
)
|
||||||
|
if mine:
|
||||||
|
txo.is_my_output = True
|
||||||
|
else:
|
||||||
|
txo.is_my_output = False
|
||||||
|
if include_sent_supports:
|
||||||
|
supports = await self.db.get_txo_sum(
|
||||||
|
claim_id=txo.claim_id, txo_type=TXO_TYPES['support'],
|
||||||
|
is_my_input=True, is_my_output=True,
|
||||||
|
is_spent=False, accounts=accounts
|
||||||
|
)
|
||||||
|
txo.sent_supports = supports
|
||||||
|
if include_sent_tips:
|
||||||
|
tips = await self.db.get_txo_sum(
|
||||||
|
claim_id=txo.claim_id, txo_type=TXO_TYPES['support'],
|
||||||
|
is_my_input=True, is_my_output=False,
|
||||||
|
accounts=accounts
|
||||||
|
)
|
||||||
|
txo.sent_tips = tips
|
||||||
|
if include_received_tips:
|
||||||
|
tips = await self.db.get_txo_sum(
|
||||||
|
claim_id=txo.claim_id, txo_type=TXO_TYPES['support'],
|
||||||
|
is_my_input=False, is_my_output=True,
|
||||||
|
accounts=accounts
|
||||||
|
)
|
||||||
|
txo.received_tips = tips
|
||||||
return txos, blocked, outputs.offset, outputs.total
|
return txos, blocked, outputs.offset, outputs.total
|
||||||
|
|
||||||
async def resolve(self, accounts, urls):
|
async def resolve(self, accounts, urls, new_sdk_server=None, **kwargs):
|
||||||
resolve = partial(self.network.retriable_call, self.network.resolve)
|
txos = []
|
||||||
txos = (await self._inflate_outputs(resolve(urls), accounts))[0]
|
urls_copy = list(urls)
|
||||||
|
if new_sdk_server:
|
||||||
|
resolve = partial(self.network.new_resolve, new_sdk_server)
|
||||||
|
else:
|
||||||
|
resolve = partial(self.network.retriable_call, self.network.resolve)
|
||||||
|
while urls_copy:
|
||||||
|
batch, urls_copy = urls_copy[:100], urls_copy[100:]
|
||||||
|
txos.extend(
|
||||||
|
(await self._inflate_outputs(
|
||||||
|
resolve(batch), accounts, **kwargs
|
||||||
|
))[0]
|
||||||
|
)
|
||||||
|
|
||||||
assert len(urls) == len(txos), "Mismatch between urls requested for resolve and responses received."
|
assert len(urls) == len(txos), "Mismatch between urls requested for resolve and responses received."
|
||||||
result = {}
|
result = {}
|
||||||
for url, txo in zip(urls, txos):
|
for url, txo in zip(urls, txos):
|
||||||
@@ -671,11 +877,25 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
result[url] = txo
|
result[url] = txo
|
||||||
return result
|
return result
|
||||||
|
|
||||||
async def claim_search(self, accounts, **kwargs) -> Tuple[List[Output], dict, int, int]:
|
async def sum_supports(self, new_sdk_server, **kwargs) -> List[Dict]:
|
||||||
return await self._inflate_outputs(self.network.claim_search(**kwargs), accounts)
|
return await self.network.sum_supports(new_sdk_server, **kwargs)
|
||||||
|
|
||||||
async def get_claim_by_claim_id(self, accounts, claim_id) -> Output:
|
async def claim_search(
|
||||||
for claim in (await self.claim_search(accounts, claim_id=claim_id))[0]:
|
self, accounts, include_purchase_receipt=False, include_is_my_output=False,
|
||||||
|
new_sdk_server=None, **kwargs) -> Tuple[List[Output], dict, int, int]:
|
||||||
|
if new_sdk_server:
|
||||||
|
claim_search = partial(self.network.new_claim_search, new_sdk_server)
|
||||||
|
else:
|
||||||
|
claim_search = self.network.claim_search
|
||||||
|
return await self._inflate_outputs(
|
||||||
|
claim_search(**kwargs), accounts,
|
||||||
|
include_purchase_receipt=include_purchase_receipt,
|
||||||
|
include_is_my_output=include_is_my_output,
|
||||||
|
hub_server=new_sdk_server is not None
|
||||||
|
)
|
||||||
|
|
||||||
|
async def get_claim_by_claim_id(self, accounts, claim_id, **kwargs) -> Output:
|
||||||
|
for claim in (await self.claim_search(accounts, claim_id=claim_id, **kwargs))[0]:
|
||||||
return claim
|
return claim
|
||||||
|
|
||||||
async def _report_state(self):
|
async def _report_state(self):
|
||||||
@@ -734,11 +954,19 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
return self.db.get_purchase_count(**constraints)
|
return self.db.get_purchase_count(**constraints)
|
||||||
|
|
||||||
async def _resolve_for_local_results(self, accounts, txos):
|
async def _resolve_for_local_results(self, accounts, txos):
|
||||||
|
txos = await self._resolve_for_local_claim_results(accounts, txos)
|
||||||
|
txos = await self._resolve_for_local_support_results(accounts, txos)
|
||||||
|
return txos
|
||||||
|
|
||||||
|
async def _resolve_for_local_claim_results(self, accounts, txos):
|
||||||
results = []
|
results = []
|
||||||
response = await self.resolve(accounts, [txo.permanent_url for txo in txos])
|
response = await self.resolve(
|
||||||
|
accounts, [txo.permanent_url for txo in txos if txo.can_decode_claim]
|
||||||
|
)
|
||||||
for txo in txos:
|
for txo in txos:
|
||||||
resolved = response[txo.permanent_url]
|
resolved = response.get(txo.permanent_url) if txo.can_decode_claim else None
|
||||||
if isinstance(resolved, Output):
|
if isinstance(resolved, Output):
|
||||||
|
resolved.update_annotations(txo)
|
||||||
results.append(resolved)
|
results.append(resolved)
|
||||||
else:
|
else:
|
||||||
if isinstance(resolved, dict) and 'error' in resolved:
|
if isinstance(resolved, dict) and 'error' in resolved:
|
||||||
@@ -746,6 +974,23 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
results.append(txo)
|
results.append(txo)
|
||||||
return results
|
return results
|
||||||
|
|
||||||
|
async def _resolve_for_local_support_results(self, accounts, txos):
|
||||||
|
channel_ids = set()
|
||||||
|
signed_support_txos = []
|
||||||
|
for txo in txos:
|
||||||
|
support = txo.can_decode_support
|
||||||
|
if support and support.signing_channel_id:
|
||||||
|
channel_ids.add(support.signing_channel_id)
|
||||||
|
signed_support_txos.append(txo)
|
||||||
|
if channel_ids:
|
||||||
|
channels = {
|
||||||
|
channel.claim_id: channel for channel in
|
||||||
|
(await self.claim_search(accounts, claim_ids=list(channel_ids)))[0]
|
||||||
|
}
|
||||||
|
for txo in signed_support_txos:
|
||||||
|
txo.channel = channels.get(txo.support.signing_channel_id)
|
||||||
|
return txos
|
||||||
|
|
||||||
async def get_claims(self, resolve=False, **constraints):
|
async def get_claims(self, resolve=False, **constraints):
|
||||||
claims = await self.db.get_claims(**constraints)
|
claims = await self.db.get_claims(**constraints)
|
||||||
if resolve:
|
if resolve:
|
||||||
@@ -774,7 +1019,7 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
return self.db.get_channel_count(**constraints)
|
return self.db.get_channel_count(**constraints)
|
||||||
|
|
||||||
async def resolve_collection(self, collection, offset=0, page_size=1):
|
async def resolve_collection(self, collection, offset=0, page_size=1):
|
||||||
claim_ids = collection.claim.collection.claims.ids[offset:page_size+offset]
|
claim_ids = collection.claim.collection.claims.ids[offset:page_size + offset]
|
||||||
try:
|
try:
|
||||||
resolve_results, _, _, _ = await self.claim_search([], claim_ids=claim_ids)
|
resolve_results, _, _, _ = await self.claim_search([], claim_ids=claim_ids)
|
||||||
except Exception as err:
|
except Exception as err:
|
||||||
@@ -794,8 +1039,10 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
claims.append(None)
|
claims.append(None)
|
||||||
return claims
|
return claims
|
||||||
|
|
||||||
async def get_collections(self, resolve_claims=0, **constraints):
|
async def get_collections(self, resolve_claims=0, resolve=False, **constraints):
|
||||||
collections = await self.db.get_collections(**constraints)
|
collections = await self.db.get_collections(**constraints)
|
||||||
|
if resolve:
|
||||||
|
collections = await self._resolve_for_local_results(constraints.get('accounts', []), collections)
|
||||||
if resolve_claims > 0:
|
if resolve_claims > 0:
|
||||||
for collection in collections:
|
for collection in collections:
|
||||||
collection.claims = await self.resolve_collection(collection, page_size=resolve_claims)
|
collection.claims = await self.resolve_collection(collection, page_size=resolve_claims)
|
||||||
@@ -810,27 +1057,30 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
def get_support_count(self, **constraints):
|
def get_support_count(self, **constraints):
|
||||||
return self.db.get_support_count(**constraints)
|
return self.db.get_support_count(**constraints)
|
||||||
|
|
||||||
async def get_transaction_history(self, **constraints):
|
async def get_transaction_history(self, read_only=False, **constraints):
|
||||||
txs: List[Transaction] = await self.db.get_transactions(**constraints)
|
txs: List[Transaction] = await self.db.get_transactions(
|
||||||
|
include_is_my_output=True, include_is_spent=True,
|
||||||
|
read_only=read_only, **constraints
|
||||||
|
)
|
||||||
headers = self.headers
|
headers = self.headers
|
||||||
history = []
|
history = []
|
||||||
for tx in txs: # pylint: disable=too-many-nested-blocks
|
for tx in txs: # pylint: disable=too-many-nested-blocks
|
||||||
ts = headers[tx.height]['timestamp'] if tx.height > 0 else None
|
ts = headers.estimated_timestamp(tx.height)
|
||||||
item = {
|
item = {
|
||||||
'txid': tx.id,
|
'txid': tx.id,
|
||||||
'timestamp': ts,
|
'timestamp': ts,
|
||||||
'date': datetime.fromtimestamp(ts).isoformat(' ')[:-3] if tx.height > 0 else None,
|
'date': datetime.fromtimestamp(ts).isoformat(' ')[:-3] if tx.height > 0 else None,
|
||||||
'confirmations': (headers.height+1) - tx.height if tx.height > 0 else 0,
|
'confirmations': (headers.height + 1) - tx.height if tx.height > 0 else 0,
|
||||||
'claim_info': [],
|
'claim_info': [],
|
||||||
'update_info': [],
|
'update_info': [],
|
||||||
'support_info': [],
|
'support_info': [],
|
||||||
'abandon_info': [],
|
'abandon_info': [],
|
||||||
'purchase_info': []
|
'purchase_info': []
|
||||||
}
|
}
|
||||||
is_my_inputs = all([txi.is_my_account for txi in tx.inputs])
|
is_my_inputs = all(txi.is_my_input for txi in tx.inputs)
|
||||||
if is_my_inputs:
|
if is_my_inputs:
|
||||||
# fees only matter if we are the ones paying them
|
# fees only matter if we are the ones paying them
|
||||||
item['value'] = dewies_to_lbc(tx.net_account_balance+tx.fee)
|
item['value'] = dewies_to_lbc(tx.net_account_balance + tx.fee)
|
||||||
item['fee'] = dewies_to_lbc(-tx.fee)
|
item['fee'] = dewies_to_lbc(-tx.fee)
|
||||||
else:
|
else:
|
||||||
# someone else paid the fees
|
# someone else paid the fees
|
||||||
@@ -843,7 +1093,8 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
'amount': dewies_to_lbc(txo.amount),
|
'amount': dewies_to_lbc(txo.amount),
|
||||||
'claim_id': txo.claim_id,
|
'claim_id': txo.claim_id,
|
||||||
'claim_name': txo.claim_name,
|
'claim_name': txo.claim_name,
|
||||||
'nout': txo.position
|
'nout': txo.position,
|
||||||
|
'is_spent': txo.is_spent,
|
||||||
})
|
})
|
||||||
for txo in tx.my_update_outputs:
|
for txo in tx.my_update_outputs:
|
||||||
if is_my_inputs: # updating my own claim
|
if is_my_inputs: # updating my own claim
|
||||||
@@ -852,17 +1103,18 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
if txi.txo_ref.txo is not None:
|
if txi.txo_ref.txo is not None:
|
||||||
other_txo = txi.txo_ref.txo
|
other_txo = txi.txo_ref.txo
|
||||||
if (other_txo.is_claim or other_txo.script.is_support_claim) \
|
if (other_txo.is_claim or other_txo.script.is_support_claim) \
|
||||||
and other_txo.claim_id == txo.claim_id:
|
and other_txo.claim_id == txo.claim_id:
|
||||||
previous = other_txo
|
previous = other_txo
|
||||||
break
|
break
|
||||||
if previous is not None:
|
if previous is not None:
|
||||||
item['update_info'].append({
|
item['update_info'].append({
|
||||||
'address': txo.get_address(self),
|
'address': txo.get_address(self),
|
||||||
'balance_delta': dewies_to_lbc(previous.amount-txo.amount),
|
'balance_delta': dewies_to_lbc(previous.amount - txo.amount),
|
||||||
'amount': dewies_to_lbc(txo.amount),
|
'amount': dewies_to_lbc(txo.amount),
|
||||||
'claim_id': txo.claim_id,
|
'claim_id': txo.claim_id,
|
||||||
'claim_name': txo.claim_name,
|
'claim_name': txo.claim_name,
|
||||||
'nout': txo.position
|
'nout': txo.position,
|
||||||
|
'is_spent': txo.is_spent,
|
||||||
})
|
})
|
||||||
else: # someone sent us their claim
|
else: # someone sent us their claim
|
||||||
item['update_info'].append({
|
item['update_info'].append({
|
||||||
@@ -871,7 +1123,8 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
'amount': dewies_to_lbc(txo.amount),
|
'amount': dewies_to_lbc(txo.amount),
|
||||||
'claim_id': txo.claim_id,
|
'claim_id': txo.claim_id,
|
||||||
'claim_name': txo.claim_name,
|
'claim_name': txo.claim_name,
|
||||||
'nout': txo.position
|
'nout': txo.position,
|
||||||
|
'is_spent': txo.is_spent,
|
||||||
})
|
})
|
||||||
for txo in tx.my_support_outputs:
|
for txo in tx.my_support_outputs:
|
||||||
item['support_info'].append({
|
item['support_info'].append({
|
||||||
@@ -881,7 +1134,8 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
'claim_id': txo.claim_id,
|
'claim_id': txo.claim_id,
|
||||||
'claim_name': txo.claim_name,
|
'claim_name': txo.claim_name,
|
||||||
'is_tip': not is_my_inputs,
|
'is_tip': not is_my_inputs,
|
||||||
'nout': txo.position
|
'nout': txo.position,
|
||||||
|
'is_spent': txo.is_spent,
|
||||||
})
|
})
|
||||||
if is_my_inputs:
|
if is_my_inputs:
|
||||||
for txo in tx.other_support_outputs:
|
for txo in tx.other_support_outputs:
|
||||||
@@ -892,7 +1146,8 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
'claim_id': txo.claim_id,
|
'claim_id': txo.claim_id,
|
||||||
'claim_name': txo.claim_name,
|
'claim_name': txo.claim_name,
|
||||||
'is_tip': is_my_inputs,
|
'is_tip': is_my_inputs,
|
||||||
'nout': txo.position
|
'nout': txo.position,
|
||||||
|
'is_spent': txo.is_spent,
|
||||||
})
|
})
|
||||||
for txo in tx.my_abandon_outputs:
|
for txo in tx.my_abandon_outputs:
|
||||||
item['abandon_info'].append({
|
item['abandon_info'].append({
|
||||||
@@ -909,13 +1164,14 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
'balance_delta': dewies_to_lbc(txo.amount if not is_my_inputs else -txo.amount),
|
'balance_delta': dewies_to_lbc(txo.amount if not is_my_inputs else -txo.amount),
|
||||||
'amount': dewies_to_lbc(txo.amount),
|
'amount': dewies_to_lbc(txo.amount),
|
||||||
'claim_id': txo.purchased_claim_id,
|
'claim_id': txo.purchased_claim_id,
|
||||||
'nout': txo.position
|
'nout': txo.position,
|
||||||
|
'is_spent': txo.is_spent,
|
||||||
})
|
})
|
||||||
history.append(item)
|
history.append(item)
|
||||||
return history
|
return history
|
||||||
|
|
||||||
def get_transaction_history_count(self, **constraints):
|
def get_transaction_history_count(self, read_only=False, **constraints):
|
||||||
return self.db.get_transaction_count(**constraints)
|
return self.db.get_transaction_count(read_only=read_only, **constraints)
|
||||||
|
|
||||||
async def get_detailed_balance(self, accounts, confirmations=0):
|
async def get_detailed_balance(self, accounts, confirmations=0):
|
||||||
result = {
|
result = {
|
||||||
@@ -931,8 +1187,8 @@ class Ledger(metaclass=LedgerRegistry):
|
|||||||
for account in accounts:
|
for account in accounts:
|
||||||
balance = self._balance_cache.get(account.id)
|
balance = self._balance_cache.get(account.id)
|
||||||
if not balance:
|
if not balance:
|
||||||
balance = self._balance_cache[account.id] =\
|
balance = self._balance_cache[account.id] = \
|
||||||
await account.get_detailed_balance(confirmations, reserved_subtotals=True)
|
await account.get_detailed_balance(confirmations)
|
||||||
for key, value in balance.items():
|
for key, value in balance.items():
|
||||||
if key == 'reserved_subtotals':
|
if key == 'reserved_subtotals':
|
||||||
for subkey, subvalue in value.items():
|
for subkey, subvalue in value.items():
|
||||||
@@ -948,6 +1204,7 @@ class TestNetLedger(Ledger):
|
|||||||
script_address_prefix = bytes((196,))
|
script_address_prefix = bytes((196,))
|
||||||
extended_public_key_prefix = unhexlify('043587cf')
|
extended_public_key_prefix = unhexlify('043587cf')
|
||||||
extended_private_key_prefix = unhexlify('04358394')
|
extended_private_key_prefix = unhexlify('04358394')
|
||||||
|
checkpoints = {}
|
||||||
|
|
||||||
|
|
||||||
class RegTestLedger(Ledger):
|
class RegTestLedger(Ledger):
|
||||||
@@ -962,3 +1219,4 @@ class RegTestLedger(Ledger):
|
|||||||
genesis_hash = '6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556'
|
genesis_hash = '6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556'
|
||||||
genesis_bits = 0x207fffff
|
genesis_bits = 0x207fffff
|
||||||
target_timespan = 1
|
target_timespan = 1
|
||||||
|
checkpoints = {}
|
||||||
+42
-19
@@ -3,12 +3,14 @@ import json
|
|||||||
import typing
|
import typing
|
||||||
import logging
|
import logging
|
||||||
import asyncio
|
import asyncio
|
||||||
|
from distutils.util import strtobool
|
||||||
|
|
||||||
from binascii import unhexlify
|
from binascii import unhexlify
|
||||||
from decimal import Decimal
|
from decimal import Decimal
|
||||||
from typing import List, Type, MutableSequence, MutableMapping, Optional
|
from typing import List, Type, MutableSequence, MutableMapping, Optional
|
||||||
|
|
||||||
from lbry.error import KeyFeeAboveMaxAllowedError
|
from lbry.error import KeyFeeAboveMaxAllowedError, WalletNotLoadedError
|
||||||
from lbry.conf import Config
|
from lbry.conf import Config, NOT_SET
|
||||||
|
|
||||||
from .dewies import dewies_to_lbc
|
from .dewies import dewies_to_lbc
|
||||||
from .account import Account
|
from .account import Account
|
||||||
@@ -95,7 +97,7 @@ class WalletManager:
|
|||||||
for wallet in self.wallets:
|
for wallet in self.wallets:
|
||||||
if wallet.id == wallet_id:
|
if wallet.id == wallet_id:
|
||||||
return wallet
|
return wallet
|
||||||
raise ValueError(f"Couldn't find wallet: {wallet_id}.")
|
raise WalletNotLoadedError(wallet_id)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_balance(wallet):
|
def get_balance(wallet):
|
||||||
@@ -181,9 +183,16 @@ class WalletManager:
|
|||||||
}[config.blockchain_name]
|
}[config.blockchain_name]
|
||||||
|
|
||||||
ledger_config = {
|
ledger_config = {
|
||||||
|
'use_go_hub': not strtobool(os.environ.get('ENABLE_LEGACY_SEARCH') or 'yes'),
|
||||||
'auto_connect': True,
|
'auto_connect': True,
|
||||||
|
'explicit_servers': [],
|
||||||
|
'hub_timeout': config.hub_timeout,
|
||||||
'default_servers': config.lbryum_servers,
|
'default_servers': config.lbryum_servers,
|
||||||
|
'known_hubs': config.known_hubs,
|
||||||
|
'jurisdiction': config.jurisdiction,
|
||||||
|
'concurrent_hub_requests': config.concurrent_hub_requests,
|
||||||
'data_path': config.wallet_dir,
|
'data_path': config.wallet_dir,
|
||||||
|
'tx_cache_size': config.transaction_cache_size
|
||||||
}
|
}
|
||||||
|
|
||||||
wallets_directory = os.path.join(config.wallet_dir, 'wallets')
|
wallets_directory = os.path.join(config.wallet_dir, 'wallets')
|
||||||
@@ -194,6 +203,10 @@ class WalletManager:
|
|||||||
os.path.join(wallets_directory, 'default_wallet')
|
os.path.join(wallets_directory, 'default_wallet')
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if Config.lbryum_servers.is_set_to_default(config):
|
||||||
|
with config.update_config() as c:
|
||||||
|
c.lbryum_servers = NOT_SET
|
||||||
|
|
||||||
manager = cls.from_config({
|
manager = cls.from_config({
|
||||||
'ledgers': {ledger_id: ledger_config},
|
'ledgers': {ledger_id: ledger_config},
|
||||||
'wallets': [
|
'wallets': [
|
||||||
@@ -223,10 +236,18 @@ class WalletManager:
|
|||||||
|
|
||||||
async def reset(self):
|
async def reset(self):
|
||||||
self.ledger.config = {
|
self.ledger.config = {
|
||||||
|
'use_go_hub': not strtobool(os.environ.get('ENABLE_LEGACY_SEARCH') or 'yes'),
|
||||||
'auto_connect': True,
|
'auto_connect': True,
|
||||||
'default_servers': self.config.lbryum_servers,
|
'explicit_servers': [],
|
||||||
|
'default_servers': Config.lbryum_servers.default,
|
||||||
|
'known_hubs': self.config.known_hubs,
|
||||||
|
'jurisdiction': self.config.jurisdiction,
|
||||||
|
'hub_timeout': self.config.hub_timeout,
|
||||||
|
'concurrent_hub_requests': self.config.concurrent_hub_requests,
|
||||||
'data_path': self.config.wallet_dir,
|
'data_path': self.config.wallet_dir,
|
||||||
}
|
}
|
||||||
|
if Config.lbryum_servers.is_set(self.config):
|
||||||
|
self.ledger.config['explicit_servers'] = self.config.lbryum_servers
|
||||||
await self.ledger.stop()
|
await self.ledger.stop()
|
||||||
await self.ledger.start()
|
await self.ledger.start()
|
||||||
|
|
||||||
@@ -248,26 +269,28 @@ class WalletManager:
|
|||||||
log.warning("Failed to migrate %s receiving addresses!",
|
log.warning("Failed to migrate %s receiving addresses!",
|
||||||
len(set(receiving_addresses).difference(set(migrated_receiving))))
|
len(set(receiving_addresses).difference(set(migrated_receiving))))
|
||||||
|
|
||||||
def get_best_blockhash(self):
|
async def get_best_blockhash(self):
|
||||||
if len(self.ledger.headers) <= 0:
|
if len(self.ledger.headers) <= 0:
|
||||||
return self.ledger.genesis_hash
|
return self.ledger.genesis_hash
|
||||||
return self.ledger.headers.hash(self.ledger.headers.height).decode()
|
return (await self.ledger.headers.hash(self.ledger.headers.height)).decode()
|
||||||
|
|
||||||
def get_unused_address(self):
|
def get_unused_address(self):
|
||||||
return self.default_account.receiving.get_or_create_usable_address()
|
return self.default_account.receiving.get_or_create_usable_address()
|
||||||
|
|
||||||
async def get_transaction(self, txid):
|
async def get_transaction(self, txid: str):
|
||||||
tx = await self.db.get_transaction(txid=txid)
|
tx = await self.db.get_transaction(txid=txid)
|
||||||
if not tx:
|
if tx:
|
||||||
try:
|
return tx
|
||||||
raw = await self.ledger.network.get_transaction(txid)
|
try:
|
||||||
height = await self.ledger.network.get_transaction_height(txid)
|
raw, merkle = await self.ledger.network.get_transaction_and_merkle(txid)
|
||||||
except CodeMessageError as e:
|
except CodeMessageError as e:
|
||||||
if 'No such mempool or blockchain transaction.' in e.message:
|
if 'No such mempool or blockchain transaction.' in e.message:
|
||||||
return {'success': False, 'code': 404, 'message': 'transaction not found'}
|
return {'success': False, 'code': 404, 'message': 'transaction not found'}
|
||||||
return {'success': False, 'code': e.code, 'message': e.message}
|
return {'success': False, 'code': e.code, 'message': e.message}
|
||||||
tx = Transaction(unhexlify(raw))
|
height = merkle.get('block_height')
|
||||||
await self.ledger.maybe_verify_transaction(tx, height)
|
tx = Transaction(unhexlify(raw), height=height)
|
||||||
|
if height and height > 0:
|
||||||
|
await self.ledger.maybe_verify_transaction(tx, height, merkle)
|
||||||
return tx
|
return tx
|
||||||
|
|
||||||
async def create_purchase_transaction(
|
async def create_purchase_transaction(
|
||||||
@@ -297,8 +320,8 @@ class WalletManager:
|
|||||||
async def broadcast_or_release(self, tx, blocking=False):
|
async def broadcast_or_release(self, tx, blocking=False):
|
||||||
try:
|
try:
|
||||||
await self.ledger.broadcast(tx)
|
await self.ledger.broadcast(tx)
|
||||||
if blocking:
|
|
||||||
await self.ledger.wait(tx, timeout=None)
|
|
||||||
except:
|
except:
|
||||||
await self.ledger.release_tx(tx)
|
await self.ledger.release_tx(tx)
|
||||||
raise
|
raise
|
||||||
|
if blocking:
|
||||||
|
await self.ledger.wait(tx, timeout=None)
|
||||||
+289
-192
@@ -1,34 +1,43 @@
|
|||||||
import logging
|
import logging
|
||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
|
import socket
|
||||||
|
import random
|
||||||
from time import perf_counter
|
from time import perf_counter
|
||||||
from operator import itemgetter
|
from collections import defaultdict
|
||||||
from typing import Dict, Optional, Tuple
|
from typing import Dict, Optional, Tuple
|
||||||
|
import aiohttp
|
||||||
|
import grpc
|
||||||
|
from lbry.schema.types.v2 import hub_pb2_grpc
|
||||||
|
from lbry.schema.types.v2.hub_pb2 import SearchRequest
|
||||||
|
|
||||||
from lbry import __version__
|
from lbry import __version__
|
||||||
|
from lbry.utils import resolve_host
|
||||||
from lbry.error import IncompatibleWalletServerError
|
from lbry.error import IncompatibleWalletServerError
|
||||||
from lbry.wallet.rpc import RPCSession as BaseClientSession, Connector, RPCError, ProtocolError
|
from lbry.wallet.rpc import RPCSession as BaseClientSession, Connector, RPCError, ProtocolError
|
||||||
from lbry.wallet.stream import StreamController
|
from lbry.wallet.stream import StreamController
|
||||||
|
from lbry.wallet.server.udp import SPVStatusClientProtocol, SPVPong
|
||||||
|
from lbry.conf import KnownHubsList
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class ClientSession(BaseClientSession):
|
class ClientSession(BaseClientSession):
|
||||||
def __init__(self, *args, network, server, timeout=30, on_connect_callback=None, **kwargs):
|
def __init__(self, *args, network: 'Network', server, timeout=30, concurrency=32, **kwargs):
|
||||||
self.network = network
|
self.network = network
|
||||||
self.server = server
|
self.server = server
|
||||||
super().__init__(*args, **kwargs)
|
super().__init__(*args, **kwargs)
|
||||||
self._on_disconnect_controller = StreamController()
|
|
||||||
self.on_disconnected = self._on_disconnect_controller.stream
|
|
||||||
self.framer.max_size = self.max_errors = 1 << 32
|
self.framer.max_size = self.max_errors = 1 << 32
|
||||||
self.timeout = timeout
|
self.timeout = timeout
|
||||||
self.max_seconds_idle = timeout * 2
|
self.max_seconds_idle = timeout * 2
|
||||||
self.response_time: Optional[float] = None
|
self.response_time: Optional[float] = None
|
||||||
self.connection_latency: Optional[float] = None
|
self.connection_latency: Optional[float] = None
|
||||||
self._response_samples = 0
|
self._response_samples = 0
|
||||||
self.pending_amount = 0
|
self._concurrency = asyncio.Semaphore(concurrency)
|
||||||
self._on_connect_cb = on_connect_callback or (lambda: None)
|
|
||||||
self.trigger_urgent_reconnect = asyncio.Event()
|
@property
|
||||||
|
def concurrency(self):
|
||||||
|
return self._concurrency._value
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def available(self):
|
def available(self):
|
||||||
@@ -54,9 +63,9 @@ class ClientSession(BaseClientSession):
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
async def send_request(self, method, args=()):
|
async def send_request(self, method, args=()):
|
||||||
self.pending_amount += 1
|
log.debug("send %s%s to %s:%i (%i timeout)", method, tuple(args), self.server[0], self.server[1], self.timeout)
|
||||||
log.debug("send %s to %s:%i", method, *self.server)
|
|
||||||
try:
|
try:
|
||||||
|
await self._concurrency.acquire()
|
||||||
if method == 'server.version':
|
if method == 'server.version':
|
||||||
return await self.send_timed_server_version_request(args, self.timeout)
|
return await self.send_timed_server_version_request(args, self.timeout)
|
||||||
request = asyncio.ensure_future(super().send_request(method, args))
|
request = asyncio.ensure_future(super().send_request(method, args))
|
||||||
@@ -66,7 +75,7 @@ class ClientSession(BaseClientSession):
|
|||||||
log.debug("Time since last packet: %s", perf_counter() - self.last_packet_received)
|
log.debug("Time since last packet: %s", perf_counter() - self.last_packet_received)
|
||||||
if (perf_counter() - self.last_packet_received) < self.timeout:
|
if (perf_counter() - self.last_packet_received) < self.timeout:
|
||||||
continue
|
continue
|
||||||
log.info("timeout sending %s to %s:%i", method, *self.server)
|
log.warning("timeout sending %s to %s:%i", method, *self.server)
|
||||||
raise asyncio.TimeoutError
|
raise asyncio.TimeoutError
|
||||||
if done:
|
if done:
|
||||||
try:
|
try:
|
||||||
@@ -86,43 +95,11 @@ class ClientSession(BaseClientSession):
|
|||||||
self.synchronous_close()
|
self.synchronous_close()
|
||||||
raise
|
raise
|
||||||
except asyncio.CancelledError:
|
except asyncio.CancelledError:
|
||||||
log.info("cancelled sending %s to %s:%i", method, *self.server)
|
log.warning("cancelled sending %s to %s:%i", method, *self.server)
|
||||||
self.synchronous_close()
|
# self.synchronous_close()
|
||||||
raise
|
raise
|
||||||
finally:
|
finally:
|
||||||
self.pending_amount -= 1
|
self._concurrency.release()
|
||||||
|
|
||||||
async def ensure_session(self):
|
|
||||||
# Handles reconnecting and maintaining a session alive
|
|
||||||
# TODO: change to 'ping' on newer protocol (above 1.2)
|
|
||||||
retry_delay = default_delay = 1.0
|
|
||||||
while True:
|
|
||||||
try:
|
|
||||||
if self.is_closing():
|
|
||||||
await self.create_connection(self.timeout)
|
|
||||||
await self.ensure_server_version()
|
|
||||||
self._on_connect_cb()
|
|
||||||
if (perf_counter() - self.last_send) > self.max_seconds_idle or self.response_time is None:
|
|
||||||
await self.ensure_server_version()
|
|
||||||
retry_delay = default_delay
|
|
||||||
except RPCError as e:
|
|
||||||
await self.close()
|
|
||||||
log.debug("Server error, ignoring for 1h: %s:%d -- %s", *self.server, e.message)
|
|
||||||
retry_delay = 60 * 60
|
|
||||||
except IncompatibleWalletServerError:
|
|
||||||
await self.close()
|
|
||||||
retry_delay = 60 * 60
|
|
||||||
log.debug("Wallet server has an incompatible version, retrying in 1h: %s:%d", *self.server)
|
|
||||||
except (asyncio.TimeoutError, OSError):
|
|
||||||
await self.close()
|
|
||||||
retry_delay = min(60, retry_delay * 2)
|
|
||||||
log.debug("Wallet server timeout (retry in %s seconds): %s:%d", retry_delay, *self.server)
|
|
||||||
try:
|
|
||||||
await asyncio.wait_for(self.trigger_urgent_reconnect.wait(), timeout=retry_delay)
|
|
||||||
except asyncio.TimeoutError:
|
|
||||||
pass
|
|
||||||
finally:
|
|
||||||
self.trigger_urgent_reconnect.clear()
|
|
||||||
|
|
||||||
async def ensure_server_version(self, required=None, timeout=3):
|
async def ensure_server_version(self, required=None, timeout=3):
|
||||||
required = required or self.network.PROTOCOL_VERSION
|
required = required or self.network.PROTOCOL_VERSION
|
||||||
@@ -133,6 +110,25 @@ class ClientSession(BaseClientSession):
|
|||||||
raise IncompatibleWalletServerError(*self.server)
|
raise IncompatibleWalletServerError(*self.server)
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
async def keepalive_loop(self, timeout=3, max_idle=60):
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
now = perf_counter()
|
||||||
|
if min(self.last_send, self.last_packet_received) + max_idle < now:
|
||||||
|
await asyncio.wait_for(
|
||||||
|
self.send_request('server.ping', []), timeout=timeout
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
await asyncio.sleep(max(0, max_idle - (now - self.last_send)))
|
||||||
|
except Exception as err:
|
||||||
|
if isinstance(err, asyncio.CancelledError):
|
||||||
|
log.warning("closing connection to %s:%i", *self.server)
|
||||||
|
else:
|
||||||
|
log.exception("lost connection to spv")
|
||||||
|
finally:
|
||||||
|
if not self.is_closing():
|
||||||
|
self._close()
|
||||||
|
|
||||||
async def create_connection(self, timeout=6):
|
async def create_connection(self, timeout=6):
|
||||||
connector = Connector(lambda: self, *self.server)
|
connector = Connector(lambda: self, *self.server)
|
||||||
start = perf_counter()
|
start = perf_counter()
|
||||||
@@ -144,28 +140,28 @@ class ClientSession(BaseClientSession):
|
|||||||
controller.add(request.args)
|
controller.add(request.args)
|
||||||
|
|
||||||
def connection_lost(self, exc):
|
def connection_lost(self, exc):
|
||||||
log.debug("Connection lost: %s:%d", *self.server)
|
log.warning("Connection lost: %s:%d", *self.server)
|
||||||
super().connection_lost(exc)
|
super().connection_lost(exc)
|
||||||
self.response_time = None
|
self.response_time = None
|
||||||
self.connection_latency = None
|
self.connection_latency = None
|
||||||
self._response_samples = 0
|
self._response_samples = 0
|
||||||
self.pending_amount = 0
|
# self._on_disconnect_controller.add(True)
|
||||||
self._on_disconnect_controller.add(True)
|
if self.network:
|
||||||
|
self.network.disconnect()
|
||||||
|
|
||||||
|
|
||||||
class Network:
|
class Network:
|
||||||
|
|
||||||
PROTOCOL_VERSION = __version__
|
PROTOCOL_VERSION = __version__
|
||||||
MINIMUM_REQUIRED = (0, 59, 0)
|
MINIMUM_REQUIRED = (0, 65, 0)
|
||||||
|
|
||||||
def __init__(self, ledger):
|
def __init__(self, ledger):
|
||||||
self.ledger = ledger
|
self.ledger = ledger
|
||||||
self.session_pool = SessionPool(network=self, timeout=self.config.get('connect_timeout', 6))
|
|
||||||
self.client: Optional[ClientSession] = None
|
self.client: Optional[ClientSession] = None
|
||||||
self._switch_task: Optional[asyncio.Task] = None
|
self.server_features = None
|
||||||
|
# self._switch_task: Optional[asyncio.Task] = None
|
||||||
self.running = False
|
self.running = False
|
||||||
self.remote_height: int = 0
|
self.remote_height: int = 0
|
||||||
self._concurrency = asyncio.Semaphore(16)
|
|
||||||
|
|
||||||
self._on_connected_controller = StreamController()
|
self._on_connected_controller = StreamController()
|
||||||
self.on_connected = self._on_connected_controller.stream
|
self.on_connected = self._on_connected_controller.stream
|
||||||
@@ -176,82 +172,256 @@ class Network:
|
|||||||
self._on_status_controller = StreamController(merge_repeated_events=True)
|
self._on_status_controller = StreamController(merge_repeated_events=True)
|
||||||
self.on_status = self._on_status_controller.stream
|
self.on_status = self._on_status_controller.stream
|
||||||
|
|
||||||
|
self._on_hub_controller = StreamController(merge_repeated_events=True)
|
||||||
|
self.on_hub = self._on_hub_controller.stream
|
||||||
|
|
||||||
self.subscription_controllers = {
|
self.subscription_controllers = {
|
||||||
'blockchain.headers.subscribe': self._on_header_controller,
|
'blockchain.headers.subscribe': self._on_header_controller,
|
||||||
'blockchain.address.subscribe': self._on_status_controller,
|
'blockchain.address.subscribe': self._on_status_controller,
|
||||||
|
'blockchain.peers.subscribe': self._on_hub_controller,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
self.aiohttp_session: Optional[aiohttp.ClientSession] = None
|
||||||
|
self._urgent_need_reconnect = asyncio.Event()
|
||||||
|
self._loop_task: Optional[asyncio.Task] = None
|
||||||
|
self._keepalive_task: Optional[asyncio.Task] = None
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def config(self):
|
def config(self):
|
||||||
return self.ledger.config
|
return self.ledger.config
|
||||||
|
|
||||||
async def switch_forever(self):
|
@property
|
||||||
while self.running:
|
def known_hubs(self):
|
||||||
if self.is_connected:
|
if 'known_hubs' not in self.config:
|
||||||
await self.client.on_disconnected.first
|
return KnownHubsList()
|
||||||
self.client = None
|
return self.config['known_hubs']
|
||||||
continue
|
|
||||||
self.client = await self.session_pool.wait_for_fastest_session()
|
@property
|
||||||
log.info("Switching to SPV wallet server: %s:%d", *self.client.server)
|
def jurisdiction(self):
|
||||||
try:
|
return self.config.get("jurisdiction")
|
||||||
self._update_remote_height((await self.subscribe_headers(),))
|
|
||||||
self._on_connected_controller.add(True)
|
def disconnect(self):
|
||||||
log.info("Subscribed to headers: %s:%d", *self.client.server)
|
if self._keepalive_task and not self._keepalive_task.done():
|
||||||
except (asyncio.TimeoutError, ConnectionError):
|
self._keepalive_task.cancel()
|
||||||
log.info("Switching to %s:%d timed out, closing and retrying.", *self.client.server)
|
self._keepalive_task = None
|
||||||
self.client.synchronous_close()
|
|
||||||
self.client = None
|
|
||||||
|
|
||||||
async def start(self):
|
async def start(self):
|
||||||
self.running = True
|
if not self.running:
|
||||||
self._switch_task = asyncio.ensure_future(self.switch_forever())
|
self.running = True
|
||||||
# this may become unnecessary when there are no more bugs found,
|
self.aiohttp_session = aiohttp.ClientSession()
|
||||||
# but for now it helps understanding log reports
|
self.on_header.listen(self._update_remote_height)
|
||||||
self._switch_task.add_done_callback(lambda _: log.info("Wallet client switching task stopped."))
|
self.on_hub.listen(self._update_hubs)
|
||||||
self.session_pool.start(self.config['default_servers'])
|
self._loop_task = asyncio.create_task(self.network_loop())
|
||||||
self.on_header.listen(self._update_remote_height)
|
self._urgent_need_reconnect.set()
|
||||||
|
|
||||||
|
def loop_task_done_callback(f):
|
||||||
|
try:
|
||||||
|
f.result()
|
||||||
|
except Exception:
|
||||||
|
if self.running:
|
||||||
|
log.exception("wallet server connection loop crashed")
|
||||||
|
|
||||||
|
self._loop_task.add_done_callback(loop_task_done_callback)
|
||||||
|
|
||||||
|
async def resolve_spv_dns(self):
|
||||||
|
hostname_to_ip = {}
|
||||||
|
ip_to_hostnames = defaultdict(list)
|
||||||
|
|
||||||
|
async def resolve_spv(server, port):
|
||||||
|
try:
|
||||||
|
server_addr = await resolve_host(server, port, 'udp')
|
||||||
|
hostname_to_ip[server] = (server_addr, port)
|
||||||
|
ip_to_hostnames[(server_addr, port)].append(server)
|
||||||
|
except socket.error:
|
||||||
|
log.warning("error looking up dns for spv server %s:%i", server, port)
|
||||||
|
except Exception:
|
||||||
|
log.exception("error looking up dns for spv server %s:%i", server, port)
|
||||||
|
|
||||||
|
# accumulate the dns results
|
||||||
|
if self.config['explicit_servers']:
|
||||||
|
hubs = self.config['explicit_servers']
|
||||||
|
elif self.known_hubs:
|
||||||
|
hubs = self.known_hubs
|
||||||
|
else:
|
||||||
|
hubs = self.config['default_servers']
|
||||||
|
await asyncio.gather(*(resolve_spv(server, port) for (server, port) in hubs))
|
||||||
|
return hostname_to_ip, ip_to_hostnames
|
||||||
|
|
||||||
|
async def get_n_fastest_spvs(self, timeout=3.0) -> Dict[Tuple[str, int], Optional[SPVPong]]:
|
||||||
|
loop = asyncio.get_event_loop()
|
||||||
|
pong_responses = asyncio.Queue()
|
||||||
|
connection = SPVStatusClientProtocol(pong_responses)
|
||||||
|
sent_ping_timestamps = {}
|
||||||
|
_, ip_to_hostnames = await self.resolve_spv_dns()
|
||||||
|
n = len(ip_to_hostnames)
|
||||||
|
log.info("%i possible spv servers to try (%i urls in config)", n, len(self.config['explicit_servers']))
|
||||||
|
pongs = {}
|
||||||
|
known_hubs = self.known_hubs
|
||||||
|
try:
|
||||||
|
await loop.create_datagram_endpoint(lambda: connection, ('0.0.0.0', 0))
|
||||||
|
# could raise OSError if it cant bind
|
||||||
|
start = perf_counter()
|
||||||
|
for server in ip_to_hostnames:
|
||||||
|
connection.ping(server)
|
||||||
|
sent_ping_timestamps[server] = perf_counter()
|
||||||
|
while len(pongs) < n:
|
||||||
|
(remote, ts), pong = await asyncio.wait_for(pong_responses.get(), timeout - (perf_counter() - start))
|
||||||
|
latency = ts - start
|
||||||
|
log.info("%s:%i has latency of %sms (available: %s, height: %i)",
|
||||||
|
'/'.join(ip_to_hostnames[remote]), remote[1], round(latency * 1000, 2),
|
||||||
|
pong.available, pong.height)
|
||||||
|
|
||||||
|
known_hubs.hubs.setdefault((ip_to_hostnames[remote][0], remote[1]), {}).update(
|
||||||
|
{"country": pong.country_name}
|
||||||
|
)
|
||||||
|
if pong.available:
|
||||||
|
pongs[(ip_to_hostnames[remote][0], remote[1])] = pong
|
||||||
|
return pongs
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
if pongs:
|
||||||
|
log.info("%i/%i probed spv servers are accepting connections", len(pongs), len(ip_to_hostnames))
|
||||||
|
return pongs
|
||||||
|
else:
|
||||||
|
log.warning("%i spv status probes failed, retrying later. servers tried: %s",
|
||||||
|
len(sent_ping_timestamps),
|
||||||
|
', '.join('/'.join(hosts) + f' ({ip})' for ip, hosts in ip_to_hostnames.items()))
|
||||||
|
random_server = random.choice(list(ip_to_hostnames.keys()))
|
||||||
|
host, port = random_server
|
||||||
|
log.warning("trying fallback to randomly selected spv: %s:%i", host, port)
|
||||||
|
known_hubs.hubs.setdefault((host, port), {})
|
||||||
|
return {(host, port): None}
|
||||||
|
finally:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
async def connect_to_fastest(self) -> Optional[ClientSession]:
|
||||||
|
fastest_spvs = await self.get_n_fastest_spvs()
|
||||||
|
for (host, port), pong in fastest_spvs.items():
|
||||||
|
if (pong is not None and self.jurisdiction is not None) and \
|
||||||
|
(pong.country_name != self.jurisdiction):
|
||||||
|
continue
|
||||||
|
client = ClientSession(network=self, server=(host, port), timeout=self.config['hub_timeout'],
|
||||||
|
concurrency=self.config['concurrent_hub_requests'])
|
||||||
|
try:
|
||||||
|
await client.create_connection()
|
||||||
|
log.warning("Connected to spv server %s:%i", host, port)
|
||||||
|
await client.ensure_server_version()
|
||||||
|
return client
|
||||||
|
except (asyncio.TimeoutError, ConnectionError, OSError, IncompatibleWalletServerError, RPCError):
|
||||||
|
log.warning("Connecting to %s:%d failed", host, port)
|
||||||
|
client._close()
|
||||||
|
return
|
||||||
|
|
||||||
|
async def network_loop(self):
|
||||||
|
sleep_delay = 30
|
||||||
|
while self.running:
|
||||||
|
await asyncio.wait(
|
||||||
|
[asyncio.sleep(30), self._urgent_need_reconnect.wait()], return_when=asyncio.FIRST_COMPLETED
|
||||||
|
)
|
||||||
|
if self._urgent_need_reconnect.is_set():
|
||||||
|
sleep_delay = 30
|
||||||
|
self._urgent_need_reconnect.clear()
|
||||||
|
if not self.is_connected:
|
||||||
|
client = await self.connect_to_fastest()
|
||||||
|
if not client:
|
||||||
|
log.warning("failed to connect to any spv servers, retrying later")
|
||||||
|
sleep_delay *= 2
|
||||||
|
sleep_delay = min(sleep_delay, 300)
|
||||||
|
continue
|
||||||
|
log.debug("get spv server features %s:%i", *client.server)
|
||||||
|
features = await client.send_request('server.features', [])
|
||||||
|
self.client, self.server_features = client, features
|
||||||
|
log.debug("discover other hubs %s:%i", *client.server)
|
||||||
|
await self._update_hubs(await client.send_request('server.peers.subscribe', []))
|
||||||
|
log.info("subscribe to headers %s:%i", *client.server)
|
||||||
|
self._update_remote_height((await self.subscribe_headers(),))
|
||||||
|
self._on_connected_controller.add(True)
|
||||||
|
server_str = "%s:%i" % client.server
|
||||||
|
log.info("maintaining connection to spv server %s", server_str)
|
||||||
|
self._keepalive_task = asyncio.create_task(self.client.keepalive_loop())
|
||||||
|
try:
|
||||||
|
if not self._urgent_need_reconnect.is_set():
|
||||||
|
await asyncio.wait(
|
||||||
|
[self._keepalive_task, self._urgent_need_reconnect.wait()],
|
||||||
|
return_when=asyncio.FIRST_COMPLETED
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
await self._keepalive_task
|
||||||
|
if self._urgent_need_reconnect.is_set():
|
||||||
|
log.warning("urgent reconnect needed")
|
||||||
|
self._urgent_need_reconnect.clear()
|
||||||
|
if self._keepalive_task and not self._keepalive_task.done():
|
||||||
|
self._keepalive_task.cancel()
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
pass
|
||||||
|
finally:
|
||||||
|
self._keepalive_task = None
|
||||||
|
self.client = None
|
||||||
|
self.server_features = None
|
||||||
|
log.warning("connection lost to %s", server_str)
|
||||||
|
log.info("network loop finished")
|
||||||
|
|
||||||
async def stop(self):
|
async def stop(self):
|
||||||
if self.running:
|
self.running = False
|
||||||
self.running = False
|
self.disconnect()
|
||||||
self._switch_task.cancel()
|
if self._loop_task and not self._loop_task.done():
|
||||||
self.session_pool.stop()
|
self._loop_task.cancel()
|
||||||
|
self._loop_task = None
|
||||||
|
if self.aiohttp_session:
|
||||||
|
await self.aiohttp_session.close()
|
||||||
|
self.aiohttp_session = None
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def is_connected(self):
|
def is_connected(self):
|
||||||
return self.client and not self.client.is_closing()
|
return self.client and not self.client.is_closing()
|
||||||
|
|
||||||
def rpc(self, list_or_method, args, restricted=True):
|
def rpc(self, list_or_method, args, restricted=True, session: Optional[ClientSession] = None):
|
||||||
session = self.client if restricted else self.session_pool.fastest_session
|
if session or self.is_connected:
|
||||||
if session and not session.is_closing():
|
session = session or self.client
|
||||||
return session.send_request(list_or_method, args)
|
return session.send_request(list_or_method, args)
|
||||||
else:
|
else:
|
||||||
self.session_pool.trigger_nodelay_connect()
|
self._urgent_need_reconnect.set()
|
||||||
raise ConnectionError("Attempting to send rpc request when connection is not available.")
|
raise ConnectionError("Attempting to send rpc request when connection is not available.")
|
||||||
|
|
||||||
async def retriable_call(self, function, *args, **kwargs):
|
async def retriable_call(self, function, *args, **kwargs):
|
||||||
async with self._concurrency:
|
while self.running:
|
||||||
while self.running:
|
if not self.is_connected:
|
||||||
if not self.is_connected:
|
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
|
||||||
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
|
self._urgent_need_reconnect.set()
|
||||||
await self.on_connected.first
|
await self.on_connected.first
|
||||||
await self.session_pool.wait_for_fastest_session()
|
try:
|
||||||
try:
|
return await function(*args, **kwargs)
|
||||||
return await function(*args, **kwargs)
|
except asyncio.TimeoutError:
|
||||||
except asyncio.TimeoutError:
|
log.warning("Wallet server call timed out, retrying.")
|
||||||
log.warning("Wallet server call timed out, retrying.")
|
except ConnectionError:
|
||||||
except ConnectionError:
|
log.warning("connection error")
|
||||||
pass
|
|
||||||
raise asyncio.CancelledError() # if we got here, we are shutting down
|
raise asyncio.CancelledError() # if we got here, we are shutting down
|
||||||
|
|
||||||
def _update_remote_height(self, header_args):
|
def _update_remote_height(self, header_args):
|
||||||
self.remote_height = header_args[0]["height"]
|
self.remote_height = header_args[0]["height"]
|
||||||
|
|
||||||
|
async def _update_hubs(self, hubs):
|
||||||
|
if hubs and hubs != ['']:
|
||||||
|
try:
|
||||||
|
if self.known_hubs.add_hubs(hubs):
|
||||||
|
self.known_hubs.save()
|
||||||
|
except Exception:
|
||||||
|
log.exception("could not add hubs: %s", hubs)
|
||||||
|
|
||||||
def get_transaction(self, tx_hash, known_height=None):
|
def get_transaction(self, tx_hash, known_height=None):
|
||||||
# use any server if its old, otherwise restrict to who gave us the history
|
# use any server if its old, otherwise restrict to who gave us the history
|
||||||
restricted = known_height in (None, -1, 0) or 0 > known_height > self.remote_height - 10
|
restricted = known_height in (None, -1, 0) or 0 > known_height > self.remote_height - 10
|
||||||
return self.rpc('blockchain.transaction.get', [tx_hash], restricted)
|
return self.rpc('blockchain.transaction.get', [tx_hash], restricted)
|
||||||
|
|
||||||
|
def get_transaction_batch(self, txids, restricted=True):
|
||||||
|
# use any server if its old, otherwise restrict to who gave us the history
|
||||||
|
return self.rpc('blockchain.transaction.get_batch', txids, restricted)
|
||||||
|
|
||||||
|
def get_transaction_and_merkle(self, tx_hash, known_height=None):
|
||||||
|
# use any server if its old, otherwise restrict to who gave us the history
|
||||||
|
restricted = known_height in (None, -1, 0) or 0 > known_height > self.remote_height - 10
|
||||||
|
return self.rpc('blockchain.transaction.info', [tx_hash], restricted)
|
||||||
|
|
||||||
def get_transaction_height(self, tx_hash, known_height=None):
|
def get_transaction_height(self, tx_hash, known_height=None):
|
||||||
restricted = not known_height or 0 > known_height > self.remote_height - 10
|
restricted = not known_height or 0 > known_height > self.remote_height - 10
|
||||||
return self.rpc('blockchain.transaction.get_height', [tx_hash], restricted)
|
return self.rpc('blockchain.transaction.get_height', [tx_hash], restricted)
|
||||||
@@ -276,12 +446,13 @@ class Network:
|
|||||||
|
|
||||||
async def subscribe_address(self, address, *addresses):
|
async def subscribe_address(self, address, *addresses):
|
||||||
addresses = list((address, ) + addresses)
|
addresses = list((address, ) + addresses)
|
||||||
|
server_addr_and_port = self.client.server_address_and_port # on disconnect client will be None
|
||||||
try:
|
try:
|
||||||
return await self.rpc('blockchain.address.subscribe', addresses, True)
|
return await self.rpc('blockchain.address.subscribe', addresses, True)
|
||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
log.warning(
|
log.warning(
|
||||||
"timed out subscribing to addresses from %s:%i",
|
"timed out subscribing to addresses from %s:%i",
|
||||||
*self.client.server_address_and_port
|
*server_addr_and_port
|
||||||
)
|
)
|
||||||
# abort and cancel, we can't lose a subscription, it will happen again on reconnect
|
# abort and cancel, we can't lose a subscription, it will happen again on reconnect
|
||||||
if self.client:
|
if self.client:
|
||||||
@@ -294,103 +465,29 @@ class Network:
|
|||||||
def get_server_features(self):
|
def get_server_features(self):
|
||||||
return self.rpc('server.features', (), restricted=True)
|
return self.rpc('server.features', (), restricted=True)
|
||||||
|
|
||||||
def get_claims_by_ids(self, claim_ids):
|
def resolve(self, urls, session_override=None):
|
||||||
return self.rpc('blockchain.claimtrie.getclaimsbyids', claim_ids)
|
return self.rpc('blockchain.claimtrie.resolve', urls, False, session_override)
|
||||||
|
|
||||||
def resolve(self, urls):
|
def claim_search(self, session_override=None, **kwargs):
|
||||||
return self.rpc('blockchain.claimtrie.resolve', urls)
|
return self.rpc('blockchain.claimtrie.search', kwargs, False, session_override)
|
||||||
|
|
||||||
def claim_search(self, **kwargs):
|
async def new_resolve(self, server, urls):
|
||||||
return self.rpc('blockchain.claimtrie.search', kwargs)
|
message = {"method": "resolve", "params": {"urls": urls, "protobuf": True}}
|
||||||
|
async with self.aiohttp_session.post(server, json=message) as r:
|
||||||
|
result = await r.json()
|
||||||
|
return result['result']
|
||||||
|
|
||||||
|
async def new_claim_search(self, server, **kwargs):
|
||||||
|
async with grpc.aio.insecure_channel(server) as channel:
|
||||||
|
stub = hub_pb2_grpc.HubStub(channel)
|
||||||
|
try:
|
||||||
|
response = await stub.Search(SearchRequest(**kwargs))
|
||||||
|
except grpc.aio.AioRpcError as error:
|
||||||
|
raise RPCError(error.code(), error.details())
|
||||||
|
return response
|
||||||
|
|
||||||
class SessionPool:
|
async def sum_supports(self, server, **kwargs):
|
||||||
|
message = {"method": "support_sum", "params": kwargs}
|
||||||
def __init__(self, network: Network, timeout: float):
|
async with self.aiohttp_session.post(server, json=message) as r:
|
||||||
self.network = network
|
result = await r.json()
|
||||||
self.sessions: Dict[ClientSession, Optional[asyncio.Task]] = dict()
|
return result['result']
|
||||||
self.timeout = timeout
|
|
||||||
self.new_connection_event = asyncio.Event()
|
|
||||||
|
|
||||||
@property
|
|
||||||
def online(self):
|
|
||||||
return any(not session.is_closing() for session in self.sessions)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def available_sessions(self):
|
|
||||||
return (session for session in self.sessions if session.available)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def fastest_session(self):
|
|
||||||
if not self.online:
|
|
||||||
return None
|
|
||||||
return min(
|
|
||||||
[((session.response_time + session.connection_latency) * (session.pending_amount + 1), session)
|
|
||||||
for session in self.available_sessions] or [(0, None)],
|
|
||||||
key=itemgetter(0)
|
|
||||||
)[1]
|
|
||||||
|
|
||||||
def _get_session_connect_callback(self, session: ClientSession):
|
|
||||||
loop = asyncio.get_event_loop()
|
|
||||||
|
|
||||||
def callback():
|
|
||||||
duplicate_connections = [
|
|
||||||
s for s in self.sessions
|
|
||||||
if s is not session and s.server_address_and_port == session.server_address_and_port
|
|
||||||
]
|
|
||||||
already_connected = None if not duplicate_connections else duplicate_connections[0]
|
|
||||||
if already_connected:
|
|
||||||
self.sessions.pop(session).cancel()
|
|
||||||
session.synchronous_close()
|
|
||||||
log.debug("wallet server %s resolves to the same server as %s, rechecking in an hour",
|
|
||||||
session.server[0], already_connected.server[0])
|
|
||||||
loop.call_later(3600, self._connect_session, session.server)
|
|
||||||
return
|
|
||||||
self.new_connection_event.set()
|
|
||||||
log.info("connected to %s:%i", *session.server)
|
|
||||||
|
|
||||||
return callback
|
|
||||||
|
|
||||||
def _connect_session(self, server: Tuple[str, int]):
|
|
||||||
session = None
|
|
||||||
for s in self.sessions:
|
|
||||||
if s.server == server:
|
|
||||||
session = s
|
|
||||||
break
|
|
||||||
if not session:
|
|
||||||
session = ClientSession(
|
|
||||||
network=self.network, server=server
|
|
||||||
)
|
|
||||||
session._on_connect_cb = self._get_session_connect_callback(session)
|
|
||||||
task = self.sessions.get(session, None)
|
|
||||||
if not task or task.done():
|
|
||||||
task = asyncio.create_task(session.ensure_session())
|
|
||||||
task.add_done_callback(lambda _: self.ensure_connections())
|
|
||||||
self.sessions[session] = task
|
|
||||||
|
|
||||||
def start(self, default_servers):
|
|
||||||
for server in default_servers:
|
|
||||||
self._connect_session(server)
|
|
||||||
|
|
||||||
def stop(self):
|
|
||||||
for session, task in self.sessions.items():
|
|
||||||
task.cancel()
|
|
||||||
session.synchronous_close()
|
|
||||||
self.sessions.clear()
|
|
||||||
|
|
||||||
def ensure_connections(self):
|
|
||||||
for session in self.sessions:
|
|
||||||
self._connect_session(session.server)
|
|
||||||
|
|
||||||
def trigger_nodelay_connect(self):
|
|
||||||
# used when other parts of the system sees we might have internet back
|
|
||||||
# bypasses the retry interval
|
|
||||||
for session in self.sessions:
|
|
||||||
session.trigger_urgent_reconnect.set()
|
|
||||||
|
|
||||||
async def wait_for_fastest_session(self):
|
|
||||||
while not self.fastest_session:
|
|
||||||
self.trigger_nodelay_connect()
|
|
||||||
self.new_connection_event.clear()
|
|
||||||
await self.new_connection_event.wait()
|
|
||||||
return self.fastest_session
|
|
||||||
@@ -1,2 +1,5 @@
|
|||||||
|
__hub_url__ = (
|
||||||
|
"https://github.com/lbryio/hub/releases/download/v0.2021.08.24-beta/hub"
|
||||||
|
)
|
||||||
from .node import Conductor
|
from .node import Conductor
|
||||||
from .service import ConductorService
|
from .service import ConductorService
|
||||||
+212
-15
@@ -8,15 +8,19 @@ import logging
|
|||||||
import tempfile
|
import tempfile
|
||||||
import subprocess
|
import subprocess
|
||||||
import importlib
|
import importlib
|
||||||
|
from distutils.util import strtobool
|
||||||
|
|
||||||
from binascii import hexlify
|
from binascii import hexlify
|
||||||
from typing import Type, Optional
|
from typing import Type, Optional
|
||||||
import urllib.request
|
import urllib.request
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
import lbry
|
import lbry
|
||||||
from lbry.wallet.server.server import Server
|
from lbry.wallet.server.server import Server
|
||||||
from lbry.wallet.server.env import Env
|
from lbry.wallet.server.env import Env
|
||||||
from lbry.wallet import Wallet, Ledger, RegTestLedger, WalletManager, Account, BlockHeightEvent
|
from lbry.wallet import Wallet, Ledger, RegTestLedger, WalletManager, Account, BlockHeightEvent
|
||||||
|
from lbry.conf import KnownHubsList, Config
|
||||||
|
from lbry.wallet.orchstr8 import __hub_url__
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -46,16 +50,19 @@ class Conductor:
|
|||||||
self.wallet_node = WalletNode(
|
self.wallet_node = WalletNode(
|
||||||
self.manager_module, RegTestLedger, default_seed=seed
|
self.manager_module, RegTestLedger, default_seed=seed
|
||||||
)
|
)
|
||||||
|
self.hub_node = HubNode(__hub_url__, "hub", self.spv_node)
|
||||||
|
|
||||||
self.blockchain_started = False
|
self.blockchain_started = False
|
||||||
self.spv_started = False
|
self.spv_started = False
|
||||||
self.wallet_started = False
|
self.wallet_started = False
|
||||||
|
self.hub_started = False
|
||||||
|
|
||||||
self.log = log.getChild('conductor')
|
self.log = log.getChild('conductor')
|
||||||
|
|
||||||
async def start_blockchain(self):
|
async def start_blockchain(self):
|
||||||
if not self.blockchain_started:
|
if not self.blockchain_started:
|
||||||
await self.blockchain_node.start()
|
asyncio.create_task(self.blockchain_node.start())
|
||||||
|
await self.blockchain_node.running.wait()
|
||||||
await self.blockchain_node.generate(200)
|
await self.blockchain_node.generate(200)
|
||||||
self.blockchain_started = True
|
self.blockchain_started = True
|
||||||
|
|
||||||
@@ -64,6 +71,17 @@ class Conductor:
|
|||||||
await self.blockchain_node.stop(cleanup=True)
|
await self.blockchain_node.stop(cleanup=True)
|
||||||
self.blockchain_started = False
|
self.blockchain_started = False
|
||||||
|
|
||||||
|
async def start_hub(self):
|
||||||
|
if not self.hub_started:
|
||||||
|
asyncio.create_task(self.hub_node.start())
|
||||||
|
await self.blockchain_node.running.wait()
|
||||||
|
self.hub_started = True
|
||||||
|
|
||||||
|
async def stop_hub(self):
|
||||||
|
if self.hub_started:
|
||||||
|
await self.hub_node.stop(cleanup=True)
|
||||||
|
self.hub_started = False
|
||||||
|
|
||||||
async def start_spv(self):
|
async def start_spv(self):
|
||||||
if not self.spv_started:
|
if not self.spv_started:
|
||||||
await self.spv_node.start(self.blockchain_node)
|
await self.spv_node.start(self.blockchain_node)
|
||||||
@@ -105,7 +123,8 @@ class Conductor:
|
|||||||
class WalletNode:
|
class WalletNode:
|
||||||
|
|
||||||
def __init__(self, manager_class: Type[WalletManager], ledger_class: Type[Ledger],
|
def __init__(self, manager_class: Type[WalletManager], ledger_class: Type[Ledger],
|
||||||
verbose: bool = False, port: int = 5280, default_seed: str = None) -> None:
|
verbose: bool = False, port: int = 5280, default_seed: str = None,
|
||||||
|
data_path: str = None) -> None:
|
||||||
self.manager_class = manager_class
|
self.manager_class = manager_class
|
||||||
self.ledger_class = ledger_class
|
self.ledger_class = ledger_class
|
||||||
self.verbose = verbose
|
self.verbose = verbose
|
||||||
@@ -113,12 +132,12 @@ class WalletNode:
|
|||||||
self.ledger: Optional[Ledger] = None
|
self.ledger: Optional[Ledger] = None
|
||||||
self.wallet: Optional[Wallet] = None
|
self.wallet: Optional[Wallet] = None
|
||||||
self.account: Optional[Account] = None
|
self.account: Optional[Account] = None
|
||||||
self.data_path: Optional[str] = None
|
self.data_path: str = data_path or tempfile.mkdtemp()
|
||||||
self.port = port
|
self.port = port
|
||||||
self.default_seed = default_seed
|
self.default_seed = default_seed
|
||||||
|
self.known_hubs = KnownHubsList()
|
||||||
|
|
||||||
async def start(self, spv_node: 'SPVNode', seed=None, connect=True):
|
async def start(self, spv_node: 'SPVNode', seed=None, connect=True, config=None):
|
||||||
self.data_path = tempfile.mkdtemp()
|
|
||||||
wallets_dir = os.path.join(self.data_path, 'wallets')
|
wallets_dir = os.path.join(self.data_path, 'wallets')
|
||||||
os.mkdir(wallets_dir)
|
os.mkdir(wallets_dir)
|
||||||
wallet_file_name = os.path.join(wallets_dir, 'my_wallet.json')
|
wallet_file_name = os.path.join(wallets_dir, 'my_wallet.json')
|
||||||
@@ -127,13 +146,19 @@ class WalletNode:
|
|||||||
self.manager = self.manager_class.from_config({
|
self.manager = self.manager_class.from_config({
|
||||||
'ledgers': {
|
'ledgers': {
|
||||||
self.ledger_class.get_id(): {
|
self.ledger_class.get_id(): {
|
||||||
|
'use_go_hub': not strtobool(os.environ.get('ENABLE_LEGACY_SEARCH') or 'yes'),
|
||||||
'api_port': self.port,
|
'api_port': self.port,
|
||||||
'default_servers': [(spv_node.hostname, spv_node.port)],
|
'explicit_servers': [(spv_node.hostname, spv_node.port)],
|
||||||
'data_path': self.data_path
|
'default_servers': Config.lbryum_servers.default,
|
||||||
|
'data_path': self.data_path,
|
||||||
|
'known_hubs': config.known_hubs if config else KnownHubsList(),
|
||||||
|
'hub_timeout': 30,
|
||||||
|
'concurrent_hub_requests': 32,
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
'wallets': [wallet_file_name]
|
'wallets': [wallet_file_name]
|
||||||
})
|
})
|
||||||
|
self.manager.config = config
|
||||||
self.ledger = self.manager.ledgers[self.ledger_class]
|
self.ledger = self.manager.ledgers[self.ledger_class]
|
||||||
self.wallet = self.manager.default_wallet
|
self.wallet = self.manager.default_wallet
|
||||||
if not self.wallet:
|
if not self.wallet:
|
||||||
@@ -167,11 +192,15 @@ class SPVNode:
|
|||||||
self.server = None
|
self.server = None
|
||||||
self.hostname = 'localhost'
|
self.hostname = 'localhost'
|
||||||
self.port = 50001 + node_number # avoid conflict with default daemon
|
self.port = 50001 + node_number # avoid conflict with default daemon
|
||||||
|
self.udp_port = self.port
|
||||||
self.session_timeout = 600
|
self.session_timeout = 600
|
||||||
self.rpc_port = '0' # disabled by default
|
self.rpc_port = '0' # disabled by default
|
||||||
|
self.stopped = False
|
||||||
|
self.index_name = None
|
||||||
|
|
||||||
async def start(self, blockchain_node: 'BlockchainNode', extraconf=None):
|
async def start(self, blockchain_node: 'BlockchainNode', extraconf=None):
|
||||||
self.data_path = tempfile.mkdtemp()
|
self.data_path = tempfile.mkdtemp()
|
||||||
|
self.index_name = uuid4().hex
|
||||||
conf = {
|
conf = {
|
||||||
'DESCRIPTION': '',
|
'DESCRIPTION': '',
|
||||||
'PAYMENT_ADDRESS': '',
|
'PAYMENT_ADDRESS': '',
|
||||||
@@ -181,10 +210,13 @@ class SPVNode:
|
|||||||
'REORG_LIMIT': '100',
|
'REORG_LIMIT': '100',
|
||||||
'HOST': self.hostname,
|
'HOST': self.hostname,
|
||||||
'TCP_PORT': str(self.port),
|
'TCP_PORT': str(self.port),
|
||||||
|
'UDP_PORT': str(self.udp_port),
|
||||||
'SESSION_TIMEOUT': str(self.session_timeout),
|
'SESSION_TIMEOUT': str(self.session_timeout),
|
||||||
'MAX_QUERY_WORKERS': '0',
|
'MAX_QUERY_WORKERS': '0',
|
||||||
'INDIVIDUAL_TAG_INDEXES': '',
|
'INDIVIDUAL_TAG_INDEXES': '',
|
||||||
'RPC_PORT': self.rpc_port
|
'RPC_PORT': self.rpc_port,
|
||||||
|
'ES_INDEX_PREFIX': self.index_name,
|
||||||
|
'ES_MODE': 'writer',
|
||||||
}
|
}
|
||||||
if extraconf:
|
if extraconf:
|
||||||
conf.update(extraconf)
|
conf.update(extraconf)
|
||||||
@@ -195,8 +227,13 @@ class SPVNode:
|
|||||||
await self.server.start()
|
await self.server.start()
|
||||||
|
|
||||||
async def stop(self, cleanup=True):
|
async def stop(self, cleanup=True):
|
||||||
|
if self.stopped:
|
||||||
|
return
|
||||||
try:
|
try:
|
||||||
|
await self.server.db.search_index.delete_index()
|
||||||
|
await self.server.db.search_index.stop()
|
||||||
await self.server.stop()
|
await self.server.stop()
|
||||||
|
self.stopped = True
|
||||||
finally:
|
finally:
|
||||||
cleanup and self.cleanup()
|
cleanup and self.cleanup()
|
||||||
|
|
||||||
@@ -255,6 +292,10 @@ class BlockchainNode:
|
|||||||
self.rpcport = 9245 + 2 # avoid conflict with default rpc port
|
self.rpcport = 9245 + 2 # avoid conflict with default rpc port
|
||||||
self.rpcuser = 'rpcuser'
|
self.rpcuser = 'rpcuser'
|
||||||
self.rpcpassword = 'rpcpassword'
|
self.rpcpassword = 'rpcpassword'
|
||||||
|
self.stopped = False
|
||||||
|
self.restart_ready = asyncio.Event()
|
||||||
|
self.restart_ready.set()
|
||||||
|
self.running = asyncio.Event()
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def rpc_url(self):
|
def rpc_url(self):
|
||||||
@@ -315,13 +356,27 @@ class BlockchainNode:
|
|||||||
f'-port={self.peerport}'
|
f'-port={self.peerport}'
|
||||||
]
|
]
|
||||||
self.log.info(' '.join(command))
|
self.log.info(' '.join(command))
|
||||||
self.transport, self.protocol = await loop.subprocess_exec(
|
while not self.stopped:
|
||||||
BlockchainProcess, *command
|
if self.running.is_set():
|
||||||
)
|
await asyncio.sleep(1)
|
||||||
await self.protocol.ready.wait()
|
continue
|
||||||
assert not self.protocol.stopped.is_set()
|
await self.restart_ready.wait()
|
||||||
|
try:
|
||||||
|
self.transport, self.protocol = await loop.subprocess_exec(
|
||||||
|
BlockchainProcess, *command
|
||||||
|
)
|
||||||
|
await self.protocol.ready.wait()
|
||||||
|
assert not self.protocol.stopped.is_set()
|
||||||
|
self.running.set()
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
self.running.clear()
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
self.running.clear()
|
||||||
|
log.exception('failed to start lbrycrdd', exc_info=e)
|
||||||
|
|
||||||
async def stop(self, cleanup=True):
|
async def stop(self, cleanup=True):
|
||||||
|
self.stopped = True
|
||||||
try:
|
try:
|
||||||
self.transport.terminate()
|
self.transport.terminate()
|
||||||
await self.protocol.stopped.wait()
|
await self.protocol.stopped.wait()
|
||||||
@@ -330,6 +385,16 @@ class BlockchainNode:
|
|||||||
if cleanup:
|
if cleanup:
|
||||||
self.cleanup()
|
self.cleanup()
|
||||||
|
|
||||||
|
async def clear_mempool(self):
|
||||||
|
self.restart_ready.clear()
|
||||||
|
self.transport.terminate()
|
||||||
|
await self.protocol.stopped.wait()
|
||||||
|
self.transport.close()
|
||||||
|
self.running.clear()
|
||||||
|
os.remove(os.path.join(self.data_path, 'regtest', 'mempool.dat'))
|
||||||
|
self.restart_ready.set()
|
||||||
|
await self.running.wait()
|
||||||
|
|
||||||
def cleanup(self):
|
def cleanup(self):
|
||||||
shutil.rmtree(self.data_path, ignore_errors=True)
|
shutil.rmtree(self.data_path, ignore_errors=True)
|
||||||
|
|
||||||
@@ -361,6 +426,12 @@ class BlockchainNode:
|
|||||||
def get_block_hash(self, block):
|
def get_block_hash(self, block):
|
||||||
return self._cli_cmnd('getblockhash', str(block))
|
return self._cli_cmnd('getblockhash', str(block))
|
||||||
|
|
||||||
|
def sendrawtransaction(self, tx):
|
||||||
|
return self._cli_cmnd('sendrawtransaction', tx)
|
||||||
|
|
||||||
|
async def get_block(self, block_hash):
|
||||||
|
return json.loads(await self._cli_cmnd('getblock', block_hash, '1'))
|
||||||
|
|
||||||
def get_raw_change_address(self):
|
def get_raw_change_address(self):
|
||||||
return self._cli_cmnd('getrawchangeaddress')
|
return self._cli_cmnd('getrawchangeaddress')
|
||||||
|
|
||||||
@@ -368,7 +439,7 @@ class BlockchainNode:
|
|||||||
return self._cli_cmnd('getnewaddress', "", address_type)
|
return self._cli_cmnd('getnewaddress', "", address_type)
|
||||||
|
|
||||||
async def get_balance(self):
|
async def get_balance(self):
|
||||||
return float(await self._cli_cmnd('getbalance'))
|
return await self._cli_cmnd('getbalance')
|
||||||
|
|
||||||
def send_to_address(self, address, amount):
|
def send_to_address(self, address, amount):
|
||||||
return self._cli_cmnd('sendtoaddress', address, str(amount))
|
return self._cli_cmnd('sendtoaddress', address, str(amount))
|
||||||
@@ -387,3 +458,129 @@ class BlockchainNode:
|
|||||||
|
|
||||||
def get_raw_transaction(self, txid):
|
def get_raw_transaction(self, txid):
|
||||||
return self._cli_cmnd('getrawtransaction', txid, '1')
|
return self._cli_cmnd('getrawtransaction', txid, '1')
|
||||||
|
|
||||||
|
|
||||||
|
class HubProcess(asyncio.SubprocessProtocol):
|
||||||
|
def __init__(self):
|
||||||
|
self.ready = asyncio.Event()
|
||||||
|
self.stopped = asyncio.Event()
|
||||||
|
self.log = log.getChild('hub')
|
||||||
|
|
||||||
|
def pipe_data_received(self, fd, data):
|
||||||
|
if self.log:
|
||||||
|
self.log.info(data.decode())
|
||||||
|
if b'error' in data.lower():
|
||||||
|
self.ready.set()
|
||||||
|
raise SystemError(data.decode())
|
||||||
|
if b'listening on' in data:
|
||||||
|
self.ready.set()
|
||||||
|
|
||||||
|
def process_exited(self):
|
||||||
|
self.stopped.set()
|
||||||
|
self.ready.set()
|
||||||
|
|
||||||
|
|
||||||
|
class HubNode:
|
||||||
|
|
||||||
|
def __init__(self, url, daemon, spv_node):
|
||||||
|
self.spv_node = spv_node
|
||||||
|
self.debug = False
|
||||||
|
|
||||||
|
self.latest_release_url = url
|
||||||
|
self.project_dir = os.path.dirname(os.path.dirname(__file__))
|
||||||
|
self.bin_dir = os.path.join(self.project_dir, 'bin')
|
||||||
|
self.daemon_bin = os.path.join(self.bin_dir, daemon)
|
||||||
|
self.cli_bin = os.path.join(self.bin_dir, daemon)
|
||||||
|
self.log = log.getChild('hub')
|
||||||
|
self.transport = None
|
||||||
|
self.protocol = None
|
||||||
|
self.hostname = 'localhost'
|
||||||
|
self.rpcport = 50051 # avoid conflict with default rpc port
|
||||||
|
self.stopped = False
|
||||||
|
self.restart_ready = asyncio.Event()
|
||||||
|
self.restart_ready.set()
|
||||||
|
self.running = asyncio.Event()
|
||||||
|
|
||||||
|
@property
|
||||||
|
def exists(self):
|
||||||
|
return (
|
||||||
|
os.path.exists(self.cli_bin) and
|
||||||
|
os.path.exists(self.daemon_bin)
|
||||||
|
)
|
||||||
|
|
||||||
|
def download(self):
|
||||||
|
downloaded_file = os.path.join(
|
||||||
|
self.bin_dir,
|
||||||
|
self.latest_release_url[self.latest_release_url.rfind('/')+1:]
|
||||||
|
)
|
||||||
|
|
||||||
|
if not os.path.exists(self.bin_dir):
|
||||||
|
os.mkdir(self.bin_dir)
|
||||||
|
|
||||||
|
if not os.path.exists(downloaded_file):
|
||||||
|
self.log.info('Downloading: %s', self.latest_release_url)
|
||||||
|
with urllib.request.urlopen(self.latest_release_url) as response:
|
||||||
|
with open(downloaded_file, 'wb') as out_file:
|
||||||
|
shutil.copyfileobj(response, out_file)
|
||||||
|
|
||||||
|
self.log.info('Extracting: %s', downloaded_file)
|
||||||
|
|
||||||
|
if downloaded_file.endswith('.zip'):
|
||||||
|
with zipfile.ZipFile(downloaded_file) as dotzip:
|
||||||
|
dotzip.extractall(self.bin_dir)
|
||||||
|
# zipfile bug https://bugs.python.org/issue15795
|
||||||
|
os.chmod(self.cli_bin, 0o755)
|
||||||
|
os.chmod(self.daemon_bin, 0o755)
|
||||||
|
|
||||||
|
elif downloaded_file.endswith('.tar.gz'):
|
||||||
|
with tarfile.open(downloaded_file) as tar:
|
||||||
|
tar.extractall(self.bin_dir)
|
||||||
|
|
||||||
|
os.chmod(self.daemon_bin, 0o755)
|
||||||
|
|
||||||
|
return self.exists
|
||||||
|
|
||||||
|
def ensure(self):
|
||||||
|
return self.exists or self.download()
|
||||||
|
|
||||||
|
async def start(self):
|
||||||
|
assert self.ensure()
|
||||||
|
loop = asyncio.get_event_loop()
|
||||||
|
asyncio.get_child_watcher().attach_loop(loop)
|
||||||
|
command = [
|
||||||
|
self.daemon_bin, 'serve', '--esindex', self.spv_node.index_name + 'claims', '--debug'
|
||||||
|
]
|
||||||
|
self.log.info(' '.join(command))
|
||||||
|
while not self.stopped:
|
||||||
|
if self.running.is_set():
|
||||||
|
await asyncio.sleep(1)
|
||||||
|
continue
|
||||||
|
await self.restart_ready.wait()
|
||||||
|
try:
|
||||||
|
if not self.debug:
|
||||||
|
self.transport, self.protocol = await loop.subprocess_exec(
|
||||||
|
HubProcess, *command
|
||||||
|
)
|
||||||
|
await self.protocol.ready.wait()
|
||||||
|
assert not self.protocol.stopped.is_set()
|
||||||
|
self.running.set()
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
self.running.clear()
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
self.running.clear()
|
||||||
|
log.exception('failed to start hub', exc_info=e)
|
||||||
|
|
||||||
|
async def stop(self, cleanup=True):
|
||||||
|
self.stopped = True
|
||||||
|
try:
|
||||||
|
if not self.debug:
|
||||||
|
self.transport.terminate()
|
||||||
|
await self.protocol.stopped.wait()
|
||||||
|
self.transport.close()
|
||||||
|
finally:
|
||||||
|
if cleanup:
|
||||||
|
self.cleanup()
|
||||||
|
|
||||||
|
def cleanup(self):
|
||||||
|
pass
|
||||||
@@ -108,9 +108,6 @@ class Response:
|
|||||||
|
|
||||||
class CodeMessageError(Exception):
|
class CodeMessageError(Exception):
|
||||||
|
|
||||||
def __init__(self, code, message):
|
|
||||||
super().__init__(code, message)
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def code(self):
|
def code(self):
|
||||||
return self.args[0]
|
return self.args[0]
|
||||||
|
|||||||
+36
-10
@@ -33,14 +33,16 @@ from asyncio import Event, CancelledError
|
|||||||
import logging
|
import logging
|
||||||
import time
|
import time
|
||||||
from contextlib import suppress
|
from contextlib import suppress
|
||||||
|
from prometheus_client import Counter, Histogram
|
||||||
from lbry.wallet.tasks import TaskGroup
|
from lbry.wallet.tasks import TaskGroup
|
||||||
|
|
||||||
from .jsonrpc import Request, JSONRPCConnection, JSONRPCv2, JSONRPC, Batch, Notification
|
from .jsonrpc import Request, JSONRPCConnection, JSONRPCv2, JSONRPC, Batch, Notification
|
||||||
from .jsonrpc import RPCError, ProtocolError
|
from .jsonrpc import RPCError, ProtocolError
|
||||||
from .framing import BadMagicError, BadChecksumError, OversizedPayloadError, BitcoinFramer, NewlineFramer
|
from .framing import BadMagicError, BadChecksumError, OversizedPayloadError, BitcoinFramer, NewlineFramer
|
||||||
from .util import Concurrency
|
|
||||||
from lbry.wallet.server.prometheus import NOTIFICATION_COUNT, RESPONSE_TIMES, REQUEST_ERRORS_COUNT
|
HISTOGRAM_BUCKETS = (
|
||||||
|
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class Connector:
|
class Connector:
|
||||||
@@ -204,7 +206,7 @@ class SessionBase(asyncio.Protocol):
|
|||||||
"""
|
"""
|
||||||
return self._address
|
return self._address
|
||||||
|
|
||||||
def peer_address_str(self):
|
def peer_address_str(self, for_log=True):
|
||||||
"""Returns the peer's IP address and port as a human-readable
|
"""Returns the peer's IP address and port as a human-readable
|
||||||
string."""
|
string."""
|
||||||
if not self._address:
|
if not self._address:
|
||||||
@@ -373,10 +375,26 @@ class BatchRequest:
|
|||||||
raise BatchError(self)
|
raise BatchError(self)
|
||||||
|
|
||||||
|
|
||||||
|
NAMESPACE = "wallet_server"
|
||||||
|
|
||||||
|
|
||||||
class RPCSession(SessionBase):
|
class RPCSession(SessionBase):
|
||||||
"""Base class for protocols where a message can lead to a response,
|
"""Base class for protocols where a message can lead to a response,
|
||||||
for example JSON RPC."""
|
for example JSON RPC."""
|
||||||
|
|
||||||
|
RESPONSE_TIMES = Histogram("response_time", "Response times", namespace=NAMESPACE,
|
||||||
|
labelnames=("method", "version"), buckets=HISTOGRAM_BUCKETS)
|
||||||
|
NOTIFICATION_COUNT = Counter("notification", "Number of notifications sent (for subscriptions)",
|
||||||
|
namespace=NAMESPACE, labelnames=("method", "version"))
|
||||||
|
REQUEST_ERRORS_COUNT = Counter(
|
||||||
|
"request_error", "Number of requests that returned errors", namespace=NAMESPACE,
|
||||||
|
labelnames=("method", "version")
|
||||||
|
)
|
||||||
|
RESET_CONNECTIONS = Counter(
|
||||||
|
"reset_clients", "Number of reset connections by client version",
|
||||||
|
namespace=NAMESPACE, labelnames=("version",)
|
||||||
|
)
|
||||||
|
|
||||||
def __init__(self, *, framer=None, loop=None, connection=None):
|
def __init__(self, *, framer=None, loop=None, connection=None):
|
||||||
super().__init__(framer=framer, loop=loop)
|
super().__init__(framer=framer, loop=loop)
|
||||||
self.connection = connection or self.default_connection()
|
self.connection = connection or self.default_connection()
|
||||||
@@ -389,6 +407,7 @@ class RPCSession(SessionBase):
|
|||||||
except MemoryError:
|
except MemoryError:
|
||||||
self.logger.warning('received oversized message from %s:%s, dropping connection',
|
self.logger.warning('received oversized message from %s:%s, dropping connection',
|
||||||
self._address[0], self._address[1])
|
self._address[0], self._address[1])
|
||||||
|
self.RESET_CONNECTIONS.labels(version=self.client_version).inc()
|
||||||
self._close()
|
self._close()
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -417,12 +436,13 @@ class RPCSession(SessionBase):
|
|||||||
except CancelledError:
|
except CancelledError:
|
||||||
raise
|
raise
|
||||||
except Exception:
|
except Exception:
|
||||||
self.logger.exception(f'exception handling {request}')
|
reqstr = str(request)
|
||||||
|
self.logger.exception(f'exception handling {reqstr[:16_000]}')
|
||||||
result = RPCError(JSONRPC.INTERNAL_ERROR,
|
result = RPCError(JSONRPC.INTERNAL_ERROR,
|
||||||
'internal server error')
|
'internal server error')
|
||||||
if isinstance(request, Request):
|
if isinstance(request, Request):
|
||||||
message = request.send_result(result)
|
message = request.send_result(result)
|
||||||
RESPONSE_TIMES.labels(
|
self.RESPONSE_TIMES.labels(
|
||||||
method=request.method,
|
method=request.method,
|
||||||
version=self.client_version
|
version=self.client_version
|
||||||
).observe(time.perf_counter() - start)
|
).observe(time.perf_counter() - start)
|
||||||
@@ -430,7 +450,7 @@ class RPCSession(SessionBase):
|
|||||||
await self._send_message(message)
|
await self._send_message(message)
|
||||||
if isinstance(result, Exception):
|
if isinstance(result, Exception):
|
||||||
self._bump_errors()
|
self._bump_errors()
|
||||||
REQUEST_ERRORS_COUNT.labels(
|
self.REQUEST_ERRORS_COUNT.labels(
|
||||||
method=request.method,
|
method=request.method,
|
||||||
version=self.client_version
|
version=self.client_version
|
||||||
).inc()
|
).inc()
|
||||||
@@ -464,11 +484,17 @@ class RPCSession(SessionBase):
|
|||||||
raise result
|
raise result
|
||||||
return result
|
return result
|
||||||
|
|
||||||
async def send_notification(self, method, args=()):
|
async def send_notification(self, method, args=()) -> bool:
|
||||||
"""Send an RPC notification over the network."""
|
"""Send an RPC notification over the network."""
|
||||||
message = self.connection.send_notification(Notification(method, args))
|
message = self.connection.send_notification(Notification(method, args))
|
||||||
NOTIFICATION_COUNT.labels(method=method, version=self.client_version).inc()
|
self.NOTIFICATION_COUNT.labels(method=method, version=self.client_version).inc()
|
||||||
await self._send_message(message)
|
try:
|
||||||
|
await self._send_message(message)
|
||||||
|
return True
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
self.logger.info("timeout sending address notification to %s", self.peer_address_str(for_log=True))
|
||||||
|
self.abort()
|
||||||
|
return False
|
||||||
|
|
||||||
def send_batch(self, raise_errors=False):
|
def send_batch(self, raise_errors=False):
|
||||||
"""Return a BatchRequest. Intended to be used like so:
|
"""Return a BatchRequest. Intended to be used like so:
|
||||||
|
|||||||
@@ -32,10 +32,13 @@ import inspect
|
|||||||
|
|
||||||
# other_params: None means cannot be called with keyword arguments only
|
# other_params: None means cannot be called with keyword arguments only
|
||||||
# any means any name is good
|
# any means any name is good
|
||||||
|
from functools import lru_cache
|
||||||
|
|
||||||
SignatureInfo = namedtuple('SignatureInfo', 'min_args max_args '
|
SignatureInfo = namedtuple('SignatureInfo', 'min_args max_args '
|
||||||
'required_names other_names')
|
'required_names other_names')
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache(256)
|
||||||
def signature_info(func):
|
def signature_info(func):
|
||||||
params = inspect.signature(func).parameters
|
params = inspect.signature(func).parameters
|
||||||
min_args = max_args = 0
|
min_args = max_args = 0
|
||||||
|
|||||||
@@ -438,6 +438,17 @@ class OutputScript(Script):
|
|||||||
SUPPORT_CLAIM_OPCODES + PAY_SCRIPT_HASH.opcodes
|
SUPPORT_CLAIM_OPCODES + PAY_SCRIPT_HASH.opcodes
|
||||||
))
|
))
|
||||||
|
|
||||||
|
SUPPORT_CLAIM_DATA_OPCODES = (
|
||||||
|
OP_SUPPORT_CLAIM, PUSH_SINGLE('claim_name'), PUSH_SINGLE('claim_id'), PUSH_SINGLE('support'),
|
||||||
|
OP_2DROP, OP_2DROP
|
||||||
|
)
|
||||||
|
SUPPORT_CLAIM_DATA_PUBKEY = Template('support_claim+data+pay_pubkey_hash', (
|
||||||
|
SUPPORT_CLAIM_DATA_OPCODES + PAY_PUBKEY_HASH.opcodes
|
||||||
|
))
|
||||||
|
SUPPORT_CLAIM_DATA_SCRIPT = Template('support_claim+data+pay_script_hash', (
|
||||||
|
SUPPORT_CLAIM_DATA_OPCODES + PAY_SCRIPT_HASH.opcodes
|
||||||
|
))
|
||||||
|
|
||||||
UPDATE_CLAIM_OPCODES = (
|
UPDATE_CLAIM_OPCODES = (
|
||||||
OP_UPDATE_CLAIM, PUSH_SINGLE('claim_name'), PUSH_SINGLE('claim_id'), PUSH_SINGLE('claim'),
|
OP_UPDATE_CLAIM, PUSH_SINGLE('claim_name'), PUSH_SINGLE('claim_id'), PUSH_SINGLE('claim'),
|
||||||
OP_2DROP, OP_2DROP
|
OP_2DROP, OP_2DROP
|
||||||
@@ -474,6 +485,8 @@ class OutputScript(Script):
|
|||||||
CLAIM_NAME_SCRIPT,
|
CLAIM_NAME_SCRIPT,
|
||||||
SUPPORT_CLAIM_PUBKEY,
|
SUPPORT_CLAIM_PUBKEY,
|
||||||
SUPPORT_CLAIM_SCRIPT,
|
SUPPORT_CLAIM_SCRIPT,
|
||||||
|
SUPPORT_CLAIM_DATA_PUBKEY,
|
||||||
|
SUPPORT_CLAIM_DATA_SCRIPT,
|
||||||
UPDATE_CLAIM_PUBKEY,
|
UPDATE_CLAIM_PUBKEY,
|
||||||
UPDATE_CLAIM_SCRIPT,
|
UPDATE_CLAIM_SCRIPT,
|
||||||
SELL_CLAIM, SELL_SCRIPT,
|
SELL_CLAIM, SELL_SCRIPT,
|
||||||
@@ -527,6 +540,16 @@ class OutputScript(Script):
|
|||||||
'pubkey_hash': pubkey_hash
|
'pubkey_hash': pubkey_hash
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def pay_support_data_pubkey_hash(
|
||||||
|
cls, claim_name: bytes, claim_id: bytes, support, pubkey_hash: bytes):
|
||||||
|
return cls(template=cls.SUPPORT_CLAIM_DATA_PUBKEY, values={
|
||||||
|
'claim_name': claim_name,
|
||||||
|
'claim_id': claim_id,
|
||||||
|
'support': support,
|
||||||
|
'pubkey_hash': pubkey_hash
|
||||||
|
})
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def sell_script(cls, price):
|
def sell_script(cls, price):
|
||||||
return cls(template=cls.SELL_SCRIPT, values={
|
return cls(template=cls.SELL_SCRIPT, values={
|
||||||
@@ -575,6 +598,10 @@ class OutputScript(Script):
|
|||||||
def is_support_claim(self):
|
def is_support_claim(self):
|
||||||
return self.template.name.startswith('support_claim+')
|
return self.template.name.startswith('support_claim+')
|
||||||
|
|
||||||
|
@property
|
||||||
|
def is_support_claim_data(self):
|
||||||
|
return self.template.name.startswith('support_claim+data+')
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def is_sell_claim(self):
|
def is_sell_claim(self):
|
||||||
return self.template.name.startswith('sell_claim+')
|
return self.template.name.startswith('sell_claim+')
|
||||||
|
|||||||
@@ -1,15 +1,16 @@
|
|||||||
import time
|
import time
|
||||||
import asyncio
|
import asyncio
|
||||||
from struct import pack, unpack
|
from struct import pack, unpack
|
||||||
|
from concurrent.futures.thread import ThreadPoolExecutor
|
||||||
|
from typing import Optional
|
||||||
|
from prometheus_client import Gauge, Histogram
|
||||||
import lbry
|
import lbry
|
||||||
from lbry.schema.claim import Claim
|
|
||||||
from lbry.wallet.server.db.writer import SQLDB
|
from lbry.wallet.server.db.writer import SQLDB
|
||||||
from lbry.wallet.server.daemon import DaemonError
|
from lbry.wallet.server.daemon import DaemonError
|
||||||
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
|
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
|
||||||
from lbry.wallet.server.util import chunks, class_logger
|
from lbry.wallet.server.util import chunks, class_logger
|
||||||
from lbry.wallet.server.leveldb import FlushData
|
from lbry.wallet.server.leveldb import FlushData
|
||||||
from lbry.wallet.server.prometheus import BLOCK_COUNT, BLOCK_UPDATE_TIMES
|
from lbry.wallet.server.udp import StatusServer
|
||||||
|
|
||||||
|
|
||||||
class Prefetcher:
|
class Prefetcher:
|
||||||
@@ -128,6 +129,12 @@ class ChainError(Exception):
|
|||||||
"""Raised on error processing blocks."""
|
"""Raised on error processing blocks."""
|
||||||
|
|
||||||
|
|
||||||
|
NAMESPACE = "wallet_server"
|
||||||
|
HISTOGRAM_BUCKETS = (
|
||||||
|
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class BlockProcessor:
|
class BlockProcessor:
|
||||||
"""Process blocks and update the DB state to match.
|
"""Process blocks and update the DB state to match.
|
||||||
|
|
||||||
@@ -135,6 +142,16 @@ class BlockProcessor:
|
|||||||
Coordinate backing up in case of chain reorganisations.
|
Coordinate backing up in case of chain reorganisations.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
block_count_metric = Gauge(
|
||||||
|
"block_count", "Number of processed blocks", namespace=NAMESPACE
|
||||||
|
)
|
||||||
|
block_update_time_metric = Histogram(
|
||||||
|
"block_time", "Block update times", namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS
|
||||||
|
)
|
||||||
|
reorg_count_metric = Gauge(
|
||||||
|
"reorg_count", "Number of reorgs", namespace=NAMESPACE
|
||||||
|
)
|
||||||
|
|
||||||
def __init__(self, env, db, daemon, notifications):
|
def __init__(self, env, db, daemon, notifications):
|
||||||
self.env = env
|
self.env = env
|
||||||
self.db = db
|
self.db = db
|
||||||
@@ -145,6 +162,7 @@ class BlockProcessor:
|
|||||||
self.blocks_event = asyncio.Event()
|
self.blocks_event = asyncio.Event()
|
||||||
self.prefetcher = Prefetcher(daemon, env.coin, self.blocks_event)
|
self.prefetcher = Prefetcher(daemon, env.coin, self.blocks_event)
|
||||||
self.logger = class_logger(__name__, self.__class__.__name__)
|
self.logger = class_logger(__name__, self.__class__.__name__)
|
||||||
|
self.executor = ThreadPoolExecutor(1)
|
||||||
|
|
||||||
# Meta
|
# Meta
|
||||||
self.next_cache_check = 0
|
self.next_cache_check = 0
|
||||||
@@ -153,7 +171,8 @@ class BlockProcessor:
|
|||||||
|
|
||||||
# Caches of unflushed items.
|
# Caches of unflushed items.
|
||||||
self.headers = []
|
self.headers = []
|
||||||
self.tx_hashes = []
|
self.block_hashes = []
|
||||||
|
self.block_txs = []
|
||||||
self.undo_infos = []
|
self.undo_infos = []
|
||||||
|
|
||||||
# UTXO cache
|
# UTXO cache
|
||||||
@@ -165,6 +184,9 @@ class BlockProcessor:
|
|||||||
self.state_lock = asyncio.Lock()
|
self.state_lock = asyncio.Lock()
|
||||||
|
|
||||||
self.search_cache = {}
|
self.search_cache = {}
|
||||||
|
self.history_cache = {}
|
||||||
|
self.status_server = StatusServer()
|
||||||
|
|
||||||
async def run_in_thread_with_lock(self, func, *args):
|
async def run_in_thread_with_lock(self, func, *args):
|
||||||
# Run in a thread to prevent blocking. Shielded so that
|
# Run in a thread to prevent blocking. Shielded so that
|
||||||
# cancellations from shutdown don't lose work - when the task
|
# cancellations from shutdown don't lose work - when the task
|
||||||
@@ -173,7 +195,7 @@ class BlockProcessor:
|
|||||||
# consistent and not being updated elsewhere.
|
# consistent and not being updated elsewhere.
|
||||||
async def run_in_thread_locked():
|
async def run_in_thread_locked():
|
||||||
async with self.state_lock:
|
async with self.state_lock:
|
||||||
return await asyncio.get_event_loop().run_in_executor(None, func, *args)
|
return await asyncio.get_event_loop().run_in_executor(self.executor, func, *args)
|
||||||
return await asyncio.shield(run_in_thread_locked())
|
return await asyncio.shield(run_in_thread_locked())
|
||||||
|
|
||||||
async def check_and_advance_blocks(self, raw_blocks):
|
async def check_and_advance_blocks(self, raw_blocks):
|
||||||
@@ -192,16 +214,24 @@ class BlockProcessor:
|
|||||||
if hprevs == chain:
|
if hprevs == chain:
|
||||||
start = time.perf_counter()
|
start = time.perf_counter()
|
||||||
await self.run_in_thread_with_lock(self.advance_blocks, blocks)
|
await self.run_in_thread_with_lock(self.advance_blocks, blocks)
|
||||||
|
if self.sql:
|
||||||
|
await self.db.search_index.claim_consumer(self.sql.claim_producer())
|
||||||
for cache in self.search_cache.values():
|
for cache in self.search_cache.values():
|
||||||
cache.clear()
|
cache.clear()
|
||||||
|
self.history_cache.clear()
|
||||||
|
self.notifications.notified_mempool_txs.clear()
|
||||||
await self._maybe_flush()
|
await self._maybe_flush()
|
||||||
processed_time = time.perf_counter() - start
|
processed_time = time.perf_counter() - start
|
||||||
BLOCK_COUNT.set(self.height)
|
self.block_count_metric.set(self.height)
|
||||||
BLOCK_UPDATE_TIMES.observe(processed_time)
|
self.block_update_time_metric.observe(processed_time)
|
||||||
|
self.status_server.set_height(self.db.fs_height, self.db.db_tip)
|
||||||
if not self.db.first_sync:
|
if not self.db.first_sync:
|
||||||
s = '' if len(blocks) == 1 else 's'
|
s = '' if len(blocks) == 1 else 's'
|
||||||
self.logger.info('processed {:,d} block{} in {:.1f}s'.format(len(blocks), s, processed_time))
|
self.logger.info('processed {:,d} block{} in {:.1f}s'.format(len(blocks), s, processed_time))
|
||||||
if self._caught_up_event.is_set():
|
if self._caught_up_event.is_set():
|
||||||
|
if self.sql:
|
||||||
|
await self.db.search_index.apply_filters(self.sql.blocked_streams, self.sql.blocked_channels,
|
||||||
|
self.sql.filtered_streams, self.sql.filtered_channels)
|
||||||
await self.notifications.on_block(self.touched, self.height)
|
await self.notifications.on_block(self.touched, self.height)
|
||||||
self.touched = set()
|
self.touched = set()
|
||||||
elif hprevs[0] != chain[0]:
|
elif hprevs[0] != chain[0]:
|
||||||
@@ -216,7 +246,7 @@ class BlockProcessor:
|
|||||||
'resetting the prefetcher')
|
'resetting the prefetcher')
|
||||||
await self.prefetcher.reset_height(self.height)
|
await self.prefetcher.reset_height(self.height)
|
||||||
|
|
||||||
async def reorg_chain(self, count=None):
|
async def reorg_chain(self, count: Optional[int] = None):
|
||||||
"""Handle a chain reorganisation.
|
"""Handle a chain reorganisation.
|
||||||
|
|
||||||
Count is the number of blocks to simulate a reorg, or None for
|
Count is the number of blocks to simulate a reorg, or None for
|
||||||
@@ -225,12 +255,12 @@ class BlockProcessor:
|
|||||||
self.logger.info('chain reorg detected')
|
self.logger.info('chain reorg detected')
|
||||||
else:
|
else:
|
||||||
self.logger.info(f'faking a reorg of {count:,d} blocks')
|
self.logger.info(f'faking a reorg of {count:,d} blocks')
|
||||||
await self.flush(True)
|
|
||||||
|
|
||||||
async def get_raw_blocks(last_height, hex_hashes):
|
async def get_raw_blocks(last_height, hex_hashes):
|
||||||
heights = range(last_height, last_height - len(hex_hashes), -1)
|
heights = range(last_height, last_height - len(hex_hashes), -1)
|
||||||
try:
|
try:
|
||||||
blocks = [self.db.read_raw_block(height) for height in heights]
|
blocks = [await self.db.read_raw_block(height) for height in heights]
|
||||||
self.logger.info(f'read {len(blocks)} blocks from disk')
|
self.logger.info(f'read {len(blocks)} blocks from disk')
|
||||||
return blocks
|
return blocks
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
@@ -242,15 +272,27 @@ class BlockProcessor:
|
|||||||
self.touched.discard(None)
|
self.touched.discard(None)
|
||||||
self.db.flush_backup(self.flush_data(), self.touched)
|
self.db.flush_backup(self.flush_data(), self.touched)
|
||||||
|
|
||||||
start, last, hashes = await self.reorg_hashes(count)
|
try:
|
||||||
# Reverse and convert to hex strings.
|
await self.flush(True)
|
||||||
hashes = [hash_to_hex_str(hash) for hash in reversed(hashes)]
|
|
||||||
for hex_hashes in chunks(hashes, 50):
|
start, last, hashes = await self.reorg_hashes(count)
|
||||||
raw_blocks = await get_raw_blocks(last, hex_hashes)
|
# Reverse and convert to hex strings.
|
||||||
await self.run_in_thread_with_lock(self.backup_blocks, raw_blocks)
|
hashes = [hash_to_hex_str(hash) for hash in reversed(hashes)]
|
||||||
await self.run_in_thread_with_lock(flush_backup)
|
self.logger.info("reorg %i block hashes", len(hashes))
|
||||||
last -= len(raw_blocks)
|
for hex_hashes in chunks(hashes, 50):
|
||||||
await self.prefetcher.reset_height(self.height)
|
raw_blocks = await get_raw_blocks(last, hex_hashes)
|
||||||
|
self.logger.info("got %i raw blocks", len(raw_blocks))
|
||||||
|
await self.run_in_thread_with_lock(self.backup_blocks, raw_blocks)
|
||||||
|
await self.run_in_thread_with_lock(flush_backup)
|
||||||
|
last -= len(raw_blocks)
|
||||||
|
|
||||||
|
await self.prefetcher.reset_height(self.height)
|
||||||
|
self.reorg_count_metric.inc()
|
||||||
|
except:
|
||||||
|
self.logger.exception("boom")
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
self.logger.info("done with reorg")
|
||||||
|
|
||||||
async def reorg_hashes(self, count):
|
async def reorg_hashes(self, count):
|
||||||
"""Return a pair (start, last, hashes) of blocks to back up during a
|
"""Return a pair (start, last, hashes) of blocks to back up during a
|
||||||
@@ -267,7 +309,7 @@ class BlockProcessor:
|
|||||||
|
|
||||||
return start, last, await self.db.fs_block_hashes(start, count)
|
return start, last, await self.db.fs_block_hashes(start, count)
|
||||||
|
|
||||||
async def calc_reorg_range(self, count):
|
async def calc_reorg_range(self, count: Optional[int]):
|
||||||
"""Calculate the reorg range"""
|
"""Calculate the reorg range"""
|
||||||
|
|
||||||
def diff_pos(hashes1, hashes2):
|
def diff_pos(hashes1, hashes2):
|
||||||
@@ -313,8 +355,8 @@ class BlockProcessor:
|
|||||||
def flush_data(self):
|
def flush_data(self):
|
||||||
"""The data for a flush. The lock must be taken."""
|
"""The data for a flush. The lock must be taken."""
|
||||||
assert self.state_lock.locked()
|
assert self.state_lock.locked()
|
||||||
return FlushData(self.height, self.tx_count, self.headers,
|
return FlushData(self.height, self.tx_count, self.headers, self.block_hashes,
|
||||||
self.tx_hashes, self.undo_infos, self.utxo_cache,
|
self.block_txs, self.undo_infos, self.utxo_cache,
|
||||||
self.db_deletes, self.tip)
|
self.db_deletes, self.tip)
|
||||||
|
|
||||||
async def flush(self, flush_utxos):
|
async def flush(self, flush_utxos):
|
||||||
@@ -328,11 +370,9 @@ class BlockProcessor:
|
|||||||
# performed on the DB.
|
# performed on the DB.
|
||||||
if self._caught_up_event.is_set():
|
if self._caught_up_event.is_set():
|
||||||
await self.flush(True)
|
await self.flush(True)
|
||||||
elif time.time() > self.next_cache_check:
|
elif time.perf_counter() > self.next_cache_check:
|
||||||
flush_arg = self.check_cache_size()
|
await self.flush(True)
|
||||||
if flush_arg is not None:
|
self.next_cache_check = time.perf_counter() + 30
|
||||||
await self.flush(flush_arg)
|
|
||||||
self.next_cache_check = time.time() + 30
|
|
||||||
|
|
||||||
def check_cache_size(self):
|
def check_cache_size(self):
|
||||||
"""Flush a cache if it gets too big."""
|
"""Flush a cache if it gets too big."""
|
||||||
@@ -371,7 +411,8 @@ class BlockProcessor:
|
|||||||
for block in blocks:
|
for block in blocks:
|
||||||
height += 1
|
height += 1
|
||||||
undo_info = self.advance_txs(
|
undo_info = self.advance_txs(
|
||||||
height, block.transactions, self.coin.electrum_header(block.header, height)
|
height, block.transactions, self.coin.electrum_header(block.header, height),
|
||||||
|
self.coin.header_hash(block.header)
|
||||||
)
|
)
|
||||||
if height >= min_height:
|
if height >= min_height:
|
||||||
self.undo_infos.append((undo_info, height))
|
self.undo_infos.append((undo_info, height))
|
||||||
@@ -382,25 +423,26 @@ class BlockProcessor:
|
|||||||
self.headers.extend(headers)
|
self.headers.extend(headers)
|
||||||
self.tip = self.coin.header_hash(headers[-1])
|
self.tip = self.coin.header_hash(headers[-1])
|
||||||
|
|
||||||
def advance_txs(self, height, txs, header):
|
def advance_txs(self, height, txs, header, block_hash):
|
||||||
self.tx_hashes.append(b''.join(tx_hash for tx, tx_hash in txs))
|
self.block_hashes.append(block_hash)
|
||||||
|
self.block_txs.append((b''.join(tx_hash for tx, tx_hash in txs), [tx.raw for tx, _ in txs]))
|
||||||
|
|
||||||
# Use local vars for speed in the loops
|
|
||||||
undo_info = []
|
undo_info = []
|
||||||
tx_num = self.tx_count
|
tx_num = self.tx_count
|
||||||
script_hashX = self.coin.hashX_from_script
|
hashXs_by_tx = []
|
||||||
s_pack = pack
|
|
||||||
|
# Use local vars for speed in the loops
|
||||||
put_utxo = self.utxo_cache.__setitem__
|
put_utxo = self.utxo_cache.__setitem__
|
||||||
spend_utxo = self.spend_utxo
|
spend_utxo = self.spend_utxo
|
||||||
undo_info_append = undo_info.append
|
undo_info_append = undo_info.append
|
||||||
update_touched = self.touched.update
|
update_touched = self.touched.update
|
||||||
hashXs_by_tx = []
|
append_hashX_by_tx = hashXs_by_tx.append
|
||||||
append_hashXs = hashXs_by_tx.append
|
hashX_from_script = self.coin.hashX_from_script
|
||||||
|
|
||||||
for tx, tx_hash in txs:
|
for tx, tx_hash in txs:
|
||||||
hashXs = []
|
hashXs = []
|
||||||
append_hashX = hashXs.append
|
append_hashX = hashXs.append
|
||||||
tx_numb = s_pack('<I', tx_num)
|
tx_numb = pack('<I', tx_num)
|
||||||
|
|
||||||
# Spend the inputs
|
# Spend the inputs
|
||||||
for txin in tx.inputs:
|
for txin in tx.inputs:
|
||||||
@@ -413,18 +455,17 @@ class BlockProcessor:
|
|||||||
# Add the new UTXOs
|
# Add the new UTXOs
|
||||||
for idx, txout in enumerate(tx.outputs):
|
for idx, txout in enumerate(tx.outputs):
|
||||||
# Get the hashX. Ignore unspendable outputs
|
# Get the hashX. Ignore unspendable outputs
|
||||||
hashX = script_hashX(txout.pk_script)
|
hashX = hashX_from_script(txout.pk_script)
|
||||||
if hashX:
|
if hashX:
|
||||||
append_hashX(hashX)
|
append_hashX(hashX)
|
||||||
put_utxo(tx_hash + s_pack('<H', idx),
|
put_utxo(tx_hash + pack('<H', idx), hashX + tx_numb + pack('<Q', txout.value))
|
||||||
hashX + tx_numb + s_pack('<Q', txout.value))
|
|
||||||
|
|
||||||
append_hashXs(hashXs)
|
append_hashX_by_tx(hashXs)
|
||||||
update_touched(hashXs)
|
update_touched(hashXs)
|
||||||
|
self.db.total_transactions.append(tx_hash)
|
||||||
tx_num += 1
|
tx_num += 1
|
||||||
|
|
||||||
self.db.history.add_unflushed(hashXs_by_tx, self.tx_count)
|
self.db.history.add_unflushed(hashXs_by_tx, self.tx_count)
|
||||||
|
|
||||||
self.tx_count = tx_num
|
self.tx_count = tx_num
|
||||||
self.db.tx_counts.append(tx_num)
|
self.db.tx_counts.append(tx_num)
|
||||||
|
|
||||||
@@ -441,6 +482,7 @@ class BlockProcessor:
|
|||||||
|
|
||||||
coin = self.coin
|
coin = self.coin
|
||||||
for raw_block in raw_blocks:
|
for raw_block in raw_blocks:
|
||||||
|
self.logger.info("backup block %i", self.height)
|
||||||
# Check and update self.tip
|
# Check and update self.tip
|
||||||
block = coin.block(raw_block, self.height)
|
block = coin.block(raw_block, self.height)
|
||||||
header_hash = coin.header_hash(block.header)
|
header_hash = coin.header_hash(block.header)
|
||||||
@@ -466,20 +508,16 @@ class BlockProcessor:
|
|||||||
|
|
||||||
# Use local vars for speed in the loops
|
# Use local vars for speed in the loops
|
||||||
s_pack = pack
|
s_pack = pack
|
||||||
put_utxo = self.utxo_cache.__setitem__
|
|
||||||
spend_utxo = self.spend_utxo
|
|
||||||
script_hashX = self.coin.hashX_from_script
|
|
||||||
touched = self.touched
|
|
||||||
undo_entry_len = 12 + HASHX_LEN
|
undo_entry_len = 12 + HASHX_LEN
|
||||||
|
|
||||||
for tx, tx_hash in reversed(txs):
|
for tx, tx_hash in reversed(txs):
|
||||||
for idx, txout in enumerate(tx.outputs):
|
for idx, txout in enumerate(tx.outputs):
|
||||||
# Spend the TX outputs. Be careful with unspendable
|
# Spend the TX outputs. Be careful with unspendable
|
||||||
# outputs - we didn't save those in the first place.
|
# outputs - we didn't save those in the first place.
|
||||||
hashX = script_hashX(txout.pk_script)
|
hashX = self.coin.hashX_from_script(txout.pk_script)
|
||||||
if hashX:
|
if hashX:
|
||||||
cache_value = spend_utxo(tx_hash, idx)
|
cache_value = self.spend_utxo(tx_hash, idx)
|
||||||
touched.add(cache_value[:-12])
|
self.touched.add(cache_value[:-12])
|
||||||
|
|
||||||
# Restore the inputs
|
# Restore the inputs
|
||||||
for txin in reversed(tx.inputs):
|
for txin in reversed(tx.inputs):
|
||||||
@@ -487,9 +525,10 @@ class BlockProcessor:
|
|||||||
continue
|
continue
|
||||||
n -= undo_entry_len
|
n -= undo_entry_len
|
||||||
undo_item = undo_info[n:n + undo_entry_len]
|
undo_item = undo_info[n:n + undo_entry_len]
|
||||||
put_utxo(txin.prev_hash + s_pack('<H', txin.prev_idx),
|
self.utxo_cache[txin.prev_hash + s_pack('<H', txin.prev_idx)] = undo_item
|
||||||
undo_item)
|
self.touched.add(undo_item[:-12])
|
||||||
touched.add(undo_item[:-12])
|
|
||||||
|
self.db.total_transactions.pop()
|
||||||
|
|
||||||
assert n == 0
|
assert n == 0
|
||||||
self.tx_count -= len(txs)
|
self.tx_count -= len(txs)
|
||||||
@@ -566,15 +605,18 @@ class BlockProcessor:
|
|||||||
# Key: b'h' + compressed_tx_hash + tx_idx + tx_num
|
# Key: b'h' + compressed_tx_hash + tx_idx + tx_num
|
||||||
# Value: hashX
|
# Value: hashX
|
||||||
prefix = b'h' + tx_hash[:4] + idx_packed
|
prefix = b'h' + tx_hash[:4] + idx_packed
|
||||||
candidates = {db_key: hashX for db_key, hashX
|
candidates = dict(self.db.utxo_db.iterator(prefix=prefix))
|
||||||
in self.db.utxo_db.iterator(prefix=prefix)}
|
|
||||||
|
|
||||||
for hdb_key, hashX in candidates.items():
|
for hdb_key, hashX in candidates.items():
|
||||||
tx_num_packed = hdb_key[-4:]
|
tx_num_packed = hdb_key[-4:]
|
||||||
|
|
||||||
if len(candidates) > 1:
|
if len(candidates) > 1:
|
||||||
|
|
||||||
tx_num, = unpack('<I', tx_num_packed)
|
tx_num, = unpack('<I', tx_num_packed)
|
||||||
hash, height = self.db.fs_tx_hash(tx_num)
|
try:
|
||||||
|
hash, height = self.db.fs_tx_hash(tx_num)
|
||||||
|
except IndexError:
|
||||||
|
self.logger.error("data integrity error for hashx history: %s missing tx #%s (%s:%s)",
|
||||||
|
hashX.hex(), tx_num, hash_to_hex_str(tx_hash), tx_idx)
|
||||||
|
continue
|
||||||
if hash != tx_hash:
|
if hash != tx_hash:
|
||||||
assert hash is not None # Should always be found
|
assert hash is not None # Should always be found
|
||||||
continue
|
continue
|
||||||
@@ -583,12 +625,17 @@ class BlockProcessor:
|
|||||||
# Value: the UTXO value as a 64-bit unsigned integer
|
# Value: the UTXO value as a 64-bit unsigned integer
|
||||||
udb_key = b'u' + hashX + hdb_key[-6:]
|
udb_key = b'u' + hashX + hdb_key[-6:]
|
||||||
utxo_value_packed = self.db.utxo_db.get(udb_key)
|
utxo_value_packed = self.db.utxo_db.get(udb_key)
|
||||||
if utxo_value_packed:
|
if utxo_value_packed is None:
|
||||||
# Remove both entries for this UTXO
|
self.logger.warning(
|
||||||
self.db_deletes.append(hdb_key)
|
"%s:%s is not found in UTXO db for %s", hash_to_hex_str(tx_hash), tx_idx, hash_to_hex_str(hashX)
|
||||||
self.db_deletes.append(udb_key)
|
)
|
||||||
return hashX + tx_num_packed + utxo_value_packed
|
raise ChainError(f"{hash_to_hex_str(tx_hash)}:{tx_idx} is not found in UTXO db for {hash_to_hex_str(hashX)}")
|
||||||
|
# Remove both entries for this UTXO
|
||||||
|
self.db_deletes.append(hdb_key)
|
||||||
|
self.db_deletes.append(udb_key)
|
||||||
|
return hashX + tx_num_packed + utxo_value_packed
|
||||||
|
|
||||||
|
self.logger.error('UTXO {hash_to_hex_str(tx_hash)} / {tx_idx} not found in "h" table')
|
||||||
raise ChainError('UTXO {} / {:,d} not found in "h" table'
|
raise ChainError('UTXO {} / {:,d} not found in "h" table'
|
||||||
.format(hash_to_hex_str(tx_hash), tx_idx))
|
.format(hash_to_hex_str(tx_hash), tx_idx))
|
||||||
|
|
||||||
@@ -601,12 +648,16 @@ class BlockProcessor:
|
|||||||
self._caught_up_event.set()
|
self._caught_up_event.set()
|
||||||
await self.blocks_event.wait()
|
await self.blocks_event.wait()
|
||||||
self.blocks_event.clear()
|
self.blocks_event.clear()
|
||||||
if self.reorg_count:
|
if self.reorg_count: # this could only happen by calling the reorg rpc
|
||||||
await self.reorg_chain(self.reorg_count)
|
await self.reorg_chain(self.reorg_count)
|
||||||
self.reorg_count = 0
|
self.reorg_count = 0
|
||||||
else:
|
else:
|
||||||
blocks = self.prefetcher.get_prefetched_blocks()
|
blocks = self.prefetcher.get_prefetched_blocks()
|
||||||
await self.check_and_advance_blocks(blocks)
|
try:
|
||||||
|
await self.check_and_advance_blocks(blocks)
|
||||||
|
except Exception:
|
||||||
|
self.logger.exception("error while processing txs")
|
||||||
|
raise
|
||||||
|
|
||||||
async def _first_caught_up(self):
|
async def _first_caught_up(self):
|
||||||
self.logger.info(f'caught up to height {self.height}')
|
self.logger.info(f'caught up to height {self.height}')
|
||||||
@@ -640,9 +691,11 @@ class BlockProcessor:
|
|||||||
disk before exiting, as otherwise a significant amount of work
|
disk before exiting, as otherwise a significant amount of work
|
||||||
could be lost.
|
could be lost.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
self._caught_up_event = caught_up_event
|
self._caught_up_event = caught_up_event
|
||||||
try:
|
try:
|
||||||
await self._first_open_dbs()
|
await self._first_open_dbs()
|
||||||
|
self.status_server.set_height(self.db.fs_height, self.db.db_tip)
|
||||||
await asyncio.wait([
|
await asyncio.wait([
|
||||||
self.prefetcher.main_loop(self.height),
|
self.prefetcher.main_loop(self.height),
|
||||||
self._process_prefetched_blocks()
|
self._process_prefetched_blocks()
|
||||||
@@ -653,10 +706,12 @@ class BlockProcessor:
|
|||||||
self.logger.exception("Block processing failed!")
|
self.logger.exception("Block processing failed!")
|
||||||
raise
|
raise
|
||||||
finally:
|
finally:
|
||||||
|
self.status_server.stop()
|
||||||
# Shut down block processing
|
# Shut down block processing
|
||||||
self.logger.info('flushing to DB for a clean shutdown...')
|
self.logger.info('flushing to DB for a clean shutdown...')
|
||||||
await self.flush(True)
|
await self.flush(True)
|
||||||
self.db.close()
|
self.db.close()
|
||||||
|
self.executor.shutdown(wait=True)
|
||||||
|
|
||||||
def force_chain_reorg(self, count):
|
def force_chain_reorg(self, count):
|
||||||
"""Force a reorg of the given number of blocks.
|
"""Force a reorg of the given number of blocks.
|
||||||
@@ -671,7 +726,6 @@ class BlockProcessor:
|
|||||||
|
|
||||||
|
|
||||||
class Timer:
|
class Timer:
|
||||||
|
|
||||||
def __init__(self, name):
|
def __init__(self, name):
|
||||||
self.name = name
|
self.name = name
|
||||||
self.total = 0
|
self.total = 0
|
||||||
@@ -734,38 +788,22 @@ class LBRYBlockProcessor(BlockProcessor):
|
|||||||
self.timer = Timer('BlockProcessor')
|
self.timer = Timer('BlockProcessor')
|
||||||
|
|
||||||
def advance_blocks(self, blocks):
|
def advance_blocks(self, blocks):
|
||||||
self.sql.begin()
|
if self.sql:
|
||||||
|
self.sql.begin()
|
||||||
try:
|
try:
|
||||||
self.timer.run(super().advance_blocks, blocks)
|
self.timer.run(super().advance_blocks, blocks)
|
||||||
except:
|
except:
|
||||||
self.logger.exception(f'Error while advancing transaction in new block.')
|
self.logger.exception(f'Error while advancing transaction in new block.')
|
||||||
raise
|
raise
|
||||||
finally:
|
finally:
|
||||||
self.sql.commit()
|
if self.sql:
|
||||||
if self.db.first_sync and self.height == self.daemon.cached_height():
|
self.sql.commit()
|
||||||
self.timer.run(self.sql.execute, self.sql.SEARCH_INDEXES, timer_name='executing SEARCH_INDEXES')
|
|
||||||
if self.env.individual_tag_indexes:
|
|
||||||
self.timer.run(self.sql.execute, self.sql.TAG_INDEXES, timer_name='executing TAG_INDEXES')
|
|
||||||
|
|
||||||
def advance_txs(self, height, txs, header):
|
def advance_txs(self, height, txs, header, block_hash):
|
||||||
timer = self.timer.sub_timers['advance_blocks']
|
timer = self.timer.sub_timers['advance_blocks']
|
||||||
undo = timer.run(super().advance_txs, height, txs, header, timer_name='super().advance_txs')
|
undo = timer.run(super().advance_txs, height, txs, header, block_hash, timer_name='super().advance_txs')
|
||||||
timer.run(self.sql.advance_txs, height, txs, header, self.daemon.cached_height(), forward_timer=True)
|
if self.sql:
|
||||||
|
timer.run(self.sql.advance_txs, height, txs, header, self.daemon.cached_height(), forward_timer=True)
|
||||||
if (height % 10000 == 0 or not self.db.first_sync) and self.logger.isEnabledFor(10):
|
if (height % 10000 == 0 or not self.db.first_sync) and self.logger.isEnabledFor(10):
|
||||||
self.timer.show(height=height)
|
self.timer.show(height=height)
|
||||||
return undo
|
return undo
|
||||||
|
|
||||||
def _checksig(self, value, address):
|
|
||||||
try:
|
|
||||||
claim_dict = Claim.from_bytes(value)
|
|
||||||
cert_id = claim_dict.signing_channel_hash
|
|
||||||
if not self.should_validate_signatures:
|
|
||||||
return cert_id
|
|
||||||
if cert_id:
|
|
||||||
cert_claim = self.db.get_claim_info(cert_id)
|
|
||||||
if cert_claim:
|
|
||||||
certificate = Claim.from_bytes(cert_claim.value)
|
|
||||||
claim_dict.validate_signature(address, certificate)
|
|
||||||
return cert_id
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
@@ -8,7 +8,7 @@ from lbry.wallet.server.server import Server
|
|||||||
|
|
||||||
def get_argument_parser():
|
def get_argument_parser():
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
prog="torba-server"
|
prog="lbry-hub"
|
||||||
)
|
)
|
||||||
parser.add_argument("spvserver", type=str, help="Python class path to SPV server implementation.",
|
parser.add_argument("spvserver", type=str, help="Python class path to SPV server implementation.",
|
||||||
nargs="?", default="lbry.wallet.server.coin.LBC")
|
nargs="?", default="lbry.wallet.server.coin.LBC")
|
||||||
@@ -25,8 +25,10 @@ def main():
|
|||||||
parser = get_argument_parser()
|
parser = get_argument_parser()
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
coin_class = get_coin_class(args.spvserver)
|
coin_class = get_coin_class(args.spvserver)
|
||||||
logging.basicConfig(level=logging.INFO)
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-4s %(name)s:%(lineno)d: %(message)s")
|
||||||
logging.info('lbry.server starting')
|
logging.info('lbry.server starting')
|
||||||
|
logging.getLogger('aiohttp').setLevel(logging.WARNING)
|
||||||
|
logging.getLogger('elasticsearch').setLevel(logging.WARNING)
|
||||||
try:
|
try:
|
||||||
server = Server(Env(coin_class))
|
server = Server(Env(coin_class))
|
||||||
server.run()
|
server.run()
|
||||||
|
|||||||
@@ -5,11 +5,12 @@ import time
|
|||||||
from functools import wraps
|
from functools import wraps
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
|
from prometheus_client import Gauge, Histogram
|
||||||
|
from lbry.utils import LRUCacheWithMetrics
|
||||||
from lbry.wallet.rpc.jsonrpc import RPCError
|
from lbry.wallet.rpc.jsonrpc import RPCError
|
||||||
from lbry.wallet.server.util import hex_to_bytes, class_logger
|
from lbry.wallet.server.util import hex_to_bytes, class_logger
|
||||||
from lbry.wallet.rpc import JSONRPC
|
from lbry.wallet.rpc import JSONRPC
|
||||||
from lbry.wallet.server.prometheus import LBRYCRD_REQUEST_TIMES, LBRYCRD_PENDING_COUNT
|
|
||||||
|
|
||||||
class DaemonError(Exception):
|
class DaemonError(Exception):
|
||||||
"""Raised when the daemon returns an error in its results."""
|
"""Raised when the daemon returns an error in its results."""
|
||||||
@@ -23,12 +24,23 @@ class WorkQueueFullError(Exception):
|
|||||||
"""Internal - when the daemon's work queue is full."""
|
"""Internal - when the daemon's work queue is full."""
|
||||||
|
|
||||||
|
|
||||||
|
NAMESPACE = "wallet_server"
|
||||||
|
|
||||||
|
|
||||||
class Daemon:
|
class Daemon:
|
||||||
"""Handles connections to a daemon at the given URL."""
|
"""Handles connections to a daemon at the given URL."""
|
||||||
|
|
||||||
WARMING_UP = -28
|
WARMING_UP = -28
|
||||||
id_counter = itertools.count()
|
id_counter = itertools.count()
|
||||||
|
|
||||||
|
lbrycrd_request_time_metric = Histogram(
|
||||||
|
"lbrycrd_request", "lbrycrd requests count", namespace=NAMESPACE, labelnames=("method",)
|
||||||
|
)
|
||||||
|
lbrycrd_pending_count_metric = Gauge(
|
||||||
|
"lbrycrd_pending_count", "Number of lbrycrd rpcs that are in flight", namespace=NAMESPACE,
|
||||||
|
labelnames=("method",)
|
||||||
|
)
|
||||||
|
|
||||||
def __init__(self, coin, url, max_workqueue=10, init_retry=0.25,
|
def __init__(self, coin, url, max_workqueue=10, init_retry=0.25,
|
||||||
max_retry=4.0):
|
max_retry=4.0):
|
||||||
self.coin = coin
|
self.coin = coin
|
||||||
@@ -42,6 +54,8 @@ class Daemon:
|
|||||||
self._height = None
|
self._height = None
|
||||||
self.available_rpcs = {}
|
self.available_rpcs = {}
|
||||||
self.connector = aiohttp.TCPConnector()
|
self.connector = aiohttp.TCPConnector()
|
||||||
|
self._block_hash_cache = LRUCacheWithMetrics(100000)
|
||||||
|
self._block_cache = LRUCacheWithMetrics(2 ** 16, metric_name='block', namespace=NAMESPACE)
|
||||||
|
|
||||||
async def close(self):
|
async def close(self):
|
||||||
if self.connector:
|
if self.connector:
|
||||||
@@ -126,7 +140,7 @@ class Daemon:
|
|||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
for method in methods:
|
for method in methods:
|
||||||
LBRYCRD_PENDING_COUNT.labels(method=method).inc()
|
self.lbrycrd_pending_count_metric.labels(method=method).inc()
|
||||||
result = await self._send_data(data)
|
result = await self._send_data(data)
|
||||||
result = processor(result)
|
result = processor(result)
|
||||||
if on_good_message:
|
if on_good_message:
|
||||||
@@ -151,7 +165,7 @@ class Daemon:
|
|||||||
on_good_message = 'running normally'
|
on_good_message = 'running normally'
|
||||||
finally:
|
finally:
|
||||||
for method in methods:
|
for method in methods:
|
||||||
LBRYCRD_PENDING_COUNT.labels(method=method).dec()
|
self.lbrycrd_pending_count_metric.labels(method=method).dec()
|
||||||
await asyncio.sleep(retry)
|
await asyncio.sleep(retry)
|
||||||
retry = max(min(self.max_retry, retry * 2), self.init_retry)
|
retry = max(min(self.max_retry, retry * 2), self.init_retry)
|
||||||
|
|
||||||
@@ -172,7 +186,7 @@ class Daemon:
|
|||||||
if params:
|
if params:
|
||||||
payload['params'] = params
|
payload['params'] = params
|
||||||
result = await self._send(payload, processor)
|
result = await self._send(payload, processor)
|
||||||
LBRYCRD_REQUEST_TIMES.labels(method=method).observe(time.perf_counter() - start)
|
self.lbrycrd_request_time_metric.labels(method=method).observe(time.perf_counter() - start)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
async def _send_vector(self, method, params_iterable, replace_errs=False):
|
async def _send_vector(self, method, params_iterable, replace_errs=False):
|
||||||
@@ -197,7 +211,7 @@ class Daemon:
|
|||||||
result = []
|
result = []
|
||||||
if payload:
|
if payload:
|
||||||
result = await self._send(payload, processor)
|
result = await self._send(payload, processor)
|
||||||
LBRYCRD_REQUEST_TIMES.labels(method=method).observe(time.perf_counter()-start)
|
self.lbrycrd_request_time_metric.labels(method=method).observe(time.perf_counter() - start)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
async def _is_rpc_available(self, method):
|
async def _is_rpc_available(self, method):
|
||||||
@@ -219,12 +233,27 @@ class Daemon:
|
|||||||
|
|
||||||
async def block_hex_hashes(self, first, count):
|
async def block_hex_hashes(self, first, count):
|
||||||
"""Return the hex hashes of count block starting at height first."""
|
"""Return the hex hashes of count block starting at height first."""
|
||||||
|
if first + count < (self.cached_height() or 0) - 200:
|
||||||
|
return await self._cached_block_hex_hashes(first, count)
|
||||||
params_iterable = ((h, ) for h in range(first, first + count))
|
params_iterable = ((h, ) for h in range(first, first + count))
|
||||||
return await self._send_vector('getblockhash', params_iterable)
|
return await self._send_vector('getblockhash', params_iterable)
|
||||||
|
|
||||||
|
async def _cached_block_hex_hashes(self, first, count):
|
||||||
|
"""Return the hex hashes of count block starting at height first."""
|
||||||
|
cached = self._block_hash_cache.get((first, count))
|
||||||
|
if cached:
|
||||||
|
return cached
|
||||||
|
params_iterable = ((h, ) for h in range(first, first + count))
|
||||||
|
self._block_hash_cache[(first, count)] = await self._send_vector('getblockhash', params_iterable)
|
||||||
|
return self._block_hash_cache[(first, count)]
|
||||||
|
|
||||||
async def deserialised_block(self, hex_hash):
|
async def deserialised_block(self, hex_hash):
|
||||||
"""Return the deserialised block with the given hex hash."""
|
"""Return the deserialised block with the given hex hash."""
|
||||||
return await self._send_single('getblock', (hex_hash, True))
|
if hex_hash not in self._block_cache:
|
||||||
|
block = await self._send_single('getblock', (hex_hash, True))
|
||||||
|
self._block_cache[hex_hash] = block
|
||||||
|
return block
|
||||||
|
return self._block_cache[hex_hash]
|
||||||
|
|
||||||
async def raw_blocks(self, hex_hashes):
|
async def raw_blocks(self, hex_hashes):
|
||||||
"""Return the raw binary blocks with the given hex hashes."""
|
"""Return the raw binary blocks with the given hex hashes."""
|
||||||
|
|||||||
@@ -17,10 +17,6 @@ class FindShortestID:
|
|||||||
if self.short_id:
|
if self.short_id:
|
||||||
return '#'+self.short_id
|
return '#'+self.short_id
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def factory(cls):
|
|
||||||
return cls(), cls.step, cls.finalize
|
|
||||||
|
|
||||||
|
|
||||||
def register_canonical_functions(connection):
|
def register_canonical_functions(connection):
|
||||||
connection.createaggregatefunction("shortest_id", FindShortestID.factory, 2)
|
connection.create_aggregate("shortest_id", 2, FindShortestID)
|
||||||
+401
-202
@@ -1,7 +1,8 @@
|
|||||||
CLAIM_TYPES = {
|
CLAIM_TYPES = {
|
||||||
'stream': 1,
|
'stream': 1,
|
||||||
'channel': 2,
|
'channel': 2,
|
||||||
'repost': 3
|
'repost': 3,
|
||||||
|
'collection': 4,
|
||||||
}
|
}
|
||||||
|
|
||||||
STREAM_TYPES = {
|
STREAM_TYPES = {
|
||||||
@@ -10,212 +11,410 @@ STREAM_TYPES = {
|
|||||||
'image': 3,
|
'image': 3,
|
||||||
'document': 4,
|
'document': 4,
|
||||||
'binary': 5,
|
'binary': 5,
|
||||||
'model': 6
|
'model': 6,
|
||||||
|
}
|
||||||
|
|
||||||
|
# 9/21/2020
|
||||||
|
MOST_USED_TAGS = {
|
||||||
|
"gaming",
|
||||||
|
"people & blogs",
|
||||||
|
"entertainment",
|
||||||
|
"music",
|
||||||
|
"pop culture",
|
||||||
|
"education",
|
||||||
|
"technology",
|
||||||
|
"blockchain",
|
||||||
|
"news",
|
||||||
|
"funny",
|
||||||
|
"science & technology",
|
||||||
|
"learning",
|
||||||
|
"gameplay",
|
||||||
|
"news & politics",
|
||||||
|
"comedy",
|
||||||
|
"bitcoin",
|
||||||
|
"beliefs",
|
||||||
|
"nature",
|
||||||
|
"art",
|
||||||
|
"economics",
|
||||||
|
"film & animation",
|
||||||
|
"lets play",
|
||||||
|
"games",
|
||||||
|
"sports",
|
||||||
|
"howto & style",
|
||||||
|
"game",
|
||||||
|
"cryptocurrency",
|
||||||
|
"playstation 4",
|
||||||
|
"automotive",
|
||||||
|
"crypto",
|
||||||
|
"mature",
|
||||||
|
"sony interactive entertainment",
|
||||||
|
"walkthrough",
|
||||||
|
"tutorial",
|
||||||
|
"video game",
|
||||||
|
"weapons",
|
||||||
|
"playthrough",
|
||||||
|
"pc",
|
||||||
|
"anime",
|
||||||
|
"how to",
|
||||||
|
"btc",
|
||||||
|
"fun",
|
||||||
|
"ethereum",
|
||||||
|
"food",
|
||||||
|
"travel & events",
|
||||||
|
"minecraft",
|
||||||
|
"science",
|
||||||
|
"autos & vehicles",
|
||||||
|
"play",
|
||||||
|
"politics",
|
||||||
|
"commentary",
|
||||||
|
"twitch",
|
||||||
|
"ps4live",
|
||||||
|
"love",
|
||||||
|
"ps4",
|
||||||
|
"nonprofits & activism",
|
||||||
|
"ps4share",
|
||||||
|
"fortnite",
|
||||||
|
"xbox",
|
||||||
|
"porn",
|
||||||
|
"video games",
|
||||||
|
"trump",
|
||||||
|
"español",
|
||||||
|
"money",
|
||||||
|
"music video",
|
||||||
|
"nintendo",
|
||||||
|
"movie",
|
||||||
|
"coronavirus",
|
||||||
|
"donald trump",
|
||||||
|
"steam",
|
||||||
|
"trailer",
|
||||||
|
"android",
|
||||||
|
"podcast",
|
||||||
|
"xbox one",
|
||||||
|
"survival",
|
||||||
|
"audio",
|
||||||
|
"linux",
|
||||||
|
"travel",
|
||||||
|
"funny moments",
|
||||||
|
"litecoin",
|
||||||
|
"animation",
|
||||||
|
"gamer",
|
||||||
|
"lets",
|
||||||
|
"playstation",
|
||||||
|
"bitcoin news",
|
||||||
|
"history",
|
||||||
|
"xxx",
|
||||||
|
"fox news",
|
||||||
|
"dance",
|
||||||
|
"god",
|
||||||
|
"adventure",
|
||||||
|
"liberal",
|
||||||
|
"2020",
|
||||||
|
"horror",
|
||||||
|
"government",
|
||||||
|
"freedom",
|
||||||
|
"reaction",
|
||||||
|
"meme",
|
||||||
|
"photography",
|
||||||
|
"truth",
|
||||||
|
"health",
|
||||||
|
"lbry",
|
||||||
|
"family",
|
||||||
|
"online",
|
||||||
|
"eth",
|
||||||
|
"crypto news",
|
||||||
|
"diy",
|
||||||
|
"trading",
|
||||||
|
"gold",
|
||||||
|
"memes",
|
||||||
|
"world",
|
||||||
|
"space",
|
||||||
|
"lol",
|
||||||
|
"covid-19",
|
||||||
|
"rpg",
|
||||||
|
"humor",
|
||||||
|
"democrat",
|
||||||
|
"film",
|
||||||
|
"call of duty",
|
||||||
|
"tech",
|
||||||
|
"religion",
|
||||||
|
"conspiracy",
|
||||||
|
"rap",
|
||||||
|
"cnn",
|
||||||
|
"hangoutsonair",
|
||||||
|
"unboxing",
|
||||||
|
"fiction",
|
||||||
|
"conservative",
|
||||||
|
"cars",
|
||||||
|
"hoa",
|
||||||
|
"epic",
|
||||||
|
"programming",
|
||||||
|
"progressive",
|
||||||
|
"cryptocurrency news",
|
||||||
|
"classical",
|
||||||
|
"jesus",
|
||||||
|
"movies",
|
||||||
|
"book",
|
||||||
|
"ps3",
|
||||||
|
"republican",
|
||||||
|
"fitness",
|
||||||
|
"books",
|
||||||
|
"multiplayer",
|
||||||
|
"animals",
|
||||||
|
"pokemon",
|
||||||
|
"bitcoin price",
|
||||||
|
"facebook",
|
||||||
|
"sharefactory",
|
||||||
|
"criptomonedas",
|
||||||
|
"cod",
|
||||||
|
"bible",
|
||||||
|
"business",
|
||||||
|
"stream",
|
||||||
|
"comics",
|
||||||
|
"how",
|
||||||
|
"fail",
|
||||||
|
"nsfw",
|
||||||
|
"new music",
|
||||||
|
"satire",
|
||||||
|
"pets & animals",
|
||||||
|
"computer",
|
||||||
|
"classical music",
|
||||||
|
"indie",
|
||||||
|
"musica",
|
||||||
|
"msnbc",
|
||||||
|
"fps",
|
||||||
|
"mod",
|
||||||
|
"sport",
|
||||||
|
"sony",
|
||||||
|
"ripple",
|
||||||
|
"auto",
|
||||||
|
"rock",
|
||||||
|
"marvel",
|
||||||
|
"complete",
|
||||||
|
"mining",
|
||||||
|
"political",
|
||||||
|
"mobile",
|
||||||
|
"pubg",
|
||||||
|
"hip hop",
|
||||||
|
"flat earth",
|
||||||
|
"xbox 360",
|
||||||
|
"reviews",
|
||||||
|
"vlogging",
|
||||||
|
"latest news",
|
||||||
|
"hack",
|
||||||
|
"tarot",
|
||||||
|
"iphone",
|
||||||
|
"media",
|
||||||
|
"cute",
|
||||||
|
"christian",
|
||||||
|
"free speech",
|
||||||
|
"trap",
|
||||||
|
"war",
|
||||||
|
"remix",
|
||||||
|
"ios",
|
||||||
|
"xrp",
|
||||||
|
"spirituality",
|
||||||
|
"song",
|
||||||
|
"league of legends",
|
||||||
|
"cat"
|
||||||
}
|
}
|
||||||
|
|
||||||
MATURE_TAGS = [
|
MATURE_TAGS = [
|
||||||
'nsfw', 'porn', 'xxx', 'mature', 'adult', 'sex'
|
'nsfw', 'porn', 'xxx', 'mature', 'adult', 'sex'
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_tag(tag):
|
||||||
|
return tag.replace(" ", "_").replace("&", "and").replace("-", "_")
|
||||||
|
|
||||||
|
|
||||||
COMMON_TAGS = {
|
COMMON_TAGS = {
|
||||||
"gaming": "gaming",
|
tag: normalize_tag(tag) for tag in list(MOST_USED_TAGS)
|
||||||
"people & blogs": "people_and_blogs",
|
|
||||||
"pop culture": "pop_culture",
|
|
||||||
"entertainment": "entertainment",
|
|
||||||
"technology": "technology",
|
|
||||||
"music": "music",
|
|
||||||
"funny": "funny",
|
|
||||||
"education": "education",
|
|
||||||
"learning": "learning",
|
|
||||||
"news": "news",
|
|
||||||
"gameplay": "gameplay",
|
|
||||||
"science & technology": "science_and_technology",
|
|
||||||
"playstation 4": "playstation_4",
|
|
||||||
"beliefs": "beliefs",
|
|
||||||
"nature": "nature",
|
|
||||||
"news & politics": "news_and_politics",
|
|
||||||
"comedy": "comedy",
|
|
||||||
"games": "games",
|
|
||||||
"sony interactive entertainment": "sony_interactive_entertainment",
|
|
||||||
"film & animation": "film_and_animation",
|
|
||||||
"game": "game",
|
|
||||||
"howto & style": "howto_and_style",
|
|
||||||
"weapons": "weapons",
|
|
||||||
"blockchain": "blockchain",
|
|
||||||
"video game": "video_game",
|
|
||||||
"sports": "sports",
|
|
||||||
"walkthrough": "walkthrough",
|
|
||||||
"ps4live": "ps4live",
|
|
||||||
"art": "art",
|
|
||||||
"pc": "pc",
|
|
||||||
"economics": "economics",
|
|
||||||
"automotive": "automotive",
|
|
||||||
"minecraft": "minecraft",
|
|
||||||
"playthrough": "playthrough",
|
|
||||||
"ps4share": "ps4share",
|
|
||||||
"tutorial": "tutorial",
|
|
||||||
"play": "play",
|
|
||||||
"twitch": "twitch",
|
|
||||||
"how to": "how_to",
|
|
||||||
"ps4": "ps4",
|
|
||||||
"bitcoin": "bitcoin",
|
|
||||||
"fortnite": "fortnite",
|
|
||||||
"commentary": "commentary",
|
|
||||||
"lets play": "lets_play",
|
|
||||||
"fun": "fun",
|
|
||||||
"politics": "politics",
|
|
||||||
"xbox": "xbox",
|
|
||||||
"autos & vehicles": "autos_and_vehicles",
|
|
||||||
"travel & events": "travel_and_events",
|
|
||||||
"food": "food",
|
|
||||||
"science": "science",
|
|
||||||
"mature": "mature",
|
|
||||||
"xbox one": "xbox_one",
|
|
||||||
"liberal": "liberal",
|
|
||||||
"democrat": "democrat",
|
|
||||||
"progressive": "progressive",
|
|
||||||
"survival": "survival",
|
|
||||||
"nonprofits & activism": "nonprofits_and_activism",
|
|
||||||
"cryptocurrency": "cryptocurrency",
|
|
||||||
"playstation": "playstation",
|
|
||||||
"nintendo": "nintendo",
|
|
||||||
"government": "government",
|
|
||||||
"steam": "steam",
|
|
||||||
"podcast": "podcast",
|
|
||||||
"horror": "horror",
|
|
||||||
"conservative": "conservative",
|
|
||||||
"reaction": "reaction",
|
|
||||||
"trailer": "trailer",
|
|
||||||
"love": "love",
|
|
||||||
"cnn": "cnn",
|
|
||||||
"republican": "republican",
|
|
||||||
"gamer": "gamer",
|
|
||||||
"political": "political",
|
|
||||||
"hangoutsonair": "hangoutsonair",
|
|
||||||
"hoa": "hoa",
|
|
||||||
"msnbc": "msnbc",
|
|
||||||
"cbs": "cbs",
|
|
||||||
"donald trump": "donald_trump",
|
|
||||||
"fiction": "fiction",
|
|
||||||
"fox news": "fox_news",
|
|
||||||
"anime": "anime",
|
|
||||||
"crypto": "crypto",
|
|
||||||
"ethereum": "ethereum",
|
|
||||||
"call of duty": "call_of_duty",
|
|
||||||
"multiplayer": "multiplayer",
|
|
||||||
"android": "android",
|
|
||||||
"epic": "epic",
|
|
||||||
"rpg": "rpg",
|
|
||||||
"adventure": "adventure",
|
|
||||||
"secular talk": "secular_talk",
|
|
||||||
"btc": "btc",
|
|
||||||
"atheist": "atheist",
|
|
||||||
"atheism": "atheism",
|
|
||||||
"ps3": "ps3",
|
|
||||||
"video games": "video_games",
|
|
||||||
"cod": "cod",
|
|
||||||
"agnostic": "agnostic",
|
|
||||||
"movie": "movie",
|
|
||||||
"online": "online",
|
|
||||||
"fps": "fps",
|
|
||||||
"mod": "mod",
|
|
||||||
"reviews": "reviews",
|
|
||||||
"sharefactory": "sharefactory",
|
|
||||||
"world": "world",
|
|
||||||
"space": "space",
|
|
||||||
"hilarious": "hilarious",
|
|
||||||
"stream": "stream",
|
|
||||||
"lol": "lol",
|
|
||||||
"sony": "sony",
|
|
||||||
"god": "god",
|
|
||||||
"lets": "lets",
|
|
||||||
"dance": "dance",
|
|
||||||
"pvp": "pvp",
|
|
||||||
"tech": "tech",
|
|
||||||
"zombies": "zombies",
|
|
||||||
"pokemon": "pokemon",
|
|
||||||
"fail": "fail",
|
|
||||||
"xbox 360": "xbox_360",
|
|
||||||
"film": "film",
|
|
||||||
"unboxing": "unboxing",
|
|
||||||
"animation": "animation",
|
|
||||||
"travel": "travel",
|
|
||||||
"money": "money",
|
|
||||||
"wwe": "wwe",
|
|
||||||
"how": "how",
|
|
||||||
"mods": "mods",
|
|
||||||
"pubg": "pubg",
|
|
||||||
"indie": "indie",
|
|
||||||
"strategy": "strategy",
|
|
||||||
"history": "history",
|
|
||||||
"rap": "rap",
|
|
||||||
"ios": "ios",
|
|
||||||
"sony computer entertainment": "sony_computer_entertainment",
|
|
||||||
"mobile": "mobile",
|
|
||||||
"trump": "trump",
|
|
||||||
"flat earth": "flat_earth",
|
|
||||||
"hack": "hack",
|
|
||||||
"trap": "trap",
|
|
||||||
"fox": "fox",
|
|
||||||
"vlogging": "vlogging",
|
|
||||||
"news radio": "news_radio",
|
|
||||||
"humor": "humor",
|
|
||||||
"facebook": "facebook",
|
|
||||||
"edm": "edm",
|
|
||||||
"fitness": "fitness",
|
|
||||||
"vaping": "vaping",
|
|
||||||
"hip hop": "hip_hop",
|
|
||||||
"secular": "secular",
|
|
||||||
"jesus": "jesus",
|
|
||||||
"vape": "vape",
|
|
||||||
"song": "song",
|
|
||||||
"remix": "remix",
|
|
||||||
"guitar": "guitar",
|
|
||||||
"daily": "daily",
|
|
||||||
"mining": "mining",
|
|
||||||
"diy": "diy",
|
|
||||||
"videogame": "videogame",
|
|
||||||
"pets & animals": "pets_and_animals",
|
|
||||||
"funny moments": "funny_moments",
|
|
||||||
"religion": "religion",
|
|
||||||
"death": "death",
|
|
||||||
"media": "media",
|
|
||||||
"nbc": "nbc",
|
|
||||||
"war": "war",
|
|
||||||
"freedom": "freedom",
|
|
||||||
"viral": "viral",
|
|
||||||
"meme": "meme",
|
|
||||||
"family": "family",
|
|
||||||
"gold": "gold",
|
|
||||||
"photography": "photography",
|
|
||||||
"chill": "chill",
|
|
||||||
"zombie": "zombie",
|
|
||||||
"computer": "computer",
|
|
||||||
"sniper": "sniper",
|
|
||||||
"bible": "bible",
|
|
||||||
"linux": "linux",
|
|
||||||
"overwatch": "overwatch",
|
|
||||||
"pro": "pro",
|
|
||||||
"dragon": "dragon",
|
|
||||||
"litecoin": "litecoin",
|
|
||||||
"gta": "gta",
|
|
||||||
"iphone": "iphone",
|
|
||||||
"house": "house",
|
|
||||||
"bass": "bass",
|
|
||||||
"bitcoin news": "bitcoin_news",
|
|
||||||
"wii": "wii",
|
|
||||||
"crash": "crash",
|
|
||||||
"league of legends": "league_of_legends",
|
|
||||||
"grand theft auto v": "grand_theft_auto_v",
|
|
||||||
"mario": "mario",
|
|
||||||
"mmorpg": "mmorpg",
|
|
||||||
"satire": "satire",
|
|
||||||
"fire": "fire",
|
|
||||||
"racing": "racing",
|
|
||||||
"apple": "apple",
|
|
||||||
"health": "health",
|
|
||||||
"instrumental": "instrumental",
|
|
||||||
"destiny": "destiny",
|
|
||||||
"truth": "truth",
|
|
||||||
"race": "race"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
INDEXED_LANGUAGES = [
|
||||||
|
'none',
|
||||||
|
'en',
|
||||||
|
'aa',
|
||||||
|
'ab',
|
||||||
|
'ae',
|
||||||
|
'af',
|
||||||
|
'ak',
|
||||||
|
'am',
|
||||||
|
'an',
|
||||||
|
'ar',
|
||||||
|
'as',
|
||||||
|
'av',
|
||||||
|
'ay',
|
||||||
|
'az',
|
||||||
|
'ba',
|
||||||
|
'be',
|
||||||
|
'bg',
|
||||||
|
'bh',
|
||||||
|
'bi',
|
||||||
|
'bm',
|
||||||
|
'bn',
|
||||||
|
'bo',
|
||||||
|
'br',
|
||||||
|
'bs',
|
||||||
|
'ca',
|
||||||
|
'ce',
|
||||||
|
'ch',
|
||||||
|
'co',
|
||||||
|
'cr',
|
||||||
|
'cs',
|
||||||
|
'cu',
|
||||||
|
'cv',
|
||||||
|
'cy',
|
||||||
|
'da',
|
||||||
|
'de',
|
||||||
|
'dv',
|
||||||
|
'dz',
|
||||||
|
'ee',
|
||||||
|
'el',
|
||||||
|
'eo',
|
||||||
|
'es',
|
||||||
|
'et',
|
||||||
|
'eu',
|
||||||
|
'fa',
|
||||||
|
'ff',
|
||||||
|
'fi',
|
||||||
|
'fj',
|
||||||
|
'fo',
|
||||||
|
'fr',
|
||||||
|
'fy',
|
||||||
|
'ga',
|
||||||
|
'gd',
|
||||||
|
'gl',
|
||||||
|
'gn',
|
||||||
|
'gu',
|
||||||
|
'gv',
|
||||||
|
'ha',
|
||||||
|
'he',
|
||||||
|
'hi',
|
||||||
|
'ho',
|
||||||
|
'hr',
|
||||||
|
'ht',
|
||||||
|
'hu',
|
||||||
|
'hy',
|
||||||
|
'hz',
|
||||||
|
'ia',
|
||||||
|
'id',
|
||||||
|
'ie',
|
||||||
|
'ig',
|
||||||
|
'ii',
|
||||||
|
'ik',
|
||||||
|
'io',
|
||||||
|
'is',
|
||||||
|
'it',
|
||||||
|
'iu',
|
||||||
|
'ja',
|
||||||
|
'jv',
|
||||||
|
'ka',
|
||||||
|
'kg',
|
||||||
|
'ki',
|
||||||
|
'kj',
|
||||||
|
'kk',
|
||||||
|
'kl',
|
||||||
|
'km',
|
||||||
|
'kn',
|
||||||
|
'ko',
|
||||||
|
'kr',
|
||||||
|
'ks',
|
||||||
|
'ku',
|
||||||
|
'kv',
|
||||||
|
'kw',
|
||||||
|
'ky',
|
||||||
|
'la',
|
||||||
|
'lb',
|
||||||
|
'lg',
|
||||||
|
'li',
|
||||||
|
'ln',
|
||||||
|
'lo',
|
||||||
|
'lt',
|
||||||
|
'lu',
|
||||||
|
'lv',
|
||||||
|
'mg',
|
||||||
|
'mh',
|
||||||
|
'mi',
|
||||||
|
'mk',
|
||||||
|
'ml',
|
||||||
|
'mn',
|
||||||
|
'mr',
|
||||||
|
'ms',
|
||||||
|
'mt',
|
||||||
|
'my',
|
||||||
|
'na',
|
||||||
|
'nb',
|
||||||
|
'nd',
|
||||||
|
'ne',
|
||||||
|
'ng',
|
||||||
|
'nl',
|
||||||
|
'nn',
|
||||||
|
'no',
|
||||||
|
'nr',
|
||||||
|
'nv',
|
||||||
|
'ny',
|
||||||
|
'oc',
|
||||||
|
'oj',
|
||||||
|
'om',
|
||||||
|
'or',
|
||||||
|
'os',
|
||||||
|
'pa',
|
||||||
|
'pi',
|
||||||
|
'pl',
|
||||||
|
'ps',
|
||||||
|
'pt',
|
||||||
|
'qu',
|
||||||
|
'rm',
|
||||||
|
'rn',
|
||||||
|
'ro',
|
||||||
|
'ru',
|
||||||
|
'rw',
|
||||||
|
'sa',
|
||||||
|
'sc',
|
||||||
|
'sd',
|
||||||
|
'se',
|
||||||
|
'sg',
|
||||||
|
'si',
|
||||||
|
'sk',
|
||||||
|
'sl',
|
||||||
|
'sm',
|
||||||
|
'sn',
|
||||||
|
'so',
|
||||||
|
'sq',
|
||||||
|
'sr',
|
||||||
|
'ss',
|
||||||
|
'st',
|
||||||
|
'su',
|
||||||
|
'sv',
|
||||||
|
'sw',
|
||||||
|
'ta',
|
||||||
|
'te',
|
||||||
|
'tg',
|
||||||
|
'th',
|
||||||
|
'ti',
|
||||||
|
'tk',
|
||||||
|
'tl',
|
||||||
|
'tn',
|
||||||
|
'to',
|
||||||
|
'tr',
|
||||||
|
'ts',
|
||||||
|
'tt',
|
||||||
|
'tw',
|
||||||
|
'ty',
|
||||||
|
'ug',
|
||||||
|
'uk',
|
||||||
|
'ur',
|
||||||
|
'uz',
|
||||||
|
've',
|
||||||
|
'vi',
|
||||||
|
'vo',
|
||||||
|
'wa',
|
||||||
|
'wo',
|
||||||
|
'xh',
|
||||||
|
'yi',
|
||||||
|
'yo',
|
||||||
|
'za',
|
||||||
|
'zh',
|
||||||
|
'zu'
|
||||||
|
]
|
||||||
Loaded 100 of 193 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user