Compare commits
808
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f7cae69704 | ||
|
|
b7d58bcdbc | ||
|
|
13a856b843 | ||
|
|
8da38985c3 | ||
|
|
60cf6c6b97 | ||
|
|
35c2b34564 | ||
|
|
ef2e048efc | ||
|
|
6b3261aa33 | ||
|
|
1849c02cb6 | ||
|
|
1ec74a89e2 | ||
|
|
c591792de9 | ||
|
|
3108543ae5 | ||
|
|
1eb221c743 | ||
|
|
bebf6bc2e7 | ||
|
|
9e91cc2138 | ||
|
|
c5b939cfb7 | ||
|
|
5bd411ca27 | ||
|
|
a533cda6f0 | ||
|
|
fe4b07b8ae | ||
|
|
f9f2ccd904 | ||
|
|
d9e87d7c32 | ||
|
|
a0092c0770 | ||
|
|
3100131125 | ||
|
|
988880cf83 | ||
|
|
c3fb9672c4 | ||
|
|
0a2d94e425 | ||
|
|
8d9073cd31 | ||
|
|
d075961ffa | ||
|
|
7a72409b61 | ||
|
|
34fc530fba | ||
|
|
f257ff2f97 | ||
|
|
7ad5822c5b | ||
|
|
9a8f9f0a94 | ||
|
|
6421cecafb | ||
|
|
be544d6d89 | ||
|
|
3c89ecafdd | ||
|
|
35ec4eec52 | ||
|
|
e47f737a2f | ||
|
|
ac671a065b | ||
|
|
74116cc550 | ||
|
|
406070a5c3 | ||
|
|
0ccafd5b53 | ||
|
|
940f517aa3 | ||
|
|
216e5f65ad | ||
|
|
a74685d66d | ||
|
|
b7791d2845 | ||
|
|
d151a82d78 | ||
|
|
8ce61fbd52 | ||
|
|
90c24aade3 | ||
|
|
6b3f787fee | ||
|
|
4ebe4ce1b7 | ||
|
|
8c79740ee8 | ||
|
|
59d027ca02 | ||
|
|
37a7345a90 | ||
|
|
c519d4651b | ||
|
|
9b3b609e40 | ||
|
|
6254f53716 | ||
|
|
f05dc46432 | ||
|
|
3de0982a4a | ||
|
|
c2184fb3bf | ||
|
|
919c09fcb0 | ||
|
|
1d9dbd40ec | ||
|
|
0cd953a6f3 | ||
|
|
4db2b72351 | ||
|
|
dd54fcbdbd | ||
|
|
3123cf7ac6 | ||
|
|
6b579dd4ce | ||
|
|
16dfaa3e27 | ||
|
|
d7842b9f84 | ||
|
|
115034fccb | ||
|
|
309e957a85 | ||
|
|
d7007e402e | ||
|
|
91323a21cf | ||
|
|
fea893d76c | ||
|
|
761bc6ba4c | ||
|
|
75172feb4e | ||
|
|
3285fb1608 | ||
|
|
03a4c6910d | ||
|
|
485b958599 | ||
|
|
da47ba2f67 | ||
|
|
c39195488a | ||
|
|
227fb0ae9b | ||
|
|
b12ff5b503 | ||
|
|
0946c72b88 | ||
|
|
7d49b046d4 | ||
|
|
5f0426c840 | ||
|
|
73e239cc5f | ||
|
|
ad670f721a | ||
|
|
028a4a70cf | ||
|
|
77d7960347 | ||
|
|
39821146bd | ||
|
|
7d505a41ac | ||
|
|
e457b2f0d6 | ||
|
|
c9cf7fd4d4 | ||
|
|
b0371dd33d | ||
|
|
25e16c3565 | ||
|
|
7b39527863 | ||
|
|
d861b08866 | ||
|
|
fb438dc108 | ||
|
|
4e6b4f179b | ||
|
|
00d038c8f3 | ||
|
|
a9f6a68952 | ||
|
|
b9142bbc5a | ||
|
|
6c812f663e | ||
|
|
a93ec9783a | ||
|
|
2d184d77b6 | ||
|
|
bce299ccc7 | ||
|
|
235cebd14a | ||
|
|
a638aa9d53 | ||
|
|
67cce0ef7e | ||
|
|
82f4267bf6 | ||
|
|
45a9ca29c4 | ||
|
|
7f4e813277 | ||
|
|
3805ff4a0c | ||
|
|
464cfd475e | ||
|
|
fe469ae57f | ||
|
|
550ef9a1c4 | ||
|
|
935adfb51a | ||
|
|
3974df4a62 | ||
|
|
4870974161 | ||
|
|
8c4b0037f5 | ||
|
|
2c6f763ef2 | ||
|
|
ca28de02d8 | ||
|
|
bfc15ea029 | ||
|
|
6e8b8a5920 | ||
|
|
099f3b6a62 | ||
|
|
142d182bc1 | ||
|
|
1437871d88 | ||
|
|
352bf69409 | ||
|
|
9bdf3d23e1 | ||
|
|
be8ecfa707 | ||
|
|
51da0d0259 | ||
|
|
f55b78a994 | ||
|
|
e1a44c93f8 | ||
|
|
07e7087a09 | ||
|
|
2c79c7e2f6 | ||
|
|
09f6637fe0 | ||
|
|
3784db3308 | ||
|
|
2b950ff5dd | ||
|
|
09339c9cfb | ||
|
|
ccadd88af5 | ||
|
|
cc02a0efc2 | ||
|
|
43a1385b79 | ||
|
|
5101464e3b | ||
|
|
3d71478d38 | ||
|
|
4989ed445e | ||
|
|
d9413039ec | ||
|
|
eba0c9be34 | ||
|
|
48c9e9f3cc | ||
|
|
81ebde88db | ||
|
|
79ced9d0f8 | ||
|
|
a4058b84ce | ||
|
|
7bf211a52b | ||
|
|
d5f722792f | ||
|
|
0f02906c9b | ||
|
|
9582e228b1 | ||
|
|
45f20431f9 | ||
|
|
7554e6d7f9 | ||
|
|
cb8f26f177 | ||
|
|
b5dfce7861 | ||
|
|
2ca5a65544 | ||
|
|
17deb136db | ||
|
|
8c9710c76c | ||
|
|
32f7ecb261 | ||
|
|
fb77fde710 | ||
|
|
3c67bb90d7 | ||
|
|
dabb168853 | ||
|
|
45e5b3b219 | ||
|
|
a6b7469923 | ||
|
|
cb5dab3033 | ||
|
|
21d0038ff2 | ||
|
|
c094d8f2e8 | ||
|
|
c465d6a6c2 | ||
|
|
73d35bc985 | ||
|
|
2a8ccb065b | ||
|
|
8f04a50ce1 | ||
|
|
888aa5586b | ||
|
|
99f56f5d22 | ||
|
|
ad6281090d | ||
|
|
f0d334d3e2 | ||
|
|
5f829b048f | ||
|
|
1a961e66ff | ||
|
|
fdb0e22656 | ||
|
|
132ee1915f | ||
|
|
44bf4f3c8f | ||
|
|
6237767d5a | ||
|
|
dec9d96417 | ||
|
|
b167c87267 | ||
|
|
2280fe8e8e | ||
|
|
575d6dcd2d | ||
|
|
f729490c6b | ||
|
|
b32124cdd6 | ||
|
|
3d4321ee38 | ||
|
|
85034b382e | ||
|
|
77a51d1ad4 | ||
|
|
33e0cdc2d7 | ||
|
|
6519faa2fe | ||
|
|
5e3a234cbe | ||
|
|
e54c31d2d5 | ||
|
|
66c0537251 | ||
|
|
ac58516593 | ||
|
|
c3da6322b5 | ||
|
|
3d241500cf | ||
|
|
ded8224f66 | ||
|
|
f8814881a1 | ||
|
|
cc2852cd48 | ||
|
|
467637a9eb | ||
|
|
3cfc292d84 | ||
|
|
6acf94a810 | ||
|
|
31367fb4c4 | ||
|
|
12d6074e3b | ||
|
|
ff30386051 | ||
|
|
601f99ac16 | ||
|
|
87fe5c6101 | ||
|
|
68399ca31c | ||
|
|
2a6d7fd80f | ||
|
|
4725f510d8 | ||
|
|
be0ba22222 | ||
|
|
c8781392be | ||
|
|
b97164fcfb | ||
|
|
0dfb92281b | ||
|
|
4fe80c40da | ||
|
|
f0fac5115a | ||
|
|
46dd389d0d | ||
|
|
1e28e21ab5 | ||
|
|
7832c62c5d | ||
|
|
d025ee9dbe | ||
|
|
a9a9cb4319 | ||
|
|
aa727cb9b1 | ||
|
|
b8c9a99f20 | ||
|
|
aff995b0d0 | ||
|
|
2cc7e5dfdc | ||
|
|
5235a150b1 | ||
|
|
c6372ea9de | ||
|
|
7df4cc44c4 | ||
|
|
d47cf40544 | ||
|
|
7f5d88e95c | ||
|
|
d09663c066 | ||
|
|
ef97c9b69f | ||
|
|
d855e6c8b1 | ||
|
|
cd66f7eb43 | ||
|
|
6a35a7ba4c | ||
|
|
a3e146dc68 | ||
|
|
b81305a4a9 | ||
|
|
73884b34bc | ||
|
|
6166a34db2 | ||
|
|
6fa7da4b1c | ||
|
|
c3e426c491 | ||
|
|
21e023f0db | ||
|
|
063be001b3 | ||
|
|
5dff02e8bc | ||
|
|
60a59407d8 | ||
|
|
20a5aecfca | ||
|
|
c2e7b5a67d | ||
|
|
8f32303d07 | ||
|
|
891b1e7782 | ||
|
|
f26394fd3b | ||
|
|
4d83d42b4c | ||
|
|
57f1108df2 | ||
|
|
2641a9abe5 | ||
|
|
6b193ab350 | ||
|
|
b1bb37511c | ||
|
|
319187d6d6 | ||
|
|
02eb789f84 | ||
|
|
5a9338a27f | ||
|
|
eb6924277f | ||
|
|
325419404d | ||
|
|
bd8f371fd5 | ||
|
|
1783ff2845 | ||
|
|
d388527ffa | ||
|
|
19494088bd | ||
|
|
920dad524a | ||
|
|
ec89bcac8e | ||
|
|
a916c1f4ad | ||
|
|
a9a0ac92d7 | ||
|
|
da8a8bd1ef | ||
|
|
d9c746891d | ||
|
|
67817005b5 | ||
|
|
24d11de5a7 | ||
|
|
9251c87323 | ||
|
|
e12fab90d1 | ||
|
|
0a194b5b01 | ||
|
|
8d028adc53 | ||
|
|
dfca15395e | ||
|
|
e21f2362fe | ||
|
|
1ce328e8a9 | ||
|
|
038a5f999f | ||
|
|
5d3704c7ea | ||
|
|
87037c06c9 | ||
|
|
dd412c0f50 | ||
|
|
bf44befff6 | ||
|
|
e61874bb6f | ||
|
|
1e5331768f | ||
|
|
ec9a3a4f7c | ||
|
|
e439a3a8dc | ||
|
|
19f70d7a11 | ||
|
|
afe7ed5b05 | ||
|
|
d4bf004d74 | ||
|
|
e4d06a088b | ||
|
|
0929088b12 | ||
|
|
7b4838fc9b | ||
|
|
0cf9533248 | ||
|
|
84ff0b8a9f | ||
|
|
d467dcfeaf | ||
|
|
8e68ba4751 | ||
|
|
0f2a85ba9f | ||
|
|
7674a0a91e | ||
|
|
5bc1a66572 | ||
|
|
9b56067213 | ||
|
|
9a9df2fc3c | ||
|
|
9989d8d1d4 | ||
|
|
f9471f297e | ||
|
|
146b693e4a | ||
|
|
7295b7e329 | ||
|
|
e2441ea3e7 | ||
|
|
119e51912e | ||
|
|
dd950f5b0d | ||
|
|
78a9bad1e1 | ||
|
|
0c6eaf5484 | ||
|
|
1010068ddb | ||
|
|
82eec3d8d7 | ||
|
|
ee7b37d3f3 | ||
|
|
143d82d242 | ||
|
|
8b91b38855 | ||
|
|
1098f0d2a3 | ||
|
|
ab53cec022 | ||
|
|
6f5f8e5648 | ||
|
|
edfd707c22 | ||
|
|
1870f30af8 | ||
|
|
90106f5f08 | ||
|
|
9924b7b438 | ||
|
|
aa37faab0a | ||
|
|
dc10f8ce72 | ||
|
|
996686c1da | ||
|
|
488785d013 | ||
|
|
3abdc01230 | ||
|
|
8da04a584f | ||
|
|
27cc61d45e | ||
|
|
7371c30064 | ||
|
|
140d163895 | ||
|
|
dc33bdc1dc | ||
|
|
74df4fab83 | ||
|
|
1e5cd3d7a1 | ||
|
|
a54e9b64aa | ||
|
|
74660704e3 | ||
|
|
7439893a2a | ||
|
|
e27e49e9dc | ||
|
|
34ed729c59 | ||
|
|
adaeeca3fd | ||
|
|
dac75563d3 | ||
|
|
cbc76adcaa | ||
|
|
69a9cb383d | ||
|
|
4343073c00 | ||
|
|
fe60d4be88 | ||
|
|
ae337807f5 | ||
|
|
9ae30ac08e | ||
|
|
62fa85c0a4 | ||
|
|
7bb873dad9 | ||
|
|
5f6c1c14cb | ||
|
|
d43189ad33 | ||
|
|
fcad76fc51 | ||
|
|
97e6e1684e | ||
|
|
67a0d3e926 | ||
|
|
183fb9f9ff | ||
|
|
9815ddef1f | ||
|
|
f6d0847453 | ||
|
|
b0b9f0d65f | ||
|
|
0cec80f676 | ||
|
|
48c64143e3 | ||
|
|
a8712422bc | ||
|
|
97f65bd283 | ||
|
|
fd3c1c50f1 | ||
|
|
b153e4bb9f | ||
|
|
db9856a8db | ||
|
|
75ecea265d | ||
|
|
be8751cb73 | ||
|
|
fb25ecb4a1 | ||
|
|
f1cb7d27ac | ||
|
|
dee494e12f | ||
|
|
b13a121915 | ||
|
|
7486ee9537 | ||
|
|
4a20ccc28e | ||
|
|
f80dd2b307 | ||
|
|
b208cf6d32 | ||
|
|
39e78ff17e | ||
|
|
a8177ea7fe | ||
|
|
bedcfc154b | ||
|
|
4c38b4aa3c | ||
|
|
f6cfe266e0 | ||
|
|
4905e65f14 | ||
|
|
ccb250b410 | ||
|
|
aca57ffc62 | ||
|
|
7f375f42d8 | ||
|
|
eedcc2034d | ||
|
|
24c9a167d7 | ||
|
|
909df8ef1f | ||
|
|
3b27cb3671 | ||
|
|
3fe0db4a7d | ||
|
|
8b55814ab2 | ||
|
|
575e471553 | ||
|
|
0f5f1aebed | ||
|
|
50e17eb1ab | ||
|
|
158cc2f660 | ||
|
|
1066a31acd | ||
|
|
1f9d0f4582 | ||
|
|
a6d65233f1 | ||
|
|
eff2fe7a1b | ||
|
|
20efdc70b3 | ||
|
|
f0d8fb8f1a | ||
|
|
f7a380e9b7 | ||
|
|
e9c7cf6f63 | ||
|
|
68f1661452 | ||
|
|
36fd1b91ae | ||
|
|
a4ec430ac0 | ||
|
|
519614b2fd | ||
|
|
bf0118c8ef | ||
|
|
a4db0820bc | ||
|
|
ee7528413e | ||
|
|
7952fc8324 | ||
|
|
652773d2cf | ||
|
|
2a17787242 | ||
|
|
0a53ad5721 | ||
|
|
6da6bdc863 | ||
|
|
42ad2bb83f | ||
|
|
f309a65cb4 | ||
|
|
77e19ab1a4 | ||
|
|
1a996b6ef3 | ||
|
|
b882f1a010 | ||
|
|
82a030e6ff | ||
|
|
0758b85179 | ||
|
|
ab3d9bd080 | ||
|
|
8ff813f689 | ||
|
|
88ae72c0d3 | ||
|
|
312aa4be26 | ||
|
|
cbb06fce9d | ||
|
|
f259e497c4 | ||
|
|
dd4172ac66 | ||
|
|
66029e60d3 | ||
|
|
364f484f04 | ||
|
|
9dd5159414 | ||
|
|
13e38d6fd8 | ||
|
|
10dcb64715 | ||
|
|
7551b51e7d | ||
|
|
adb418aafc | ||
|
|
270da80d64 | ||
|
|
b2027cfd66 | ||
|
|
7f1f4eeac6 | ||
|
|
7a7446c8bd | ||
|
|
ddbae294e6 | ||
|
|
8c71b744f3 | ||
|
|
479b5d31a9 | ||
|
|
4cbf4230e8 | ||
|
|
6a610187e0 | ||
|
|
eb2a4aebba | ||
|
|
21a2e67755 | ||
|
|
3b9e312615 | ||
|
|
26dab04c9e | ||
|
|
00713c0d11 | ||
|
|
751b5f3027 | ||
|
|
e8261b000e | ||
|
|
41ecb70297 | ||
|
|
09ee104b8c | ||
|
|
e3a4964787 | ||
|
|
9bf72910a4 | ||
|
|
ee39e20e6d | ||
|
|
399d6db6f6 | ||
|
|
0821ce44b5 | ||
|
|
ea279111c6 | ||
|
|
674ce02e58 | ||
|
|
8dfa2767ec | ||
|
|
20dad7f07f | ||
|
|
751cc4c44d | ||
|
|
2318e6d8e9 | ||
|
|
61b4a492c3 | ||
|
|
9db3d01e09 | ||
|
|
8da73ad3dd | ||
|
|
b8c16d8ac5 | ||
|
|
429c0951f3 | ||
|
|
74e103c791 | ||
|
|
f941950ee2 | ||
|
|
846df2eef1 | ||
|
|
34ed058c97 | ||
|
|
eae0290978 | ||
|
|
561368570e | ||
|
|
3467d1fed0 | ||
|
|
d02ff232e5 | ||
|
|
2d1c6a5402 | ||
|
|
eab3b65629 | ||
|
|
20b435732a | ||
|
|
929617273d | ||
|
|
2717bf7d49 | ||
|
|
5cd2ebc960 | ||
|
|
9b4afe9816 | ||
|
|
23bb5598d5 | ||
|
|
af1d7813e9 | ||
|
|
16c2e5a585 | ||
|
|
c02750edbd | ||
|
|
7204ddafec | ||
|
|
faeba9a7e4 | ||
|
|
190d238a1f | ||
|
|
715451b5fb | ||
|
|
87f1895405 | ||
|
|
923d817751 | ||
|
|
0728209b66 | ||
|
|
b8b9dcc2ee | ||
|
|
f35e879852 | ||
|
|
34f4f12eb9 | ||
|
|
fa63bf758d | ||
|
|
f6b396ae64 | ||
|
|
2c7fd58e34 | ||
|
|
982f2c9634 | ||
|
|
f2fd42b47a | ||
|
|
1b4ccad938 | ||
|
|
a9de1ce8e0 | ||
|
|
ac752d5ec2 | ||
|
|
632d8d02d2 | ||
|
|
48aeb26e02 | ||
|
|
1694af8b5e | ||
|
|
83bcab9cd2 | ||
|
|
bdc7f4b3f5 | ||
|
|
39202a3d79 | ||
|
|
912065a121 | ||
|
|
c8466afac2 | ||
|
|
2619e162c1 | ||
|
|
e1112e17f8 | ||
|
|
92b2ead74c | ||
|
|
bbed9b94c1 | ||
|
|
73d07311db | ||
|
|
1cdff47477 | ||
|
|
511a5c3f82 | ||
|
|
853885e2ff | ||
|
|
d83936a66a | ||
|
|
5517d2bf56 | ||
|
|
f21ab49ac5 | ||
|
|
925a458abe | ||
|
|
76946c447f | ||
|
|
2faa29b1c4 | ||
|
|
6826cc311d | ||
|
|
5e17ce0a0b | ||
|
|
e8d299d3b6 | ||
|
|
7637aa2ab6 | ||
|
|
ab067d1d3a | ||
|
|
e6f84666c7 | ||
|
|
4c5429af15 | ||
|
|
0a0ac3b7c9 | ||
|
|
24833ce9fb | ||
|
|
ec2c18dc87 | ||
|
|
7384609e74 | ||
|
|
3047649650 | ||
|
|
d298dac3f3 | ||
|
|
1574bca8a8 | ||
|
|
ec2f6c6b80 | ||
|
|
838cc60161 | ||
|
|
310c61a5cc | ||
|
|
318cc15323 | ||
|
|
3a64ceb4d6 | ||
|
|
d0f21c0095 | ||
|
|
46dc15dd29 | ||
|
|
8dc654b513 | ||
|
|
7000ac3f3f | ||
|
|
43c2e8d8e9 | ||
|
|
0231139b01 | ||
|
|
d6ee6446dd | ||
|
|
7b666efcf8 | ||
|
|
eb5d2198fc | ||
|
|
34e44ebd1c | ||
|
|
bf2f4bc040 | ||
|
|
9dc4559aba | ||
|
|
eba8856261 | ||
|
|
a8c5aa471a | ||
|
|
52f6dcf092 | ||
|
|
dec79f3742 | ||
|
|
8bdcac0f3e | ||
|
|
8426b674a3 | ||
|
|
da391bcc8d | ||
|
|
2d7443d454 | ||
|
|
991987ed76 | ||
|
|
ec24ebf2cf | ||
|
|
6ba0976085 | ||
|
|
2b88d01a01 | ||
|
|
0c09f24cbf | ||
|
|
61d22afeba | ||
|
|
9f1ed6e8c3 | ||
|
|
bbc4113cac | ||
|
|
91194bf422 | ||
|
|
9c5f940b00 | ||
|
|
455b4043b8 | ||
|
|
bd83ee7931 | ||
|
|
f6bdf7c09a | ||
|
|
2db8afb8c2 | ||
|
|
e033129dd3 | ||
|
|
f9dc590100 | ||
|
|
8996aafe0d | ||
|
|
9dc25ef7af | ||
|
|
2b84f4d407 | ||
|
|
097c8b674c | ||
|
|
ba649d4b94 | ||
|
|
6ed1614db0 | ||
|
|
df5b6a8380 | ||
|
|
1f82a8b99e | ||
|
|
0c95d96f32 | ||
|
|
c2f5f84118 | ||
|
|
b3b5e3d8f0 | ||
|
|
506d3f3cd9 | ||
|
|
516a8c5ee5 | ||
|
|
2f81e9d374 | ||
|
|
2d8703bb8d | ||
|
|
76e60d9bc3 | ||
|
|
9d5370be5f | ||
|
|
fc1a06bc45 | ||
|
|
fce80374f4 | ||
|
|
420c9f10c2 | ||
|
|
5a39681a2e | ||
|
|
7a1b7db7c8 | ||
|
|
03a643da52 | ||
|
|
383f0ce450 | ||
|
|
c6c668676c | ||
|
|
7b01dde063 | ||
|
|
8c25f65024 | ||
|
|
1e478e3545 | ||
|
|
6c75a8978b | ||
|
|
640267fc2b | ||
|
|
33c7c3ee12 | ||
|
|
f9b41d34ae | ||
|
|
d4bec79451 | ||
|
|
ac1a8b4daf | ||
|
|
28838c1759 | ||
|
|
50ecb0dac9 | ||
|
|
e22bc01cbd | ||
|
|
6c28713a4c | ||
|
|
fc9023386c | ||
|
|
e6cae9bcc3 | ||
|
|
a9eeca1302 | ||
|
|
8c695e42ca | ||
|
|
0aa7fd47d5 | ||
|
|
70596042d6 | ||
|
|
caf616234b | ||
|
|
375187aa70 | ||
|
|
71eccdc0e3 | ||
|
|
639b1e48f5 | ||
|
|
0bb4cb4472 | ||
|
|
cc51543851 | ||
|
|
22540390e1 | ||
|
|
98565eb67c | ||
|
|
cf6a47ecb7 | ||
|
|
fa60b9f9d3 | ||
|
|
644120ca31 | ||
|
|
a50a625b3b | ||
|
|
7297c13331 | ||
|
|
f73399bfac | ||
|
|
a056cd78f7 | ||
|
|
a30f3c86c2 | ||
|
|
e70bdd86a7 | ||
|
|
bc9f33c2e0 | ||
|
|
872b89ee93 | ||
|
|
ae53062518 | ||
|
|
17f76c9cb3 | ||
|
|
5de944146a | ||
|
|
9dc6092cb0 | ||
|
|
a32a2ef04e | ||
|
|
ecfa0ae3da | ||
|
|
03595052ce | ||
|
|
1f94c53dd2 | ||
|
|
9c426373f2 | ||
|
|
c03e30a01f | ||
|
|
07f7a77ac0 | ||
|
|
3b9ea2c9a4 | ||
|
|
1beb13dd80 | ||
|
|
ddae84abb3 | ||
|
|
863b9a2c98 | ||
|
|
9d44bbdb48 | ||
|
|
8d93dd5adc | ||
|
|
48502961cf | ||
|
|
3c8bec61d3 | ||
|
|
7296c7df1a | ||
|
|
f3ee6603de | ||
|
|
ee0aabda1d | ||
|
|
08d37a4b0f | ||
|
|
f975ea99cb | ||
|
|
f030d41dc7 | ||
|
|
8d079bfcd1 | ||
|
|
d39b8654a6 | ||
|
|
ce7816a968 | ||
|
|
2ee572e68f | ||
|
|
4bbd850898 | ||
|
|
34eae6e608 | ||
|
|
6a0302fec6 | ||
|
|
c94cc293c2 | ||
|
|
cae7792a1e | ||
|
|
7f6b2fe4f1 | ||
|
|
eba430bbc0 | ||
|
|
01280c8d04 | ||
|
|
ec141ac9c9 | ||
|
|
590c892a6a | ||
|
|
ff8a50c366 | ||
|
|
46ef6c8ab7 | ||
|
|
b09eabc478 | ||
|
|
e49fcea6e3 | ||
|
|
68ed9f4ffc | ||
|
|
af94687d45 | ||
|
|
26964ecf0f | ||
|
|
77d19af359 | ||
|
|
120b82f243 | ||
|
|
a0fea30a11 | ||
|
|
8bb3e0a64d | ||
|
|
bbded12923 | ||
|
|
e8ba5d7606 | ||
|
|
af66b31a44 | ||
|
|
b000a40f28 | ||
|
|
3c85322523 | ||
|
|
a469b8bc04 | ||
|
|
78b8261a3a | ||
|
|
f20ca70c01 | ||
|
|
4e4148fc1c | ||
|
|
c22482f907 | ||
|
|
870e139fce | ||
|
|
4d58648c02 | ||
|
|
ebbb182537 | ||
|
|
1cd5377b45 | ||
|
|
1d1f0527ee | ||
|
|
37a5f77415 | ||
|
|
ced3c7efe4 | ||
|
|
c3b8f366ed | ||
|
|
de78876b1a | ||
|
|
64c25b049c | ||
|
|
8a4fe4f3ad | ||
|
|
8811b8c1fd | ||
|
|
190b01fdf9 | ||
|
|
f145d08c10 | ||
|
|
53382b7e15 | ||
|
|
6ad0242617 | ||
|
|
a7c2408c0a | ||
|
|
ce1eabaed6 | ||
|
|
f602541ede | ||
|
|
3f718e6efc | ||
|
|
ce7a985df6 | ||
|
|
6d83f7e7bd | ||
|
|
b73c00943c | ||
|
|
abaac8ef48 | ||
|
|
a2f8e7068e | ||
|
|
4d47873219 | ||
|
|
cf985486e5 | ||
|
|
b930c3fc93 | ||
|
|
dd26a96828 | ||
|
|
6865ddfc12 | ||
|
|
e888e69d4d | ||
|
|
2089059792 | ||
|
|
27739e0364 | ||
|
|
698ee271d6 | ||
|
|
543c75b293 | ||
|
|
b09c46f6f7 | ||
|
|
f2cc19e6aa | ||
|
|
814a0a123f | ||
|
|
179383540f | ||
|
|
8be1c8310d | ||
|
|
ef02d776ca | ||
|
|
750ff448ad | ||
|
|
e3abab6d4d | ||
|
|
d3ffae72fb | ||
|
|
87f751188e | ||
|
|
3469abaefd | ||
|
|
797364ee5c | ||
|
|
36c05fc4b9 | ||
|
|
79624febc0 | ||
|
|
0a9d4de126 | ||
|
|
b6f6994db4 | ||
|
|
ff7bed720a | ||
|
|
00c0f48b02 | ||
|
|
94c45cf2a0 | ||
|
|
58f77b2a1c | ||
|
|
7170e69b22 | ||
|
|
239ee2437c | ||
|
|
ced368db31 | ||
|
|
6a991e5c15 | ||
|
|
523d22262b | ||
|
|
08197a327e | ||
|
|
b7cb2a7aa5 | ||
|
|
decc5c74ef | ||
|
|
fbe0f886b6 | ||
|
|
7e23d6e2ef | ||
|
|
49458d1085 | ||
|
|
289c12bd9a | ||
|
|
9a6326b027 | ||
|
|
51f573f1ea | ||
|
|
c8b7cd8862 | ||
|
|
21c112d059 | ||
|
|
9432e1b5b2 | ||
|
|
8269d2f83c | ||
|
|
5a4b6be974 | ||
|
|
35e8ce60a9 | ||
|
|
8b6dd9f603 | ||
|
|
084f0ebdab | ||
|
|
58640c1521 | ||
|
|
7ffdfd12f8 | ||
|
|
cb9a30f285 | ||
|
|
e48bef809f | ||
|
|
f5d7570102 | ||
|
|
a600c60cf8 | ||
|
|
695eabd026 | ||
|
|
e81b51a647 | ||
|
|
b7e95ff090 | ||
|
|
3ca41be686 | ||
|
|
3152046173 | ||
|
|
3a98dc8a95 | ||
|
|
d737b28916 | ||
|
|
97c0dac876 | ||
|
|
006494b1fa | ||
|
|
8586762dde |
No files matched your search
+66
-98
@@ -1,12 +1,17 @@
|
||||
name: ci
|
||||
on: push
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
tags:
|
||||
- 'v[0-9]+.[0-9]+.[0-9]+'
|
||||
|
||||
jobs:
|
||||
|
||||
lint:
|
||||
name: lint
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
@@ -16,9 +21,8 @@ jobs:
|
||||
path: ~/.cache/pip
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
|
||||
restore-keys: ${{ runner.os }}-pip-
|
||||
- run: |
|
||||
pip install --user --upgrade pip wheel
|
||||
pip install -e .[lint]
|
||||
- run: pip install --user --upgrade pip wheel
|
||||
- run: pip install -e .[torrent,lint]
|
||||
- run: make lint
|
||||
|
||||
tests-unit:
|
||||
@@ -27,11 +31,11 @@ jobs:
|
||||
matrix:
|
||||
os:
|
||||
- ubuntu-latest
|
||||
- macos-latest
|
||||
- windows-latest
|
||||
#- macos-latest
|
||||
#- windows-latest
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
@@ -44,44 +48,33 @@ jobs:
|
||||
path: ${{ steps.pip-cache.outputs.dir }}
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
|
||||
restore-keys: ${{ runner.os }}-pip-
|
||||
- run: |
|
||||
pip install --user --upgrade pip wheel
|
||||
pip install -e .[test]
|
||||
- run: pip install --user --upgrade pip wheel
|
||||
- run: pip install -e .[torrent,test]
|
||||
- env:
|
||||
HOME: /tmp
|
||||
run: coverage run -m unittest discover -v tests.unit
|
||||
- env:
|
||||
COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
COVERALLS_PARALLEL: true
|
||||
name: Submit to coveralls
|
||||
run: |
|
||||
pip install https://github.com/bboe/coveralls-python/archive/github_actions.zip
|
||||
coveralls
|
||||
run: make test-unit-coverage
|
||||
|
||||
tests-integration:
|
||||
name: "tests / integration"
|
||||
runs-on: ubuntu-20.04
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
test:
|
||||
# - datanetwork
|
||||
- datanetwork
|
||||
- blockchain
|
||||
# - other
|
||||
db:
|
||||
- sqlite
|
||||
- postgres
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:12
|
||||
env:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: postgres
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
|
||||
- other
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- name: Configure sysctl limits
|
||||
run: |
|
||||
sudo swapoff -a
|
||||
sudo sysctl -w vm.swappiness=1
|
||||
sudo sysctl -w fs.file-max=262144
|
||||
sudo sysctl -w vm.max_map_count=262144
|
||||
- name: Runs Elasticsearch
|
||||
uses: elastic/elastic-github-actions/elasticsearch@master
|
||||
with:
|
||||
stack-version: 7.6.0
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
@@ -93,34 +86,14 @@ jobs:
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ./.tox
|
||||
key: tox-integration-${{ matrix.test }}-${{ matrix.db }}-${{ hashFiles('setup.py') }}
|
||||
restore-keys: txo-integration-${{ matrix.test }}-${{ matrix.db }}-
|
||||
key: tox-integration-${{ matrix.test }}-${{ hashFiles('setup.py') }}
|
||||
restore-keys: txo-integration-${{ matrix.test }}-
|
||||
- run: pip install tox
|
||||
- env:
|
||||
TEST_DB: ${{ matrix.db }}
|
||||
run: tox -e ${{ matrix.test }}
|
||||
- env:
|
||||
COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
COVERALLS_PARALLEL: true
|
||||
name: Submit to coveralls
|
||||
run: |
|
||||
pip install https://github.com/bboe/coveralls-python/archive/github_actions.zip
|
||||
coverage combine tests
|
||||
coveralls
|
||||
|
||||
coveralls-finished:
|
||||
needs: ["tests-unit", "tests-integration"]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Coveralls Finished
|
||||
uses: coverallsapp/github-action@57daa114
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
parallel-finished: true
|
||||
- run: tox -e ${{ matrix.test }}
|
||||
|
||||
build:
|
||||
needs: ["lint", "tests-unit", "tests-integration"]
|
||||
name: "build"
|
||||
name: "build / binary"
|
||||
strategy:
|
||||
matrix:
|
||||
os:
|
||||
@@ -129,10 +102,14 @@ jobs:
|
||||
- windows-latest
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- id: os-name
|
||||
uses: ASzc/change-string-case-action@v1
|
||||
with:
|
||||
string: ${{ runner.os }}
|
||||
- name: set pip cache dir
|
||||
id: pip-cache
|
||||
run: echo "::set-output name=dir::$(pip cache dir)"
|
||||
@@ -142,56 +119,47 @@ jobs:
|
||||
path: ${{ steps.pip-cache.outputs.dir }}
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
|
||||
restore-keys: ${{ runner.os }}-pip-
|
||||
- name: Setup
|
||||
run: |
|
||||
pip install --user --upgrade pip wheel
|
||||
pip install pyinstaller@git+https://github.com/eukreign/pyinstaller.git@sqlalchemy
|
||||
- if: startsWith(runner.os, 'linux')
|
||||
run: |
|
||||
sudo apt-get install libzmq3-dev
|
||||
pip install -e .[postgres]
|
||||
- if: startsWith(runner.os, 'mac')
|
||||
run: |
|
||||
brew install zeromq
|
||||
pip install -e .
|
||||
- run: pip install pyinstaller==4.4
|
||||
- run: pip install -e .
|
||||
- if: startsWith(github.ref, 'refs/tags/v')
|
||||
run: python docker/set_build.py
|
||||
- if: startsWith(runner.os, 'linux') || startsWith(runner.os, 'mac')
|
||||
name: Build & Run (Unix)
|
||||
run: |
|
||||
pyinstaller --onefile --name lbrynet lbry/cli.py
|
||||
chmod +x dist/lbrynet
|
||||
pyinstaller --onefile --name lbrynet lbry/extras/cli.py
|
||||
dist/lbrynet --version
|
||||
- if: startsWith(runner.os, 'windows')
|
||||
name: Build & Run (Windows)
|
||||
run: |
|
||||
pip install pywin32
|
||||
pip install -e .
|
||||
pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico --onefile --name lbrynet lbry/cli.py
|
||||
pip install pywin32==301
|
||||
pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico --onefile --name lbrynet lbry/extras/cli.py
|
||||
dist/lbrynet.exe --version
|
||||
- uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: lbrynet-${{ matrix.os }}
|
||||
name: lbrynet-${{ steps.os-name.outputs.lowercase }}
|
||||
path: dist/
|
||||
|
||||
docker:
|
||||
release:
|
||||
name: "release"
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
needs: ["build"]
|
||||
name: "build (docker)"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- name: fetch lbrynet binary
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: lbrynet-ubuntu-16.04
|
||||
- run: |
|
||||
chmod +x lbrynet
|
||||
mv lbrynet docker
|
||||
- name: build and push docker image
|
||||
uses: docker/build-push-action@v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
repository: lbry/lbrynet
|
||||
path: docker
|
||||
tag_with_ref: true
|
||||
tag_with_sha: true
|
||||
add_git_labels: true
|
||||
- uses: actions/download-artifact@v2
|
||||
- name: upload binaries
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.RELEASE_API_TOKEN }}
|
||||
run: |
|
||||
pip install githubrelease
|
||||
chmod +x lbrynet-macos/lbrynet
|
||||
chmod +x lbrynet-linux/lbrynet
|
||||
zip --junk-paths lbrynet-mac.zip lbrynet-macos/lbrynet
|
||||
zip --junk-paths lbrynet-linux.zip lbrynet-linux/lbrynet
|
||||
zip --junk-paths lbrynet-windows.zip lbrynet-windows/lbrynet.exe
|
||||
ls -lh
|
||||
githubrelease release lbryio/lbry-sdk info ${GITHUB_REF#refs/tags/}
|
||||
githubrelease asset lbryio/lbry-sdk upload ${GITHUB_REF#refs/tags/} \
|
||||
lbrynet-mac.zip lbrynet-linux.zip lbrynet-windows.zip
|
||||
githubrelease release lbryio/lbry-sdk publish ${GITHUB_REF#refs/tags/}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
name: slack
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
|
||||
jobs:
|
||||
release:
|
||||
name: "slack notification"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: LoveToKnow/slackify-markdown-action@v1.0.0
|
||||
id: markdown
|
||||
with:
|
||||
text: "There is a new SDK release: ${{github.event.release.html_url}}\n${{ github.event.release.body }}"
|
||||
- uses: slackapi/slack-github-action@v1.14.0
|
||||
env:
|
||||
CHANGELOG: ${{ toJSON(steps.markdown.outputs.text) }}
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_RELEASE_BOT_WEBHOOK }}
|
||||
with:
|
||||
payload: '{"type": "mrkdwn", "text": ${{ env.CHANGELOG }}}'
|
||||
|
||||
+6
-2
@@ -6,13 +6,17 @@
|
||||
/.coverage*
|
||||
/lbry-venv
|
||||
/venv
|
||||
/lbry/blockchain
|
||||
|
||||
lbry.egg-info
|
||||
__pycache__
|
||||
_trial_temp/
|
||||
trending*.log
|
||||
|
||||
/tests/integration/commands/files
|
||||
/tests/integration/blockchain/files
|
||||
/tests/.coverage.*
|
||||
|
||||
/lbry/blockchain/bin
|
||||
/lbry/wallet/bin
|
||||
|
||||
/.vscode
|
||||
/.gitignore
|
||||
-210
@@ -1,210 +0,0 @@
|
||||
default:
|
||||
image: python:3.7
|
||||
|
||||
|
||||
#cache:
|
||||
# directories:
|
||||
# - $HOME/venv
|
||||
# - $HOME/.cache/pip
|
||||
# - $HOME/Library/Caches/pip
|
||||
# - $HOME/Library/Caches/Homebrew
|
||||
# - $TRAVIS_BUILD_DIR/.tox
|
||||
|
||||
|
||||
stages:
|
||||
- test
|
||||
- build
|
||||
- assets
|
||||
- release
|
||||
|
||||
|
||||
.tagged:
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG =~ /^v[0-9\.]+$/'
|
||||
when: on_success
|
||||
|
||||
|
||||
|
||||
test:lint:
|
||||
stage: test
|
||||
script:
|
||||
- make install tools
|
||||
- make lint
|
||||
|
||||
test:unit:
|
||||
stage: test
|
||||
script:
|
||||
- make install tools
|
||||
- HOME=/tmp coverage run -p --source=lbry -m unittest discover -vv tests.unit
|
||||
|
||||
test:datanetwork-integration:
|
||||
stage: test
|
||||
script:
|
||||
- pip install tox-travis
|
||||
- tox -e datanetwork
|
||||
|
||||
test:blockchain-integration:
|
||||
stage: test
|
||||
script:
|
||||
- pip install tox-travis
|
||||
- tox -e blockchain
|
||||
|
||||
test:other-integration:
|
||||
stage: test
|
||||
script:
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends ffmpeg
|
||||
- pip install tox-travis
|
||||
- tox -e other
|
||||
|
||||
test:json-api:
|
||||
stage: test
|
||||
script:
|
||||
- make install tools
|
||||
- HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
|
||||
|
||||
|
||||
|
||||
.build:
|
||||
stage: build
|
||||
artifacts:
|
||||
expire_in: 1 day
|
||||
paths:
|
||||
- lbrynet-${OS}.zip
|
||||
script:
|
||||
- pip install --upgrade 'setuptools<45.0.0'
|
||||
- pip install pyinstaller
|
||||
- pip install -e .
|
||||
- python3.7 docker/set_build.py # must come after lbry is installed because it imports lbry
|
||||
- pyinstaller --onefile --name lbrynet lbry/extras/cli.py
|
||||
- chmod +x dist/lbrynet
|
||||
- zip --junk-paths ${CI_PROJECT_DIR}/lbrynet-${OS}.zip dist/lbrynet # gitlab expects artifacts to be in $CI_PROJECT_DIR
|
||||
- openssl dgst -sha256 ${CI_PROJECT_DIR}/lbrynet-${OS}.zip | egrep -o [0-9a-f]+$ # get sha256 of asset. works on mac and ubuntu
|
||||
- dist/lbrynet --version
|
||||
|
||||
build:linux:
|
||||
extends: .build
|
||||
image: ubuntu:16.04
|
||||
variables:
|
||||
OS: linux
|
||||
before_script:
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends software-properties-common zip curl build-essential
|
||||
- add-apt-repository -y ppa:deadsnakes/ppa
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends python3.7-dev
|
||||
- python3.7 <(curl -q https://bootstrap.pypa.io/get-pip.py) # make sure we get pip with python3.7
|
||||
|
||||
build:mac:
|
||||
extends: .build
|
||||
tags: [macos] # makes gitlab use the mac runner
|
||||
variables:
|
||||
OS: mac
|
||||
GIT_DEPTH: 5
|
||||
VENV: /tmp/gitlab-lbry-sdk-venv
|
||||
before_script:
|
||||
# - brew upgrade python || true
|
||||
- python3 --version | grep -q '^Python 3\.7\.' # dont upgrade python on every run. just make sure we're on the right Python
|
||||
# - pip3 install --user --upgrade pip virtualenv
|
||||
- pip3 --version | grep -q '\(python 3\.7\)'
|
||||
- virtualenv --python=python3.7 "${VENV}"
|
||||
- source "${VENV}/bin/activate"
|
||||
after_script:
|
||||
- rm -rf "${VENV}"
|
||||
|
||||
build:windows:
|
||||
extends: .build
|
||||
tags: [windows] # makes gitlab use the windows runner
|
||||
variables:
|
||||
OS: windows
|
||||
GIT_DEPTH: 5
|
||||
before_script:
|
||||
- ./docker/install_choco.ps1
|
||||
- choco install -y --x86 python3 7zip checksum
|
||||
- python --version # | findstr /B "Python 3\.7\." # dont upgrade python on every run. just make sure we're on the right Python
|
||||
- pip --version # | findstr /E '\(python 3\.7\)'
|
||||
- pip install virtualenv pywin32
|
||||
- virtualenv venv
|
||||
- venv/Scripts/activate.ps1
|
||||
- pip install pip==19.3.1; $true # $true ignores errors. need this to get the correct coincurve wheel. see commit notes for details.
|
||||
after_script:
|
||||
- rmdir -Recurse venv
|
||||
script:
|
||||
- pip install --upgrade 'setuptools<45.0.0'
|
||||
- pip install pyinstaller==3.5
|
||||
- pip install -e .
|
||||
- python docker/set_build.py # must come after lbry is installed because it imports lbry
|
||||
- pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico -F -n lbrynet lbry/extras/cli.py
|
||||
- 7z a -tzip $env:CI_PROJECT_DIR/lbrynet-${OS}.zip ./dist/lbrynet.exe
|
||||
- checksum --type=sha256 --file=$env:CI_PROJECT_DIR/lbrynet-${OS}.zip
|
||||
- dist/lbrynet.exe --version
|
||||
|
||||
|
||||
|
||||
# s3 = upload asset to s3 (build.lbry.io)
|
||||
.s3:
|
||||
stage: assets
|
||||
variables:
|
||||
GIT_STRATEGY: none
|
||||
script:
|
||||
- "[ -f lbrynet-${OS}.zip ]" # check that asset exists before trying to upload
|
||||
- pip install awscli
|
||||
- S3_PATH="daemon/gitlab-build-${CI_PIPELINE_ID}_commit-${CI_COMMIT_SHA:0:7}$( if [ ! -z ${CI_COMMIT_TAG} ]; then echo _tag-${CI_COMMIT_TAG}; else echo _branch-${CI_COMMIT_REF_NAME}; fi )"
|
||||
- AWS_ACCESS_KEY_ID=${ARTIFACTS_KEY} AWS_SECRET_ACCESS_KEY=${ARTIFACTS_SECRET} AWS_REGION=${ARTIFACTS_REGION}
|
||||
aws s3 cp lbrynet-${OS}.zip s3://${ARTIFACTS_BUCKET}/${S3_PATH}/lbrynet-${OS}.zip
|
||||
|
||||
s3:linux:
|
||||
extends: .s3
|
||||
variables: {OS: linux}
|
||||
needs: ["build:linux"]
|
||||
|
||||
s3:mac:
|
||||
extends: .s3
|
||||
variables: {OS: mac}
|
||||
needs: ["build:mac"]
|
||||
|
||||
s3:windows:
|
||||
extends: .s3
|
||||
variables: {OS: windows}
|
||||
needs: ["build:windows"]
|
||||
|
||||
|
||||
|
||||
# github = upload assets to github when there's a tagged release
|
||||
.github:
|
||||
extends: .tagged
|
||||
stage: assets
|
||||
variables:
|
||||
GIT_STRATEGY: none
|
||||
script:
|
||||
- "[ -f lbrynet-${OS}.zip ]" # check that asset exists before trying to upload. githubrelease won't error if its missing
|
||||
- pip install githubrelease
|
||||
- githubrelease --no-progress --github-token ${GITHUB_CI_USER_ACCESS_TOKEN} asset lbryio/lbry-sdk upload ${CI_COMMIT_TAG} lbrynet-${OS}.zip
|
||||
|
||||
github:linux:
|
||||
extends: .github
|
||||
variables: {OS: linux}
|
||||
needs: ["build:linux"]
|
||||
|
||||
github:mac:
|
||||
extends: .github
|
||||
variables: {OS: mac}
|
||||
needs: ["build:mac"]
|
||||
|
||||
github:windows:
|
||||
extends: .github
|
||||
variables: {OS: windows}
|
||||
needs: ["build:windows"]
|
||||
|
||||
|
||||
|
||||
publish:
|
||||
extends: .tagged
|
||||
stage: release
|
||||
variables:
|
||||
GIT_STRATEGY: none
|
||||
script:
|
||||
- pip install githubrelease
|
||||
- githubrelease --no-progress --github-token ${GITHUB_CI_USER_ACCESS_TOKEN} release lbryio/lbry-sdk publish ${CI_COMMIT_TAG}
|
||||
- >
|
||||
curl -X POST -H 'Content-type: application/json' --data '{"text":"<!channel> There is a new SDK release: https://github.com/lbryio/lbry-sdk/releases/tag/'"${CI_COMMIT_TAG}"'\n'"$(curl -s "https://api.github.com/repos/lbryio/lbry-sdk/releases/tags/${CI_COMMIT_TAG}" | egrep '\w*\"body\":' | cut -d':' -f 2- | tail -c +3 | head -c -2)"'", "channel":"tech"}' "$(echo ${SLACK_WEBHOOK_URL_BASE64} | base64 -d)"
|
||||
+113
-44
@@ -9,20 +9,29 @@ Here's a video walkthrough of this setup, which is itself hosted by the LBRY net
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Running `lbrynet` from source requires Python 3.7 or higher. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/).
|
||||
Running `lbrynet` from source requires Python 3.7. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/).
|
||||
|
||||
After installing python 3, you'll need to install some additional libraries depending on your operating system.
|
||||
After installing Python 3.7, you'll need to install some additional libraries depending on your operating system.
|
||||
|
||||
Because of [issue #2769](https://github.com/lbryio/lbry-sdk/issues/2769)
|
||||
at the moment the `lbrynet` daemon will only work correctly with Python 3.7.
|
||||
If Python 3.8+ is used, the daemon will start but the RPC server
|
||||
may not accept messages, returning the following:
|
||||
```
|
||||
Could not connect to daemon. Are you sure it's running?
|
||||
```
|
||||
|
||||
### macOS
|
||||
|
||||
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
|
||||
|
||||
These environment variables also need to be set:
|
||||
1. PYTHONUNBUFFERED=1
|
||||
2. EVENT_NOKQUEUE=1
|
||||
```
|
||||
PYTHONUNBUFFERED=1
|
||||
EVENT_NOKQUEUE=1
|
||||
```
|
||||
|
||||
Remaining dependencies can then be installed by running:
|
||||
|
||||
```
|
||||
brew install python protobuf
|
||||
```
|
||||
@@ -31,14 +40,17 @@ Assistance installing Python3: https://docs.python-guide.org/starting/install3/o
|
||||
|
||||
### Linux
|
||||
|
||||
On Ubuntu (16.04 minimum, we recommend 18.04), install the following:
|
||||
|
||||
On Ubuntu (we recommend 18.04 or 20.04), install the following:
|
||||
```
|
||||
sudo add-apt-repository ppa:deadsnakes/ppa
|
||||
sudo apt-get update
|
||||
sudo apt-get install build-essential python3.7 python3.7-dev git python3.7-venv libssl-dev python-protobuf
|
||||
```
|
||||
|
||||
The [deadsnakes PPA](https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa) provides Python 3.7
|
||||
for those Ubuntu distributions that no longer have it in their
|
||||
official repositories.
|
||||
|
||||
On Raspbian, you will also need to install `python-pyparsing`.
|
||||
|
||||
If you're running another Linux distro, install the equivalent of the above packages for your system.
|
||||
@@ -47,62 +59,119 @@ If you're running another Linux distro, install the equivalent of the above pack
|
||||
|
||||
### Linux/Mac
|
||||
|
||||
To install on Linux/Mac:
|
||||
Clone the repository:
|
||||
```bash
|
||||
git clone https://github.com/lbryio/lbry-sdk.git
|
||||
cd lbry-sdk
|
||||
```
|
||||
|
||||
```
|
||||
Clone the repository:
|
||||
$ git clone https://github.com/lbryio/lbry-sdk.git
|
||||
$ cd lbry-sdk
|
||||
Create a Python virtual environment for lbry-sdk:
|
||||
```bash
|
||||
python3.7 -m venv lbry-venv
|
||||
```
|
||||
|
||||
Create a Python virtual environment for lbry-sdk:
|
||||
$ python3.7 -m venv lbry-venv
|
||||
|
||||
Activating lbry-sdk virtual environment:
|
||||
$ source lbry-venv/bin/activate
|
||||
|
||||
Make sure you're on Python 3.7+ (as the default Python in virtual environment):
|
||||
$ python --version
|
||||
Activate virtual environment:
|
||||
```bash
|
||||
source lbry-venv/bin/activate
|
||||
```
|
||||
|
||||
Install packages:
|
||||
$ make install
|
||||
Make sure you're on Python 3.7+ as default in the virtual environment:
|
||||
```bash
|
||||
python --version
|
||||
```
|
||||
|
||||
If you are on Linux and using PyCharm, generates initial configs:
|
||||
$ make idea
|
||||
```
|
||||
Install packages:
|
||||
```bash
|
||||
make install
|
||||
```
|
||||
|
||||
To verify your installation, `which lbrynet` should return a path inside of the `lbry-venv` folder created by the `python3.7 -m venv lbry-venv` command.
|
||||
If you are on Linux and using PyCharm, generates initial configs:
|
||||
```bash
|
||||
make idea
|
||||
```
|
||||
|
||||
To verify your installation, `which lbrynet` should return a path inside
|
||||
of the `lbry-venv` folder.
|
||||
```bash
|
||||
(lbry-venv) $ which lbrynet
|
||||
/opt/lbry-sdk/lbry-venv/bin/lbrynet
|
||||
```
|
||||
|
||||
To exit the virtual environment simply use the command `deactivate`.
|
||||
|
||||
### Windows
|
||||
|
||||
To install on Windows:
|
||||
Clone the repository:
|
||||
```bash
|
||||
git clone https://github.com/lbryio/lbry-sdk.git
|
||||
cd lbry-sdk
|
||||
```
|
||||
|
||||
```
|
||||
Clone the repository:
|
||||
> git clone https://github.com/lbryio/lbry-sdk.git
|
||||
> cd lbry-sdk
|
||||
Create a Python virtual environment for lbry-sdk:
|
||||
```bash
|
||||
python -m venv lbry-venv
|
||||
```
|
||||
|
||||
Create a Python virtual environment for lbry-sdk:
|
||||
> python -m venv lbry-venv
|
||||
Activate virtual environment:
|
||||
```bash
|
||||
lbry-venv\Scripts\activate
|
||||
```
|
||||
|
||||
Activating lbry-sdk virtual environment:
|
||||
> lbry-venv\Scripts\activate
|
||||
|
||||
Install packages:
|
||||
> pip install -e .
|
||||
```
|
||||
Install packages:
|
||||
```bash
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
## Run the tests
|
||||
### Elasticsearch
|
||||
|
||||
To run the unit tests from the repo directory:
|
||||
For running integration tests, Elasticsearch is required to be available at localhost:9200/
|
||||
|
||||
```
|
||||
python -m unittest discover tests.unit
|
||||
```
|
||||
The easiest way to start it is using docker with:
|
||||
```bash
|
||||
make elastic-docker
|
||||
```
|
||||
|
||||
Alternative installation methods are available [at Elasticsearch website](https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html).
|
||||
|
||||
To run the unit and integration tests from the repo directory:
|
||||
```
|
||||
python -m unittest discover tests.unit
|
||||
python -m unittest discover tests.integration
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
To start the API server:
|
||||
`lbrynet start`
|
||||
```
|
||||
lbrynet start
|
||||
```
|
||||
|
||||
Whenever the code inside [lbry-sdk/lbry](./lbry)
|
||||
is modified we should run `make install` to recompile the `lbrynet`
|
||||
executable with the newest code.
|
||||
|
||||
## Development
|
||||
|
||||
When developing, remember to enter the environment,
|
||||
and if you wish start the server interactively.
|
||||
```bash
|
||||
$ source lbry-venv/bin/activate
|
||||
|
||||
(lbry-venv) $ python lbry/extras/cli.py start
|
||||
```
|
||||
|
||||
Parameters can be passed in the same way.
|
||||
```bash
|
||||
(lbry-venv) $ python lbry/extras/cli.py wallet balance
|
||||
```
|
||||
|
||||
If a Python debugger (`pdb` or `ipdb`) is installed we can also start it
|
||||
in this way, set up break points, and step through the code.
|
||||
```bash
|
||||
(lbry-venv) $ pip install ipdb
|
||||
|
||||
(lbry-venv) $ ipdb lbry/extras/cli.py
|
||||
```
|
||||
|
||||
Happy hacking!
|
||||
@@ -1,20 +1,30 @@
|
||||
.PHONY: tools lint test idea
|
||||
.PHONY: install tools lint test test-unit test-unit-coverage test-integration idea
|
||||
|
||||
install:
|
||||
pip install -e .
|
||||
|
||||
tools:
|
||||
pip install mypy==0.701 pylint==2.4.4
|
||||
pip install coverage astroid pylint
|
||||
|
||||
lint:
|
||||
pylint --rcfile=setup.cfg lbry
|
||||
#mypy --ignore-missing-imports lbry
|
||||
|
||||
test:
|
||||
test: test-unit test-integration
|
||||
|
||||
test-unit:
|
||||
python -m unittest discover tests.unit
|
||||
|
||||
test-unit-coverage:
|
||||
coverage run -p --source=lbry -m unittest discover -vv tests.unit
|
||||
|
||||
test-integration:
|
||||
tox
|
||||
|
||||
idea:
|
||||
mkdir -p .idea
|
||||
cp -r scripts/idea/* .idea
|
||||
|
||||
start:
|
||||
dropdb lbry --if-exists
|
||||
createdb lbry
|
||||
lbrynet start node \
|
||||
--db-url=postgresql:///lbry --workers=0 --console=advanced --no-spv-address-filters \
|
||||
--lbrycrd-rpc-user=lbry --lbrycrd-rpc-pass=somethingelse \
|
||||
--lbrycrd-dir=${HOME}/.lbrycrd --data-dir=/tmp/tmp-lbrynet
|
||||
elastic-docker:
|
||||
docker run -d -v lbryhub:/usr/share/elasticsearch/data -p 9200:9200 -p 9300:9300 -e"ES_JAVA_OPTS=-Xms512m -Xmx512m" -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.12.1
|
||||
@@ -4,7 +4,7 @@ LBRY is a decentralized peer-to-peer protocol for publishing and accessing digit
|
||||
|
||||
LBRY SDK for Python is currently the most fully featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components include:
|
||||
|
||||
* Built on Python 3.7+ and `asyncio`.
|
||||
* Built on Python 3.7 and `asyncio`.
|
||||
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbry.dht](https://github.com/lbryio/lbry-sdk/tree/master/lbry/dht)).
|
||||
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbry.blob_exchange](https://github.com/lbryio/lbry-sdk/tree/master/lbry/blob_exchange)).
|
||||
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbry.schema](https://github.com/lbryio/lbry-sdk/tree/master/lbry/schema)).
|
||||
@@ -27,10 +27,6 @@ With the daemon running, `lbrynet commands` will show you a list of commands.
|
||||
|
||||
The full API is documented [here](https://lbry.tech/api/sdk).
|
||||
|
||||
## Recommended hardware
|
||||
|
||||
The minimum hardware for a full node is 16cpus, 92gb of RAM, and 160gb of NVMe storage. The recommended hardware is 32cpus, 128gb of RAM, and 160gb of NVMe storage.
|
||||
|
||||
## Running from source
|
||||
|
||||
Installing from source is also relatively painless. Full instructions are in [INSTALL.md](INSTALL.md)
|
||||
@@ -45,7 +41,7 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
|
||||
|
||||
## Security
|
||||
|
||||
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.com/faq/gpg-key) if you need it.
|
||||
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our PGP key is here](https://lbry.com/faq/pgp-key) if you need it.
|
||||
|
||||
## Contact
|
||||
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
# Security Policy
|
||||
|
||||
## Supported Versions
|
||||
|
||||
While we are not at v1.0 yet, only the latest release will be supported.
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
See https://lbry.com/faq/security
|
||||
@@ -1,5 +0,0 @@
|
||||
FROM ubuntu:20.04
|
||||
COPY lbrynet /bin
|
||||
RUN lbrynet --version
|
||||
ENTRYPOINT ["lbrynet"]
|
||||
CMD ["start", "node"]
|
||||
@@ -1,8 +0,0 @@
|
||||
FROM ubuntu:20.04
|
||||
RUN apt-get update && \
|
||||
apt-get install -y wget unzip && \
|
||||
wget -nv https://build.lbry.io/lbrycrd/block_info_fix_try2/lbrycrd-linux.zip && \
|
||||
unzip -d /bin lbrycrd-linux.zip && \
|
||||
rm -rf lbrycrd-linux.zip /var/lib/apt/lists/*
|
||||
RUN lbrycrdd --version
|
||||
ENTRYPOINT ["lbrycrdd"]
|
||||
@@ -0,0 +1,55 @@
|
||||
FROM debian:10-slim
|
||||
|
||||
ARG user=lbry
|
||||
ARG db_dir=/database
|
||||
ARG projects_dir=/home/$user
|
||||
|
||||
ARG DOCKER_TAG
|
||||
ARG DOCKER_COMMIT=docker
|
||||
ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get -y --no-install-recommends install \
|
||||
wget \
|
||||
tar unzip \
|
||||
build-essential \
|
||||
automake libtool \
|
||||
pkg-config \
|
||||
libleveldb-dev \
|
||||
python3.7 \
|
||||
python3-dev \
|
||||
python3-pip \
|
||||
python3-wheel \
|
||||
python3-setuptools && \
|
||||
update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN groupadd -g 999 $user && useradd -m -u 999 -g $user $user
|
||||
RUN mkdir -p $db_dir
|
||||
RUN chown -R $user:$user $db_dir
|
||||
|
||||
COPY . $projects_dir
|
||||
RUN chown -R $user:$user $projects_dir
|
||||
|
||||
USER $user
|
||||
WORKDIR $projects_dir
|
||||
|
||||
RUN pip install uvloop
|
||||
RUN make install
|
||||
RUN python3 docker/set_build.py
|
||||
RUN rm ~/.cache -rf
|
||||
|
||||
# entry point
|
||||
ARG host=0.0.0.0
|
||||
ARG tcp_port=50001
|
||||
ARG daemon_url=http://lbry:lbry@localhost:9245/
|
||||
VOLUME $db_dir
|
||||
ENV TCP_PORT=$tcp_port
|
||||
ENV HOST=$host
|
||||
ENV DAEMON_URL=$daemon_url
|
||||
ENV DB_DIRECTORY=$db_dir
|
||||
ENV MAX_SESSIONS=1000000000
|
||||
ENV MAX_SEND=1000000000000000000
|
||||
ENV EVENT_LOOP_POLICY=uvloop
|
||||
COPY ./docker/wallet_server_entrypoint.sh /entrypoint.sh
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
@@ -0,0 +1,45 @@
|
||||
FROM debian:10-slim
|
||||
|
||||
ARG user=lbry
|
||||
ARG downloads_dir=/database
|
||||
ARG projects_dir=/home/$user
|
||||
|
||||
ARG DOCKER_TAG
|
||||
ARG DOCKER_COMMIT=docker
|
||||
ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get -y --no-install-recommends install \
|
||||
wget \
|
||||
automake libtool \
|
||||
tar unzip \
|
||||
build-essential \
|
||||
pkg-config \
|
||||
libleveldb-dev \
|
||||
python3.7 \
|
||||
python3-dev \
|
||||
python3-pip \
|
||||
python3-wheel \
|
||||
python3-setuptools && \
|
||||
update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN groupadd -g 999 $user && useradd -m -u 999 -g $user $user
|
||||
RUN mkdir -p $downloads_dir
|
||||
RUN chown -R $user:$user $downloads_dir
|
||||
|
||||
COPY . $projects_dir
|
||||
RUN chown -R $user:$user $projects_dir
|
||||
|
||||
USER $user
|
||||
WORKDIR $projects_dir
|
||||
|
||||
RUN pip install uvloop
|
||||
RUN make install
|
||||
RUN python3 docker/set_build.py
|
||||
RUN rm ~/.cache -rf
|
||||
|
||||
# entry point
|
||||
VOLUME $downloads_dir
|
||||
COPY ./docker/webconf.yaml /webconf.yaml
|
||||
ENTRYPOINT ["/home/lbry/.local/bin/lbrynet", "start", "--config=/webconf.yaml"]
|
||||
@@ -0,0 +1,9 @@
|
||||
### How to run with docker-compose
|
||||
1. Edit config file and after that fix permissions with
|
||||
```
|
||||
sudo chown -R 999:999 webconf.yaml
|
||||
```
|
||||
2. Start SDK with
|
||||
```
|
||||
docker-compose up -d
|
||||
```
|
||||
@@ -0,0 +1,45 @@
|
||||
version: "3"
|
||||
|
||||
volumes:
|
||||
wallet_server:
|
||||
es01:
|
||||
|
||||
services:
|
||||
wallet_server:
|
||||
depends_on:
|
||||
- es01
|
||||
image: lbry/wallet-server:${WALLET_SERVER_TAG:-latest-release}
|
||||
restart: always
|
||||
network_mode: host
|
||||
ports:
|
||||
- "50001:50001" # rpc port
|
||||
- "2112:2112" # uncomment to enable prometheus
|
||||
volumes:
|
||||
- "wallet_server:/database"
|
||||
environment:
|
||||
- DAEMON_URL=http://lbry:lbry@127.0.0.1:9245
|
||||
- TCP_PORT=50001
|
||||
- PROMETHEUS_PORT=2112
|
||||
- QUERY_TIMEOUT_MS=3000 # how long search queries allowed to run before cancelling, in milliseconds
|
||||
- TRENDING_ALGORITHMS=variable_decay
|
||||
- MAX_SEND=10000000000000 # deprecated. leave it high until its removed
|
||||
- MAX_SUBS=1000000000000 # deprecated. leave it high until its removed
|
||||
- FILTERING_CHANNEL_IDS=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8
|
||||
- BLOCKING_CHANNEL_IDS=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6 e4e230b131082f6b10c8f7994bbb83f29e8e6fb9
|
||||
es01:
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0
|
||||
container_name: es01
|
||||
environment:
|
||||
- node.name=es01
|
||||
- discovery.type=single-node
|
||||
- indices.query.bool.max_clause_count=4096
|
||||
- bootstrap.memory_lock=true
|
||||
- "ES_JAVA_OPTS=-Xms8g -Xmx8g" # no more than 32, remember to disable swap
|
||||
ulimits:
|
||||
memlock:
|
||||
soft: -1
|
||||
hard: -1
|
||||
volumes:
|
||||
- es01:/usr/share/elasticsearch/data
|
||||
ports:
|
||||
- 127.0.0.1:9200:9200
|
||||
@@ -1,41 +1,9 @@
|
||||
version: "3.8"
|
||||
volumes:
|
||||
lbrycrd-data:
|
||||
version: '3'
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:12
|
||||
environment:
|
||||
POSTGRES_USER: lbry
|
||||
POSTGRES_PASSWORD: lbry
|
||||
lbrycrd:
|
||||
image: lbry/lbrycrd
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.lbrycrd
|
||||
volumes:
|
||||
- lbrycrd-data:/root/.lbrycrd
|
||||
command: >
|
||||
-rpcbind=lbrycrd
|
||||
-rpcallowip=0.0.0.0/0
|
||||
-rpcuser=lbryuser
|
||||
-rpcpassword=lbrypass
|
||||
-zmqpubhashblock=tcp://lbrycrd:29000
|
||||
lbrynet:
|
||||
image: lbry/lbrynet:fast_wallet_server_sync
|
||||
depends_on:
|
||||
- postgres
|
||||
- lbrycrd
|
||||
volumes:
|
||||
- lbrycrd-data:/lbrycrd
|
||||
command: >
|
||||
start
|
||||
--full-node
|
||||
--api=0.0.0.0:5279
|
||||
--db-url=postgresql://lbry:lbry@postgres:5432/lbry
|
||||
--workers=12
|
||||
--console=basic
|
||||
--no-spv-address-filters
|
||||
--lbrycrd-rpc-host=lbrycrd
|
||||
--lbrycrd-rpc-user=lbryuser
|
||||
--lbrycrd-rpc-pass=lbrypass
|
||||
--lbrycrd-dir=/lbrycrd
|
||||
websdk:
|
||||
image: vshyba/websdk
|
||||
ports:
|
||||
- '5279:5279'
|
||||
- '5280:5280'
|
||||
volumes:
|
||||
- ./webconf.yaml:/webconf.yaml
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
cd "$DIR/../.." ## make sure we're in the right place. Docker Hub screws this up sometimes
|
||||
echo "docker build dir: $(pwd)"
|
||||
|
||||
docker build --build-arg DOCKER_TAG=$DOCKER_TAG --build-arg DOCKER_COMMIT=$SOURCE_COMMIT -f $DOCKERFILE_PATH -t $IMAGE_NAME .
|
||||
@@ -0,0 +1,11 @@
|
||||
# requires powershell and .NET 4+. see https://chocolatey.org/install for more info.
|
||||
|
||||
$chocoVersion = powershell choco -v
|
||||
if(-not($chocoVersion)){
|
||||
Write-Output "Chocolatey is not installed, installing now"
|
||||
Write-Output "IF YOU KEEP GETTING THIS MESSAGE ON EVERY BUILD, TRY RESTARTING THE GITLAB RUNNER SO IT GETS CHOCO INTO IT'S ENV"
|
||||
Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
|
||||
}
|
||||
else{
|
||||
Write-Output "Chocolatey version $chocoVersion is already installed"
|
||||
}
|
||||
+5
-3
@@ -20,7 +20,7 @@ def _check_and_set(d: dict, key: str, value: str):
|
||||
def main():
|
||||
build_info = {item: build_info_mod.__dict__[item] for item in dir(build_info_mod) if not item.startswith("__")}
|
||||
|
||||
commit_hash = os.getenv('DOCKER_COMMIT', os.getenv('CI_COMMIT_SHA', os.getenv('TRAVIS_COMMIT')))
|
||||
commit_hash = os.getenv('DOCKER_COMMIT', os.getenv('GITHUB_SHA'))
|
||||
if commit_hash is None:
|
||||
raise ValueError("Commit hash not found in env vars")
|
||||
_check_and_set(build_info, "COMMIT_HASH", commit_hash[:6])
|
||||
@@ -30,8 +30,10 @@ def main():
|
||||
_check_and_set(build_info, "DOCKER_TAG", docker_tag)
|
||||
_check_and_set(build_info, "BUILD", "docker")
|
||||
else:
|
||||
ci_tag = os.getenv('CI_COMMIT_TAG', os.getenv('TRAVIS_TAG'))
|
||||
_check_and_set(build_info, "BUILD", "release" if re.match(r'v\d+\.\d+\.\d+$', str(ci_tag)) else "qa")
|
||||
if re.match(r'refs/tags/v\d+\.\d+\.\d+$', str(os.getenv('GITHUB_REF'))):
|
||||
_check_and_set(build_info, "BUILD", "release")
|
||||
else:
|
||||
_check_and_set(build_info, "BUILD", "qa")
|
||||
|
||||
log.debug("build info: %s", ", ".join([f"{k}={v}" for k, v in build_info.items()]))
|
||||
with open(build_info_mod.__file__, 'w') as f:
|
||||
|
||||
Executable
+25
@@ -0,0 +1,25 @@
|
||||
#!/bin/bash
|
||||
|
||||
# entrypoint for wallet server Docker image
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SNAPSHOT_URL="${SNAPSHOT_URL:-}" #off by default. latest snapshot at https://lbry.com/snapshot/wallet
|
||||
|
||||
if [[ -n "$SNAPSHOT_URL" ]] && [[ ! -f /database/claims.db ]]; then
|
||||
files="$(ls)"
|
||||
echo "Downloading wallet snapshot from $SNAPSHOT_URL"
|
||||
wget --no-verbose --trust-server-names --content-disposition "$SNAPSHOT_URL"
|
||||
echo "Extracting snapshot..."
|
||||
filename="$(grep -vf <(echo "$files") <(ls))" # finds the file that was not there before
|
||||
case "$filename" in
|
||||
*.tgz|*.tar.gz|*.tar.bz2 ) tar xvf "$filename" --directory /database ;;
|
||||
*.zip ) unzip "$filename" -d /database ;;
|
||||
* ) echo "Don't know how to extract ${filename}. SNAPSHOT COULD NOT BE LOADED" && exit 1 ;;
|
||||
esac
|
||||
rm "$filename"
|
||||
fi
|
||||
|
||||
/home/lbry/.local/bin/lbry-hub-elastic-sync /database/claims.db
|
||||
echo 'starting server'
|
||||
/home/lbry/.local/bin/lbry-hub "$@"
|
||||
@@ -0,0 +1,9 @@
|
||||
allowed_origin: "*"
|
||||
max_key_fee: "0.0 USD"
|
||||
save_files: false
|
||||
save_blobs: false
|
||||
streaming_server: "0.0.0.0:5280"
|
||||
api: "0.0.0.0:5279"
|
||||
data_dir: /tmp
|
||||
download_dir: /tmp
|
||||
wallet_dir: /tmp
|
||||
+604
-170
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,7 @@
|
||||
.git
|
||||
.tox
|
||||
__pycache__
|
||||
dist
|
||||
lbry.egg-info
|
||||
docs
|
||||
tests
|
||||
+2
-8
@@ -1,8 +1,2 @@
|
||||
__version__ = "1.0.0"
|
||||
from lbry.wallet import Account, Wallet, WalletManager
|
||||
from lbry.blockchain import Ledger, RegTestLedger, TestNetLedger
|
||||
from lbry.blockchain import Transaction, Output, Input
|
||||
from lbry.blockchain import dewies_to_lbc, lbc_to_dewies, dict_values_to_lbc
|
||||
from lbry.service import API, Daemon, FullNode, LightClient
|
||||
from lbry.db.database import Database
|
||||
from lbry.conf import Config
|
||||
__version__ = "0.101.1"
|
||||
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
|
||||
+24
-10
@@ -110,7 +110,7 @@ class AbstractBlob:
|
||||
if reader in self.readers:
|
||||
self.readers.remove(reader)
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
def _write_blob(self, blob_bytes: bytes) -> asyncio.Task:
|
||||
raise NotImplementedError()
|
||||
|
||||
def set_length(self, length) -> None:
|
||||
@@ -198,11 +198,17 @@ class AbstractBlob:
|
||||
def save_verified_blob(self, verified_bytes: bytes):
|
||||
if self.verified.is_set():
|
||||
return
|
||||
if self.is_writeable():
|
||||
self._write_blob(verified_bytes)
|
||||
|
||||
def update_events(_):
|
||||
self.verified.set()
|
||||
self.writing.clear()
|
||||
|
||||
if self.is_writeable():
|
||||
self.writing.set()
|
||||
task = self._write_blob(verified_bytes)
|
||||
task.add_done_callback(update_events)
|
||||
if self.blob_completed_callback:
|
||||
self.blob_completed_callback(self)
|
||||
task.add_done_callback(lambda _: self.blob_completed_callback(self))
|
||||
|
||||
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
|
||||
peer_port: typing.Optional[int] = None) -> HashBlobWriter:
|
||||
@@ -261,9 +267,11 @@ class BlobBuffer(AbstractBlob):
|
||||
self.verified.clear()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
if self._verified_bytes:
|
||||
raise OSError("already have bytes for blob")
|
||||
self._verified_bytes = BytesIO(blob_bytes)
|
||||
async def write():
|
||||
if self._verified_bytes:
|
||||
raise OSError("already have bytes for blob")
|
||||
self._verified_bytes = BytesIO(blob_bytes)
|
||||
return self.loop.create_task(write())
|
||||
|
||||
def delete(self):
|
||||
if self._verified_bytes:
|
||||
@@ -319,13 +327,19 @@ class BlobFile(AbstractBlob):
|
||||
handle.close()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
with open(self.file_path, 'wb') as f:
|
||||
f.write(blob_bytes)
|
||||
def _write_blob():
|
||||
with open(self.file_path, 'wb') as f:
|
||||
f.write(blob_bytes)
|
||||
|
||||
async def write_blob():
|
||||
await self.loop.run_in_executor(None, _write_blob)
|
||||
|
||||
return self.loop.create_task(write_blob())
|
||||
|
||||
def delete(self):
|
||||
super().delete()
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
return super().delete()
|
||||
|
||||
@classmethod
|
||||
async def create_from_unencrypted(
|
||||
|
||||
@@ -2,7 +2,7 @@ import os
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
from lbry.utils import LRUCache
|
||||
from lbry.utils import LRUCacheWithMetrics
|
||||
from lbry.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
|
||||
from lbry.stream.descriptor import StreamDescriptor
|
||||
from lbry.connection_manager import ConnectionManager
|
||||
@@ -10,11 +10,7 @@ from lbry.connection_manager import ConnectionManager
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.dht.protocol.data_store import DictDataStore
|
||||
|
||||
|
||||
class SQLiteStorage:
|
||||
pass
|
||||
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -36,7 +32,7 @@ class BlobManager:
|
||||
else self._node_data_store.completed_blobs
|
||||
self.blobs: typing.Dict[str, AbstractBlob] = {}
|
||||
self.config = config
|
||||
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCache(
|
||||
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCacheWithMetrics(
|
||||
self.config.blob_lru_cache_size)
|
||||
self.connection_manager = ConnectionManager(loop)
|
||||
|
||||
|
||||
@@ -152,6 +152,8 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
log.debug(msg)
|
||||
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
|
||||
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
|
||||
# wait for the io to finish
|
||||
await self.blob.verified.wait()
|
||||
log.info("%s at %fMB/s", msg,
|
||||
round((float(self._blob_bytes_received) /
|
||||
float(time.perf_counter() - start_time)) / 1000000.0, 2))
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
from .ledger import Ledger, RegTestLedger, TestNetLedger, ledger_class_from_name
|
||||
from .transaction import Transaction, Output, Input
|
||||
from .bcd_data_stream import BCDataStream
|
||||
from .dewies import dewies_to_lbc, lbc_to_dewies, dict_values_to_lbc
|
||||
@@ -1,60 +0,0 @@
|
||||
import struct
|
||||
from typing import NamedTuple, List
|
||||
|
||||
from chiabip158 import PyBIP158 # pylint: disable=no-name-in-module
|
||||
|
||||
from lbry.crypto.hash import double_sha256
|
||||
from lbry.blockchain.transaction import Transaction
|
||||
from lbry.blockchain.bcd_data_stream import BCDataStream
|
||||
|
||||
|
||||
ZERO_BLOCK = bytes((0,)*32)
|
||||
|
||||
|
||||
def create_address_filter(address_hashes: List[bytes]) -> bytes:
|
||||
return bytes(PyBIP158([bytearray(a) for a in address_hashes]).GetEncoded())
|
||||
|
||||
|
||||
def get_address_filter(address_filter: bytes) -> PyBIP158:
|
||||
return PyBIP158(bytearray(address_filter))
|
||||
|
||||
|
||||
class Block(NamedTuple):
|
||||
height: int
|
||||
version: int
|
||||
file_number: int
|
||||
block_hash: bytes
|
||||
prev_block_hash: bytes
|
||||
merkle_root: bytes
|
||||
claim_trie_root: bytes
|
||||
timestamp: int
|
||||
bits: int
|
||||
nonce: int
|
||||
txs: List[Transaction]
|
||||
|
||||
@staticmethod
|
||||
def from_data_stream(stream: BCDataStream, height: int, file_number: int):
|
||||
header = stream.data.read(112)
|
||||
version, = struct.unpack('<I', header[:4])
|
||||
timestamp, bits, nonce = struct.unpack('<III', header[100:112])
|
||||
tx_count = stream.read_compact_size()
|
||||
return Block(
|
||||
height=height,
|
||||
version=version,
|
||||
file_number=file_number,
|
||||
block_hash=double_sha256(header),
|
||||
prev_block_hash=header[4:36],
|
||||
merkle_root=header[36:68],
|
||||
claim_trie_root=header[68:100][::-1],
|
||||
timestamp=timestamp,
|
||||
bits=bits,
|
||||
nonce=nonce,
|
||||
txs=[
|
||||
Transaction(height=height, position=i, timestamp=timestamp).deserialize(stream)
|
||||
for i in range(tx_count)
|
||||
]
|
||||
)
|
||||
|
||||
@property
|
||||
def is_first_block(self):
|
||||
return self.prev_block_hash == ZERO_BLOCK
|
||||
@@ -1,245 +0,0 @@
|
||||
import os.path
|
||||
import asyncio
|
||||
import sqlite3
|
||||
from typing import List, Optional
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from lbry.schema.url import normalize_name
|
||||
|
||||
from .bcd_data_stream import BCDataStream
|
||||
|
||||
|
||||
FILES = [
|
||||
'claims',
|
||||
'block_index',
|
||||
]
|
||||
|
||||
|
||||
def make_short_url(r):
|
||||
try:
|
||||
# TODO: we describe it as normalized but the old SDK didnt do that
|
||||
name = r["name"].decode().replace("\x00", "")
|
||||
return f'{name}#{r["shortestID"] or r["claimID"][::-1].hex()[0]}'
|
||||
except UnicodeDecodeError:
|
||||
# print(f'failed making short url due to name parse error for claim_id: {r["claimID"][::-1].hex()}')
|
||||
return "INVALID NAME"
|
||||
|
||||
|
||||
class FindShortestID:
|
||||
__slots__ = 'short_id', 'new_id'
|
||||
|
||||
def __init__(self):
|
||||
self.short_id = ''
|
||||
self.new_id = None
|
||||
|
||||
def step(self, other_id, new_id):
|
||||
other_id = other_id[::-1].hex()
|
||||
if self.new_id is None:
|
||||
self.new_id = new_id[::-1].hex()
|
||||
for i in range(len(self.new_id)):
|
||||
if other_id[i] != self.new_id[i]:
|
||||
if i > len(self.short_id)-1:
|
||||
self.short_id = self.new_id[:i+1]
|
||||
break
|
||||
|
||||
def finalize(self):
|
||||
return self.short_id
|
||||
|
||||
|
||||
class BlockchainDB:
|
||||
|
||||
def __init__(self, directory: str):
|
||||
self.directory = directory
|
||||
self.connection: Optional[sqlite3.Connection] = None
|
||||
self.executor: Optional[ThreadPoolExecutor] = None
|
||||
|
||||
async def run_in_executor(self, *args):
|
||||
return await asyncio.get_running_loop().run_in_executor(self.executor, *args)
|
||||
|
||||
def sync_open(self):
|
||||
self.connection = sqlite3.connect(
|
||||
os.path.join(self.directory, FILES[0]+'.sqlite'),
|
||||
timeout=60.0 * 5
|
||||
)
|
||||
for file in FILES[1:]:
|
||||
self.connection.execute(
|
||||
f"ATTACH DATABASE '{os.path.join(self.directory, file+'.sqlite')}' AS {file}"
|
||||
)
|
||||
self.connection.create_aggregate("find_shortest_id", 2, FindShortestID)
|
||||
self.connection.execute("CREATE INDEX IF NOT EXISTS claim_originalheight ON claim (originalheight);")
|
||||
self.connection.execute("CREATE INDEX IF NOT EXISTS claim_updateheight ON claim (updateheight);")
|
||||
self.connection.execute("create index IF NOT EXISTS support_blockheight on support (blockheight);")
|
||||
self.connection.row_factory = sqlite3.Row
|
||||
|
||||
async def open(self):
|
||||
assert self.executor is None, "Database is already open."
|
||||
self.executor = ThreadPoolExecutor(max_workers=1)
|
||||
return await self.run_in_executor(self.sync_open)
|
||||
|
||||
def sync_close(self):
|
||||
self.connection.close()
|
||||
self.connection = None
|
||||
|
||||
async def close(self):
|
||||
if self.executor is not None:
|
||||
if self.connection is not None:
|
||||
await self.run_in_executor(self.sync_close)
|
||||
self.executor.shutdown()
|
||||
self.executor = None
|
||||
|
||||
async def commit(self):
|
||||
await self.run_in_executor(self.connection.commit)
|
||||
|
||||
def sync_execute(self, sql: str, *args):
|
||||
return self.connection.execute(sql, *args)
|
||||
|
||||
async def execute(self, sql: str, *args):
|
||||
return await self.run_in_executor(self.sync_execute, sql, *args)
|
||||
|
||||
def sync_execute_fetchall(self, sql: str, *args) -> List[dict]:
|
||||
return self.connection.execute(sql, *args).fetchall()
|
||||
|
||||
async def execute_fetchall(self, sql: str, *args) -> List[dict]:
|
||||
return await self.run_in_executor(self.sync_execute_fetchall, sql, *args)
|
||||
|
||||
def sync_get_best_height(self) -> int:
|
||||
sql = "SELECT MAX(height) FROM block_info"
|
||||
return self.connection.execute(sql).fetchone()[0]
|
||||
|
||||
async def get_best_height(self) -> int:
|
||||
return await self.run_in_executor(self.sync_get_best_height)
|
||||
|
||||
def sync_get_block_files(self, file_number: int = None, start_height: int = None) -> List[dict]:
|
||||
sql = """
|
||||
SELECT
|
||||
file as file_number,
|
||||
COUNT(hash) as blocks,
|
||||
SUM(txcount) as txs,
|
||||
MAX(height) as best_height,
|
||||
MIN(height) as start_height
|
||||
FROM block_info
|
||||
WHERE status&1 AND status&4 AND NOT status&32 AND NOT status&64
|
||||
"""
|
||||
args = ()
|
||||
if file_number is not None and start_height is not None:
|
||||
sql += "AND file = ? AND height >= ?"
|
||||
args = (file_number, start_height)
|
||||
return [dict(r) for r in self.sync_execute_fetchall(sql + " GROUP BY file ORDER BY file ASC;", args)]
|
||||
|
||||
async def get_block_files(self, file_number: int = None, start_height: int = None) -> List[dict]:
|
||||
return await self.run_in_executor(
|
||||
self.sync_get_block_files, file_number, start_height
|
||||
)
|
||||
|
||||
def sync_get_blocks_in_file(self, block_file: int, start_height=0) -> List[dict]:
|
||||
return [dict(r) for r in self.sync_execute_fetchall(
|
||||
"""
|
||||
SELECT datapos as data_offset, height, hash as block_hash, txCount as txs
|
||||
FROM block_info
|
||||
WHERE file = ? AND height >= ?
|
||||
AND status&1 AND status&4 AND NOT status&32 AND NOT status&64
|
||||
ORDER BY datapos ASC;
|
||||
""", (block_file, start_height)
|
||||
)]
|
||||
|
||||
async def get_blocks_in_file(self, block_file: int, start_height=0) -> List[dict]:
|
||||
return await self.run_in_executor(self.sync_get_blocks_in_file, block_file, start_height)
|
||||
|
||||
def sync_get_claim_support_txo_hashes(self, at_height: int) -> set:
|
||||
return {
|
||||
r['txID'] + BCDataStream.uint32.pack(r['txN'])
|
||||
for r in self.connection.execute(
|
||||
"""
|
||||
SELECT txID, txN FROM claim WHERE updateHeight = ?
|
||||
UNION
|
||||
SELECT txID, txN FROM support WHERE blockHeight = ?
|
||||
""", (at_height, at_height)
|
||||
).fetchall()
|
||||
}
|
||||
|
||||
def sync_get_takeover_count(self, start_height: int, end_height: int) -> int:
|
||||
sql = """
|
||||
SELECT COUNT(*) FROM claim WHERE name IN (
|
||||
SELECT name FROM takeover
|
||||
WHERE name IS NOT NULL AND height BETWEEN ? AND ?
|
||||
)
|
||||
""", (start_height, end_height)
|
||||
return self.connection.execute(*sql).fetchone()[0]
|
||||
|
||||
async def get_takeover_count(self, start_height: int, end_height: int) -> int:
|
||||
return await self.run_in_executor(self.sync_get_takeover_count, start_height, end_height)
|
||||
|
||||
def sync_get_takeovers(self, start_height: int, end_height: int) -> List[dict]:
|
||||
sql = """
|
||||
SELECT name, claimID, MAX(height) AS height FROM takeover
|
||||
WHERE name IS NOT NULL AND height BETWEEN ? AND ?
|
||||
GROUP BY name
|
||||
""", (start_height, end_height)
|
||||
return [{
|
||||
'normalized': normalize_name(r['name'].decode()),
|
||||
'claim_hash': r['claimID'],
|
||||
'height': r['height']
|
||||
} for r in self.sync_execute_fetchall(*sql)]
|
||||
|
||||
async def get_takeovers(self, start_height: int, end_height: int) -> List[dict]:
|
||||
return await self.run_in_executor(self.sync_get_takeovers, start_height, end_height)
|
||||
|
||||
def sync_get_claim_metadata_count(self, start_height: int, end_height: int) -> int:
|
||||
sql = "SELECT COUNT(*) FROM claim WHERE originalHeight BETWEEN ? AND ?"
|
||||
return self.connection.execute(sql, (start_height, end_height)).fetchone()[0]
|
||||
|
||||
async def get_claim_metadata_count(self, start_height: int, end_height: int) -> int:
|
||||
return await self.run_in_executor(self.sync_get_claim_metadata_count, start_height, end_height)
|
||||
|
||||
def sync_get_claim_metadata(self, claim_hashes) -> List[dict]:
|
||||
sql = f"""
|
||||
SELECT
|
||||
name, claimID, activationHeight, expirationHeight, originalHeight,
|
||||
(SELECT
|
||||
CASE WHEN takeover.claimID = claim.claimID THEN takeover.height END
|
||||
FROM takeover WHERE takeover.name = claim.nodename
|
||||
ORDER BY height DESC LIMIT 1
|
||||
) AS takeoverHeight,
|
||||
(SELECT find_shortest_id(c.claimid, claim.claimid) FROM claim AS c
|
||||
WHERE
|
||||
c.nodename = claim.nodename AND
|
||||
c.originalheight <= claim.originalheight AND
|
||||
c.claimid != claim.claimid
|
||||
) AS shortestID
|
||||
FROM claim
|
||||
WHERE claimID IN ({','.join(['?' for _ in claim_hashes])})
|
||||
ORDER BY claimID
|
||||
""", claim_hashes
|
||||
return [{
|
||||
"name": r["name"],
|
||||
"claim_hash": r["claimID"],
|
||||
"activation_height": r["activationHeight"],
|
||||
"expiration_height": r["expirationHeight"],
|
||||
"takeover_height": r["takeoverHeight"],
|
||||
"creation_height": r["originalHeight"],
|
||||
"short_url": make_short_url(r),
|
||||
} for r in self.sync_execute_fetchall(*sql)]
|
||||
|
||||
async def get_claim_metadata(self, start_height: int, end_height: int) -> List[dict]:
|
||||
return await self.run_in_executor(self.sync_get_claim_metadata, start_height, end_height)
|
||||
|
||||
def sync_get_support_metadata_count(self, start_height: int, end_height: int) -> int:
|
||||
sql = "SELECT COUNT(*) FROM support WHERE blockHeight BETWEEN ? AND ?"
|
||||
return self.connection.execute(sql, (start_height, end_height)).fetchone()[0]
|
||||
|
||||
async def get_support_metadata_count(self, start_height: int, end_height: int) -> int:
|
||||
return await self.run_in_executor(self.sync_get_support_metadata_count, start_height, end_height)
|
||||
|
||||
def sync_get_support_metadata(self, start_height: int, end_height: int) -> List[dict]:
|
||||
sql = """
|
||||
SELECT name, txid, txn, activationHeight, expirationHeight
|
||||
FROM support WHERE blockHeight BETWEEN ? AND ?
|
||||
""", (start_height, end_height)
|
||||
return [{
|
||||
"name": r['name'],
|
||||
"txo_hash_pk": r['txID'] + BCDataStream.uint32.pack(r['txN']),
|
||||
"activation_height": r['activationHeight'],
|
||||
"expiration_height": r['expirationHeight'],
|
||||
} for r in self.sync_execute_fetchall(*sql)]
|
||||
|
||||
async def get_support_metadata(self, start_height: int, end_height: int) -> List[dict]:
|
||||
return await self.run_in_executor(self.sync_get_support_metadata, start_height, end_height)
|
||||
@@ -1,346 +0,0 @@
|
||||
import os
|
||||
import struct
|
||||
import shutil
|
||||
import asyncio
|
||||
import logging
|
||||
import zipfile
|
||||
import tempfile
|
||||
import urllib.request
|
||||
from typing import Optional
|
||||
from binascii import hexlify
|
||||
|
||||
import aiohttp
|
||||
import zmq
|
||||
import zmq.asyncio
|
||||
|
||||
from lbry.conf import Config
|
||||
from lbry.event import EventController
|
||||
from lbry.error import LbrycrdEventSubscriptionError, LbrycrdUnauthorizedError
|
||||
|
||||
from .database import BlockchainDB
|
||||
from .ledger import Ledger, RegTestLedger
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
DOWNLOAD_URL = (
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.6/lbrycrd-linux-1746.zip'
|
||||
)
|
||||
|
||||
|
||||
class Process(asyncio.SubprocessProtocol):
|
||||
|
||||
IGNORE_OUTPUT = [
|
||||
b'keypool keep',
|
||||
b'keypool reserve',
|
||||
b'keypool return',
|
||||
]
|
||||
|
||||
def __init__(self):
|
||||
self.ready = asyncio.Event()
|
||||
self.stopped = asyncio.Event()
|
||||
|
||||
def pipe_data_received(self, fd, data):
|
||||
if not any(ignore in data for ignore in self.IGNORE_OUTPUT):
|
||||
if b'Error:' in data:
|
||||
log.error(data.decode())
|
||||
else:
|
||||
for line in data.decode().splitlines():
|
||||
log.debug(line.rstrip())
|
||||
if b'Error:' in data:
|
||||
self.ready.set()
|
||||
raise SystemError(data.decode())
|
||||
if b'Done loading' in data:
|
||||
self.ready.set()
|
||||
|
||||
def process_exited(self):
|
||||
self.stopped.set()
|
||||
self.ready.set()
|
||||
|
||||
|
||||
ZMQ_BLOCK_EVENT = 'pubhashblock'
|
||||
|
||||
|
||||
class Lbrycrd:
|
||||
|
||||
def __init__(self, ledger: Ledger):
|
||||
self.ledger, self.conf = ledger, ledger.conf
|
||||
self.data_dir = self.actual_data_dir = ledger.conf.lbrycrd_dir
|
||||
if self.is_regtest:
|
||||
self.actual_data_dir = os.path.join(self.data_dir, 'regtest')
|
||||
self.blocks_dir = os.path.join(self.actual_data_dir, 'blocks')
|
||||
self.bin_dir = os.path.join(os.path.dirname(__file__), 'bin')
|
||||
self.daemon_bin = os.path.join(self.bin_dir, 'lbrycrdd')
|
||||
self.cli_bin = os.path.join(self.bin_dir, 'lbrycrd-cli')
|
||||
self.protocol = None
|
||||
self.transport = None
|
||||
self.subscribed = False
|
||||
self.subscription: Optional[asyncio.Task] = None
|
||||
self.default_generate_address = None
|
||||
self._on_block_hash_controller = EventController()
|
||||
self.on_block_hash = self._on_block_hash_controller.stream
|
||||
self.on_block_hash.listen(lambda e: log.info('%s %s', hexlify(e['hash']), e['msg']))
|
||||
self._on_tx_hash_controller = EventController()
|
||||
self.on_tx_hash = self._on_tx_hash_controller.stream
|
||||
|
||||
self.db = BlockchainDB(self.actual_data_dir)
|
||||
self._session: Optional[aiohttp.ClientSession] = None
|
||||
self._loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
|
||||
@property
|
||||
def session(self) -> aiohttp.ClientSession:
|
||||
if self._session is None:
|
||||
self._session = aiohttp.ClientSession()
|
||||
return self._session
|
||||
|
||||
@classmethod
|
||||
def temp_regtest(cls):
|
||||
return cls(RegTestLedger(
|
||||
Config.with_same_dir(tempfile.mkdtemp()).set(
|
||||
blockchain="regtest",
|
||||
lbrycrd_rpc_port=9245 + 2, # avoid conflict with default rpc port
|
||||
lbrycrd_peer_port=9246 + 2, # avoid conflict with default peer port
|
||||
lbrycrd_zmq="tcp://127.0.0.1:29002"
|
||||
)
|
||||
))
|
||||
|
||||
@staticmethod
|
||||
def get_block_file_name(block_file_number):
|
||||
return f'blk{block_file_number:05}.dat'
|
||||
|
||||
def get_block_file_path(self, block_file_number):
|
||||
return os.path.join(
|
||||
self.actual_data_dir, 'blocks',
|
||||
self.get_block_file_name(block_file_number)
|
||||
)
|
||||
|
||||
@property
|
||||
def is_regtest(self):
|
||||
return isinstance(self.ledger, RegTestLedger)
|
||||
|
||||
@property
|
||||
def rpc_url(self):
|
||||
return (
|
||||
f'http://{self.conf.lbrycrd_rpc_user}:{self.conf.lbrycrd_rpc_pass}'
|
||||
f'@{self.conf.lbrycrd_rpc_host}:{self.conf.lbrycrd_rpc_port}/'
|
||||
)
|
||||
|
||||
@property
|
||||
def exists(self):
|
||||
return (
|
||||
os.path.exists(self.cli_bin) and
|
||||
os.path.exists(self.daemon_bin)
|
||||
)
|
||||
|
||||
async def download(self):
|
||||
downloaded_file = os.path.join(
|
||||
self.bin_dir, DOWNLOAD_URL[DOWNLOAD_URL.rfind('/')+1:]
|
||||
)
|
||||
|
||||
if not os.path.exists(self.bin_dir):
|
||||
os.mkdir(self.bin_dir)
|
||||
|
||||
if not os.path.exists(downloaded_file):
|
||||
log.info('Downloading: %s', DOWNLOAD_URL)
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(DOWNLOAD_URL) as response:
|
||||
with open(downloaded_file, 'wb') as out_file:
|
||||
while True:
|
||||
chunk = await response.content.read(4096)
|
||||
if not chunk:
|
||||
break
|
||||
out_file.write(chunk)
|
||||
with urllib.request.urlopen(DOWNLOAD_URL) as response:
|
||||
with open(downloaded_file, 'wb') as out_file:
|
||||
shutil.copyfileobj(response, out_file)
|
||||
|
||||
log.info('Extracting: %s', downloaded_file)
|
||||
|
||||
with zipfile.ZipFile(downloaded_file) as dotzip:
|
||||
dotzip.extractall(self.bin_dir)
|
||||
# zipfile bug https://bugs.python.org/issue15795
|
||||
os.chmod(self.cli_bin, 0o755)
|
||||
os.chmod(self.daemon_bin, 0o755)
|
||||
|
||||
return self.exists
|
||||
|
||||
async def ensure(self):
|
||||
return self.exists or await self.download()
|
||||
|
||||
def get_start_command(self, *args):
|
||||
if self.is_regtest:
|
||||
args += ('-regtest',)
|
||||
if self.conf.lbrycrd_zmq:
|
||||
args += (
|
||||
f'-zmqpubhashblock={self.conf.lbrycrd_zmq}',
|
||||
f'-zmqpubhashtx={self.conf.lbrycrd_zmq}',
|
||||
)
|
||||
return (
|
||||
self.daemon_bin,
|
||||
f'-datadir={self.data_dir}',
|
||||
f'-port={self.conf.lbrycrd_peer_port}',
|
||||
f'-rpcport={self.conf.lbrycrd_rpc_port}',
|
||||
f'-rpcuser={self.conf.lbrycrd_rpc_user}',
|
||||
f'-rpcpassword={self.conf.lbrycrd_rpc_pass}',
|
||||
'-server', '-printtoconsole',
|
||||
*args
|
||||
)
|
||||
|
||||
async def open(self):
|
||||
await self.db.open()
|
||||
|
||||
async def close(self):
|
||||
await self.db.close()
|
||||
await self.close_session()
|
||||
|
||||
async def close_session(self):
|
||||
if self._session is not None:
|
||||
await self._session.close()
|
||||
self._session = None
|
||||
|
||||
async def start(self, *args):
|
||||
loop = asyncio.get_running_loop()
|
||||
command = self.get_start_command(*args)
|
||||
log.info(' '.join(command))
|
||||
self.transport, self.protocol = await loop.subprocess_exec(Process, *command)
|
||||
await self.protocol.ready.wait()
|
||||
assert not self.protocol.stopped.is_set()
|
||||
await self.open()
|
||||
|
||||
async def stop(self, cleanup=True):
|
||||
try:
|
||||
await self.close()
|
||||
self.transport.terminate()
|
||||
await self.protocol.stopped.wait()
|
||||
assert self.transport.get_returncode() == 0, "lbrycrd daemon exit with error"
|
||||
self.transport.close()
|
||||
finally:
|
||||
if cleanup:
|
||||
await self.cleanup()
|
||||
|
||||
async def cleanup(self):
|
||||
await asyncio.get_running_loop().run_in_executor(
|
||||
None, shutil.rmtree, self.data_dir, True
|
||||
)
|
||||
|
||||
async def ensure_subscribable(self):
|
||||
zmq_notifications = await self.get_zmq_notifications()
|
||||
subs = {e['type']: e['address'] for e in zmq_notifications}
|
||||
if ZMQ_BLOCK_EVENT not in subs:
|
||||
raise LbrycrdEventSubscriptionError(ZMQ_BLOCK_EVENT)
|
||||
if not self.conf.lbrycrd_zmq:
|
||||
self.conf.lbrycrd_zmq = subs[ZMQ_BLOCK_EVENT]
|
||||
|
||||
async def subscribe(self):
|
||||
if not self.subscribed:
|
||||
self.subscribed = True
|
||||
ctx = zmq.asyncio.Context.instance()
|
||||
sock = ctx.socket(zmq.SUB) # pylint: disable=no-member
|
||||
sock.connect(self.conf.lbrycrd_zmq)
|
||||
sock.subscribe("hashblock")
|
||||
sock.subscribe("hashtx")
|
||||
self.subscription = asyncio.create_task(self.subscription_handler(sock))
|
||||
|
||||
async def subscription_handler(self, sock):
|
||||
try:
|
||||
while self.subscribed:
|
||||
msg = await sock.recv_multipart()
|
||||
if msg[0] == b'hashtx':
|
||||
await self._on_tx_hash_controller.add({
|
||||
'hash': msg[1],
|
||||
'msg': struct.unpack('<I', msg[2])[0]
|
||||
})
|
||||
elif msg[0] == b'hashblock':
|
||||
await self._on_block_hash_controller.add({
|
||||
'hash': msg[1],
|
||||
'msg': struct.unpack('<I', msg[2])[0]
|
||||
})
|
||||
except asyncio.CancelledError:
|
||||
sock.close()
|
||||
raise
|
||||
|
||||
def unsubscribe(self):
|
||||
if self.subscribed:
|
||||
self.subscribed = False
|
||||
self.subscription.cancel()
|
||||
self.subscription = None
|
||||
|
||||
def sync_run(self, coro):
|
||||
if self._loop is None:
|
||||
try:
|
||||
self._loop = asyncio.get_event_loop()
|
||||
except RuntimeError:
|
||||
self._loop = asyncio.new_event_loop()
|
||||
return self._loop.run_until_complete(coro)
|
||||
|
||||
async def rpc(self, method, params=None):
|
||||
if self._session is not None and self._session.closed:
|
||||
raise Exception("session is closed! RPC attempted during shutting down.")
|
||||
message = {
|
||||
"jsonrpc": "1.0",
|
||||
"id": "1",
|
||||
"method": method,
|
||||
"params": params or []
|
||||
}
|
||||
async with self.session.post(self.rpc_url, json=message) as resp:
|
||||
if resp.status == 401:
|
||||
raise LbrycrdUnauthorizedError()
|
||||
try:
|
||||
result = await resp.json()
|
||||
except aiohttp.ContentTypeError as e:
|
||||
raise Exception(await resp.text()) from e
|
||||
if not result['error']:
|
||||
return result['result']
|
||||
else:
|
||||
result['error'].update(method=method, params=params)
|
||||
raise Exception(result['error'])
|
||||
|
||||
async def get_zmq_notifications(self):
|
||||
return await self.rpc("getzmqnotifications")
|
||||
|
||||
async def generate(self, blocks):
|
||||
if self.default_generate_address is None:
|
||||
self.default_generate_address = await self.get_new_address()
|
||||
return await self.generate_to_address(blocks, self.default_generate_address)
|
||||
|
||||
async def get_new_address(self):
|
||||
return await self.rpc("getnewaddress")
|
||||
|
||||
async def generate_to_address(self, blocks, address):
|
||||
return await self.rpc("generatetoaddress", [blocks, address])
|
||||
|
||||
async def send_to_address(self, address, amount):
|
||||
return await self.rpc("sendtoaddress", [address, amount])
|
||||
|
||||
async def get_block(self, block_hash):
|
||||
return await self.rpc("getblock", [block_hash])
|
||||
|
||||
async def get_raw_mempool(self):
|
||||
return await self.rpc("getrawmempool")
|
||||
|
||||
async def get_raw_transaction(self, txid):
|
||||
return await self.rpc("getrawtransaction", [txid])
|
||||
|
||||
async def fund_raw_transaction(self, tx):
|
||||
return await self.rpc("fundrawtransaction", [tx])
|
||||
|
||||
async def sign_raw_transaction_with_wallet(self, tx):
|
||||
return await self.rpc("signrawtransactionwithwallet", [tx])
|
||||
|
||||
async def send_raw_transaction(self, tx):
|
||||
return await self.rpc("sendrawtransaction", [tx])
|
||||
|
||||
async def claim_name(self, name, data, amount):
|
||||
return await self.rpc("claimname", [name, data, amount])
|
||||
|
||||
async def update_claim(self, txid, data, amount):
|
||||
return await self.rpc("updateclaim", [txid, data, amount])
|
||||
|
||||
async def abandon_claim(self, txid, address):
|
||||
return await self.rpc("abandonclaim", [txid, address])
|
||||
|
||||
async def support_claim(self, name, claim_id, amount, value="", istip=False):
|
||||
return await self.rpc("supportclaim", [name, claim_id, amount, value, istip])
|
||||
|
||||
async def abandon_support(self, txid, address):
|
||||
return await self.rpc("abandonsupport", [txid, address])
|
||||
@@ -1,179 +0,0 @@
|
||||
from binascii import unhexlify
|
||||
from string import hexdigits
|
||||
from typing import TYPE_CHECKING, Type
|
||||
|
||||
from lbry.crypto.hash import hash160, double_sha256
|
||||
from lbry.crypto.base58 import Base58
|
||||
from lbry.schema.url import URL
|
||||
from .header import Headers, UnvalidatedHeaders
|
||||
from .checkpoints import HASHES
|
||||
from .dewies import lbc_to_dewies
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
|
||||
|
||||
class Ledger:
|
||||
name = 'LBRY Credits'
|
||||
symbol = 'LBC'
|
||||
network_name = 'mainnet'
|
||||
|
||||
headers_class = Headers
|
||||
|
||||
secret_prefix = bytes((0x1c,))
|
||||
pubkey_address_prefix = bytes((0x55,))
|
||||
script_address_prefix = bytes((0x7a,))
|
||||
extended_public_key_prefix = unhexlify('0488b21e')
|
||||
extended_private_key_prefix = unhexlify('0488ade4')
|
||||
|
||||
max_target = 0x0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
|
||||
genesis_hash = '9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463'
|
||||
genesis_bits = 0x1f00ffff
|
||||
target_timespan = 150
|
||||
|
||||
fee_per_byte = 50
|
||||
fee_per_name_char = 200000
|
||||
|
||||
checkpoints = HASHES
|
||||
|
||||
def __init__(self, conf: 'Config'):
|
||||
self.conf = conf
|
||||
self.coin_selection_strategy = None
|
||||
|
||||
@classmethod
|
||||
def get_id(cls):
|
||||
return '{}_{}'.format(cls.symbol.lower(), cls.network_name.lower())
|
||||
|
||||
@staticmethod
|
||||
def address_to_hash160(address) -> bytes:
|
||||
return Base58.decode(address)[1:21]
|
||||
|
||||
@classmethod
|
||||
def pubkey_hash_to_address(cls, h160):
|
||||
raw_address = cls.pubkey_address_prefix + h160
|
||||
return Base58.encode(bytearray(raw_address + double_sha256(raw_address)[0:4]))
|
||||
|
||||
@classmethod
|
||||
def public_key_to_address(cls, public_key):
|
||||
return cls.pubkey_hash_to_address(hash160(public_key))
|
||||
|
||||
@classmethod
|
||||
def script_hash_to_address(cls, h160):
|
||||
raw_address = cls.script_address_prefix + h160
|
||||
return Base58.encode(bytearray(raw_address + double_sha256(raw_address)[0:4]))
|
||||
|
||||
@staticmethod
|
||||
def private_key_to_wif(private_key):
|
||||
return b'\x1c' + private_key + b'\x01'
|
||||
|
||||
@classmethod
|
||||
def is_valid_address(cls, address):
|
||||
decoded = Base58.decode_check(address)
|
||||
return decoded[0] == cls.pubkey_address_prefix[0]
|
||||
|
||||
@classmethod
|
||||
def valid_address_or_error(cls, address):
|
||||
try:
|
||||
assert cls.is_valid_address(address)
|
||||
except:
|
||||
raise Exception(f"'{address}' is not a valid address")
|
||||
|
||||
@staticmethod
|
||||
def valid_claim_id(claim_id: str):
|
||||
if not len(claim_id) == 40:
|
||||
raise Exception(f"Incorrect claimid length: {len(claim_id)}")
|
||||
if set(claim_id).difference(hexdigits):
|
||||
raise Exception("Claim id is not hex encoded")
|
||||
|
||||
@staticmethod
|
||||
def valid_channel_name_or_error(name: str):
|
||||
try:
|
||||
if not name:
|
||||
raise Exception("Channel name cannot be blank.")
|
||||
parsed = URL.parse(name)
|
||||
if not parsed.has_channel:
|
||||
raise Exception("Channel names must start with '@' symbol.")
|
||||
if parsed.channel.name != name:
|
||||
raise Exception("Channel name has invalid character")
|
||||
except (TypeError, ValueError):
|
||||
raise Exception("Invalid channel name.")
|
||||
|
||||
@staticmethod
|
||||
def valid_stream_name_or_error(name: str):
|
||||
try:
|
||||
if not name:
|
||||
raise Exception('Stream name cannot be blank.')
|
||||
parsed = URL.parse(name)
|
||||
if parsed.has_channel:
|
||||
raise Exception(
|
||||
"Stream names cannot start with '@' symbol. This is reserved for channels claims."
|
||||
)
|
||||
if not parsed.has_stream or parsed.stream.name != name:
|
||||
raise Exception('Stream name has invalid characters.')
|
||||
except (TypeError, ValueError):
|
||||
raise Exception("Invalid stream name.")
|
||||
|
||||
@staticmethod
|
||||
def valid_collection_name_or_error(name: str):
|
||||
try:
|
||||
if not name:
|
||||
raise Exception('Collection name cannot be blank.')
|
||||
parsed = URL.parse(name)
|
||||
if parsed.has_channel:
|
||||
raise Exception(
|
||||
"Collection names cannot start with '@' symbol. This is reserved for channels claims."
|
||||
)
|
||||
if not parsed.has_stream or parsed.stream.name != name:
|
||||
raise Exception('Collection name has invalid characters.')
|
||||
except (TypeError, ValueError):
|
||||
raise Exception("Invalid collection name.")
|
||||
|
||||
@staticmethod
|
||||
def get_dewies_or_error(argument: str, lbc: str, positive_value=False):
|
||||
try:
|
||||
dewies = lbc_to_dewies(lbc)
|
||||
if positive_value and dewies <= 0:
|
||||
raise ValueError(f"'{argument}' value must be greater than 0.0")
|
||||
return dewies
|
||||
except ValueError as e:
|
||||
raise ValueError(f"Invalid value for '{argument}': {e.args[0]}")
|
||||
|
||||
def get_fee_address(self, kwargs: dict, claim_address: str) -> str:
|
||||
if 'fee_address' in kwargs:
|
||||
self.valid_address_or_error(kwargs['fee_address'])
|
||||
return kwargs['fee_address']
|
||||
if 'fee_currency' in kwargs or 'fee_amount' in kwargs:
|
||||
return claim_address
|
||||
|
||||
|
||||
class TestNetLedger(Ledger):
|
||||
network_name = 'testnet'
|
||||
pubkey_address_prefix = bytes((111,))
|
||||
script_address_prefix = bytes((196,))
|
||||
extended_public_key_prefix = unhexlify('043587cf')
|
||||
extended_private_key_prefix = unhexlify('04358394')
|
||||
checkpoints = {}
|
||||
|
||||
|
||||
class RegTestLedger(Ledger):
|
||||
network_name = 'regtest'
|
||||
headers_class = UnvalidatedHeaders
|
||||
pubkey_address_prefix = bytes((111,))
|
||||
script_address_prefix = bytes((196,))
|
||||
extended_public_key_prefix = unhexlify('043587cf')
|
||||
extended_private_key_prefix = unhexlify('04358394')
|
||||
|
||||
max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
|
||||
genesis_hash = '6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556'
|
||||
genesis_bits = 0x207fffff
|
||||
target_timespan = 1
|
||||
checkpoints = {}
|
||||
|
||||
|
||||
def ledger_class_from_name(name) -> Type[Ledger]:
|
||||
return {
|
||||
Ledger.network_name: Ledger,
|
||||
TestNetLedger.network_name: TestNetLedger,
|
||||
RegTestLedger.network_name: RegTestLedger
|
||||
}[name]
|
||||
@@ -1 +0,0 @@
|
||||
from .synchronizer import BlockchainSync
|
||||
@@ -1,340 +0,0 @@
|
||||
import logging
|
||||
from binascii import hexlify, unhexlify
|
||||
from typing import Tuple, List
|
||||
|
||||
from sqlalchemy import table, text, func, union, between
|
||||
from sqlalchemy.future import select
|
||||
from sqlalchemy.schema import CreateTable
|
||||
|
||||
from lbry.db.tables import (
|
||||
Block as BlockTable, BlockFilter,
|
||||
TX, TXFilter, TXO, TXI, Claim, Tag, Support
|
||||
)
|
||||
from lbry.db.tables import (
|
||||
pg_add_block_constraints_and_indexes,
|
||||
pg_add_block_filter_constraints_and_indexes,
|
||||
pg_add_tx_constraints_and_indexes,
|
||||
pg_add_tx_filter_constraints_and_indexes,
|
||||
pg_add_txo_constraints_and_indexes,
|
||||
pg_add_txi_constraints_and_indexes,
|
||||
)
|
||||
from lbry.db.query_context import ProgressContext, event_emitter, context
|
||||
from lbry.db.sync import set_input_addresses, update_spent_outputs
|
||||
from lbry.blockchain.transaction import Transaction
|
||||
from lbry.blockchain.block import Block, create_address_filter
|
||||
from lbry.blockchain.bcd_data_stream import BCDataStream
|
||||
|
||||
from .context import get_or_initialize_lbrycrd
|
||||
from .filter_builder import FilterBuilder
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_best_block_height_for_file(file_number):
|
||||
return context().fetchone(
|
||||
select(func.coalesce(func.max(BlockTable.c.height), -1).label('height'))
|
||||
.where(BlockTable.c.file_number == file_number)
|
||||
)['height']
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.blocks.file", "blocks", "txs", throttle=100)
|
||||
def sync_block_file(
|
||||
file_number: int, start_height: int, txs: int, flush_size: int, p: ProgressContext
|
||||
):
|
||||
chain = get_or_initialize_lbrycrd(p.ctx)
|
||||
new_blocks = chain.db.sync_get_blocks_in_file(file_number, start_height)
|
||||
if not new_blocks:
|
||||
return -1
|
||||
file_name = chain.get_block_file_name(file_number)
|
||||
p.start(len(new_blocks), txs, progress_id=file_number, label=file_name)
|
||||
block_file_path = chain.get_block_file_path(file_number)
|
||||
done_blocks = done_txs = 0
|
||||
last_block_processed, loader = -1, p.ctx.get_bulk_loader()
|
||||
with open(block_file_path, "rb") as fp:
|
||||
stream = BCDataStream(fp=fp)
|
||||
for done_blocks, block_info in enumerate(new_blocks, start=1):
|
||||
block_height = block_info["height"]
|
||||
fp.seek(block_info["data_offset"])
|
||||
block = Block.from_data_stream(stream, block_height, file_number)
|
||||
loader.add_block(block)
|
||||
if len(loader.txs) >= flush_size:
|
||||
done_txs += loader.flush(TX)
|
||||
p.step(done_blocks, done_txs)
|
||||
last_block_processed = block_height
|
||||
if p.ctx.stop_event.is_set():
|
||||
return last_block_processed
|
||||
if loader.txs:
|
||||
done_txs += loader.flush(TX)
|
||||
p.step(done_blocks, done_txs)
|
||||
return last_block_processed
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.blocks.indexes", "steps")
|
||||
def blocks_constraints_and_indexes(p: ProgressContext):
|
||||
p.start(1 + len(pg_add_block_constraints_and_indexes))
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM ANALYZE block;"))
|
||||
p.step()
|
||||
for constraint in pg_add_block_constraints_and_indexes:
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute(text(constraint))
|
||||
p.step()
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.blocks.vacuum", "steps")
|
||||
def blocks_vacuum(p: ProgressContext):
|
||||
p.start(1)
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM block;"))
|
||||
p.step()
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.spends.main", "steps")
|
||||
def sync_spends(initial_sync: bool, p: ProgressContext):
|
||||
if initial_sync:
|
||||
p.start(
|
||||
7 +
|
||||
len(pg_add_tx_constraints_and_indexes) +
|
||||
len(pg_add_txi_constraints_and_indexes) +
|
||||
len(pg_add_txo_constraints_and_indexes)
|
||||
)
|
||||
# 1. tx table stuff
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM ANALYZE tx;"))
|
||||
p.step()
|
||||
for constraint in pg_add_tx_constraints_and_indexes:
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute(text(constraint))
|
||||
p.step()
|
||||
# A. Update TXIs to have the address of TXO they are spending.
|
||||
# 2. txi table reshuffling
|
||||
p.ctx.execute(text("ALTER TABLE txi RENAME TO old_txi;"))
|
||||
p.ctx.execute(CreateTable(TXI, include_foreign_key_constraints=[]))
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute(text("ALTER TABLE txi DROP CONSTRAINT txi_pkey;"))
|
||||
p.step()
|
||||
# 3. insert
|
||||
old_txi = table("old_txi", *(c.copy() for c in TXI.columns)) # pylint: disable=not-an-iterable
|
||||
columns = [c for c in old_txi.columns if c.name != "address"] + [TXO.c.address]
|
||||
join_txi_on_txo = old_txi.join(TXO, old_txi.c.txo_hash == TXO.c.txo_hash)
|
||||
select_txis = select(*columns).select_from(join_txi_on_txo)
|
||||
insert_txis = TXI.insert().from_select(columns, select_txis)
|
||||
p.ctx.execute(insert_txis)
|
||||
p.step()
|
||||
# 4. drop old txi and vacuum
|
||||
p.ctx.execute(text("DROP TABLE old_txi;"))
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM ANALYZE txi;"))
|
||||
p.step()
|
||||
for constraint in pg_add_txi_constraints_and_indexes:
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute(text(constraint))
|
||||
p.step()
|
||||
# B. Update TXOs to have the height at which they were spent (if they were).
|
||||
# 5. txo table reshuffling
|
||||
p.ctx.execute(text("ALTER TABLE txo RENAME TO old_txo;"))
|
||||
p.ctx.execute(CreateTable(TXO, include_foreign_key_constraints=[]))
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute(text("ALTER TABLE txo DROP CONSTRAINT txo_pkey;"))
|
||||
p.step()
|
||||
# 6. insert
|
||||
old_txo = table("old_txo", *(c.copy() for c in TXO.columns)) # pylint: disable=not-an-iterable
|
||||
columns = [c for c in old_txo.columns if c.name != "spent_height"]
|
||||
insert_columns = columns + [TXO.c.spent_height]
|
||||
select_columns = columns + [func.coalesce(TXI.c.height, 0).label("spent_height")]
|
||||
join_txo_on_txi = old_txo.join(TXI, old_txo.c.txo_hash == TXI.c.txo_hash, isouter=True)
|
||||
select_txos = select(*select_columns).select_from(join_txo_on_txi)
|
||||
insert_txos = TXO.insert().from_select(insert_columns, select_txos)
|
||||
p.ctx.execute(insert_txos)
|
||||
p.step()
|
||||
# 7. drop old txo
|
||||
p.ctx.execute(text("DROP TABLE old_txo;"))
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM ANALYZE txo;"))
|
||||
p.step()
|
||||
for constraint in pg_add_txo_constraints_and_indexes:
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute(text(constraint))
|
||||
p.step()
|
||||
else:
|
||||
p.start(5)
|
||||
# 1. Update spent TXOs setting spent_height
|
||||
update_spent_outputs(p.ctx)
|
||||
p.step()
|
||||
# 2. Update TXIs to have the address of TXO they are spending.
|
||||
set_input_addresses(p.ctx)
|
||||
p.step()
|
||||
# 3. Update tx visibility map, which speeds up index-only scans.
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM tx;"))
|
||||
p.step()
|
||||
# 4. Update txi visibility map, which speeds up index-only scans.
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM txi;"))
|
||||
p.step()
|
||||
# 4. Update txo visibility map, which speeds up index-only scans.
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM txo;"))
|
||||
p.step()
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.mempool.clear", "txs")
|
||||
def clear_mempool(p: ProgressContext):
|
||||
delete_all_the_things(-1, p)
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.mempool.main", "txs")
|
||||
def sync_mempool(p: ProgressContext) -> List[str]:
|
||||
chain = get_or_initialize_lbrycrd(p.ctx)
|
||||
mempool = chain.sync_run(chain.get_raw_mempool())
|
||||
current = [hexlify(r['tx_hash'][::-1]).decode() for r in p.ctx.fetchall(
|
||||
select(TX.c.tx_hash).where(TX.c.height < 0)
|
||||
)]
|
||||
loader = p.ctx.get_bulk_loader()
|
||||
added = []
|
||||
for txid in mempool:
|
||||
if txid not in current:
|
||||
raw_tx = chain.sync_run(chain.get_raw_transaction(txid))
|
||||
loader.add_transaction(
|
||||
None, Transaction(unhexlify(raw_tx), height=-1)
|
||||
)
|
||||
added.append(txid)
|
||||
if p.ctx.stop_event.is_set():
|
||||
return
|
||||
loader.flush(TX)
|
||||
return added
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.filters.generate", "blocks", throttle=100)
|
||||
def sync_filters(start, end, p: ProgressContext):
|
||||
fp = FilterBuilder(start, end)
|
||||
p.start((end-start)+1, progress_id=start, label=f"generate filters {start}-{end}")
|
||||
with p.ctx.connect_streaming() as c:
|
||||
loader = p.ctx.get_bulk_loader()
|
||||
|
||||
tx_hash, height, addresses, last_added = None, None, set(), None
|
||||
address_to_hash = p.ctx.ledger.address_to_hash160
|
||||
for row in c.execute(get_block_tx_addresses_sql(*fp.query_heights)):
|
||||
if tx_hash != row.tx_hash:
|
||||
if tx_hash is not None:
|
||||
last_added = tx_hash
|
||||
fp.add(tx_hash, height, addresses)
|
||||
tx_hash, height, addresses = row.tx_hash, row.height, set()
|
||||
addresses.add(address_to_hash(row.address))
|
||||
if all([last_added, tx_hash]) and last_added != tx_hash: # pickup last tx
|
||||
fp.add(tx_hash, height, addresses)
|
||||
|
||||
for tx_hash, height, addresses in fp.tx_filters:
|
||||
loader.add_transaction_filter(
|
||||
tx_hash, height, create_address_filter(list(addresses))
|
||||
)
|
||||
|
||||
for height, addresses in fp.block_filters.items():
|
||||
loader.add_block_filter(
|
||||
height, 1, create_address_filter(list(addresses))
|
||||
)
|
||||
|
||||
for group_filter in fp.group_filters:
|
||||
for height, addresses in group_filter.groups.items():
|
||||
loader.add_block_filter(
|
||||
height, group_filter.factor, create_address_filter(list(addresses))
|
||||
)
|
||||
|
||||
p.add(loader.flush(BlockFilter))
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.filters.indexes", "steps")
|
||||
def filters_constraints_and_indexes(p: ProgressContext):
|
||||
is_postgres = p.ctx.is_postgres and p.ctx.pg_has_pk_constraint('block_filter')
|
||||
constraints = (
|
||||
pg_add_tx_filter_constraints_and_indexes +
|
||||
pg_add_block_filter_constraints_and_indexes
|
||||
)
|
||||
p.start(2 + len(constraints))
|
||||
if is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM ANALYZE block_filter;"))
|
||||
p.step()
|
||||
if is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM ANALYZE tx_filter;"))
|
||||
p.step()
|
||||
for constraint in constraints:
|
||||
if is_postgres:
|
||||
p.ctx.execute(text(constraint))
|
||||
p.step()
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.filters.vacuum", "steps")
|
||||
def filters_vacuum(p: ProgressContext):
|
||||
p.start(2)
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM block_filter;"))
|
||||
p.step()
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM tx_filter;"))
|
||||
p.step()
|
||||
|
||||
|
||||
def get_block_range_without_filters() -> Tuple[int, int]:
|
||||
sql = (
|
||||
select(
|
||||
func.coalesce(func.min(BlockTable.c.height), -1).label('start_height'),
|
||||
func.coalesce(func.max(BlockTable.c.height), -1).label('end_height'),
|
||||
)
|
||||
.select_from(
|
||||
BlockTable.join(
|
||||
BlockFilter,
|
||||
(BlockTable.c.height == BlockFilter.c.height) &
|
||||
(BlockFilter.c.factor == 1),
|
||||
isouter=True
|
||||
)
|
||||
)
|
||||
.where(BlockFilter.c.height.is_(None))
|
||||
)
|
||||
result = context().fetchone(sql)
|
||||
return result['start_height'], result['end_height']
|
||||
|
||||
|
||||
def get_block_tx_addresses_sql(start_height, end_height):
|
||||
return union(
|
||||
select(TXO.c.tx_hash, TXO.c.height, TXO.c.address).where(
|
||||
(TXO.c.address.isnot(None)) & between(TXO.c.height, start_height, end_height)
|
||||
),
|
||||
select(TXI.c.tx_hash, TXI.c.height, TXI.c.address).where(
|
||||
(TXI.c.address.isnot(None)) & between(TXI.c.height, start_height, end_height)
|
||||
),
|
||||
).order_by('height', 'tx_hash')
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.rewind.main", "steps")
|
||||
def rewind(height: int, p: ProgressContext):
|
||||
delete_all_the_things(height, p)
|
||||
|
||||
|
||||
def delete_all_the_things(height: int, p: ProgressContext):
|
||||
def constrain(col):
|
||||
if height == -1:
|
||||
return col == -1
|
||||
return col >= height
|
||||
|
||||
deletes = [
|
||||
BlockTable.delete().where(constrain(BlockTable.c.height)),
|
||||
TXI.delete().where(constrain(TXI.c.height)),
|
||||
TXO.delete().where(constrain(TXO.c.height)),
|
||||
TX.delete().where(constrain(TX.c.height)),
|
||||
Tag.delete().where(
|
||||
Tag.c.claim_hash.in_(
|
||||
select(Claim.c.claim_hash).where(constrain(Claim.c.height))
|
||||
)
|
||||
),
|
||||
Claim.delete().where(constrain(Claim.c.height)),
|
||||
Support.delete().where(constrain(Support.c.height)),
|
||||
]
|
||||
if height > 0:
|
||||
deletes.extend([
|
||||
# TODO: block and tx filters need better where() clauses (below actually breaks things)
|
||||
BlockFilter.delete().where(BlockFilter.c.height >= height),
|
||||
TXFilter.delete(),
|
||||
])
|
||||
for delete in p.iter(deletes):
|
||||
p.ctx.execute(delete)
|
||||
@@ -1,335 +0,0 @@
|
||||
import logging
|
||||
from typing import Tuple
|
||||
|
||||
from sqlalchemy import case, func, text
|
||||
from sqlalchemy.future import select
|
||||
|
||||
from lbry.db.queries.txio import (
|
||||
minimum_txo_columns, row_to_txo,
|
||||
where_unspent_txos, where_claims_with_changed_supports,
|
||||
count_unspent_txos, where_channels_with_changed_content,
|
||||
where_abandoned_claims, count_channels_with_changed_content,
|
||||
where_claims_with_changed_reposts, where_claims_with_stale_signatures
|
||||
)
|
||||
from lbry.db.query_context import ProgressContext, event_emitter
|
||||
from lbry.db.tables import (
|
||||
TX, TXO, Claim, Support, CensoredClaim,
|
||||
pg_add_claim_and_tag_constraints_and_indexes
|
||||
)
|
||||
from lbry.db.utils import least
|
||||
from lbry.db.constants import TXO_TYPES, CLAIM_TYPE_CODES
|
||||
from lbry.schema.result import Censor
|
||||
from lbry.blockchain.transaction import Output
|
||||
|
||||
from .context import get_or_initialize_lbrycrd
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def channel_content_count_calc(signable):
|
||||
return (
|
||||
select(func.count(signable.c.claim_hash))
|
||||
.where((signable.c.channel_hash == Claim.c.claim_hash) & signable.c.is_signature_valid)
|
||||
.scalar_subquery()
|
||||
)
|
||||
|
||||
|
||||
support = TXO.alias('support')
|
||||
|
||||
|
||||
def staked_support_aggregation(aggregate):
|
||||
return (
|
||||
select(aggregate).where(
|
||||
(support.c.txo_type == TXO_TYPES['support']) &
|
||||
(support.c.spent_height == 0)
|
||||
).scalar_subquery()
|
||||
)
|
||||
|
||||
|
||||
def staked_support_amount_calc(other):
|
||||
return (
|
||||
staked_support_aggregation(func.coalesce(func.sum(support.c.amount), 0))
|
||||
.where(support.c.claim_hash == other.c.claim_hash)
|
||||
)
|
||||
|
||||
|
||||
def staked_support_count_calc(other):
|
||||
return (
|
||||
staked_support_aggregation(func.coalesce(func.count('*'), 0))
|
||||
.where(support.c.claim_hash == other.c.claim_hash)
|
||||
)
|
||||
|
||||
|
||||
def reposted_claim_count_calc(other):
|
||||
repost = TXO.alias('repost')
|
||||
return (
|
||||
select(func.coalesce(func.count(repost.c.reposted_claim_hash), 0))
|
||||
.where(
|
||||
(repost.c.reposted_claim_hash == other.c.claim_hash) &
|
||||
(repost.c.spent_height == 0)
|
||||
).scalar_subquery()
|
||||
)
|
||||
|
||||
|
||||
def make_label(action, blocks):
|
||||
if blocks[0] == blocks[-1]:
|
||||
return f"{action} {blocks[0]:>6}"
|
||||
else:
|
||||
return f"{action} {blocks[0]:>6}-{blocks[-1]:>6}"
|
||||
|
||||
|
||||
def select_claims_for_saving(
|
||||
blocks: Tuple[int, int],
|
||||
missing_in_claims_table=False,
|
||||
missing_or_stale_in_claims_table=False,
|
||||
):
|
||||
channel_txo = TXO.alias('channel_txo')
|
||||
return select(
|
||||
*minimum_txo_columns, TXO.c.claim_hash,
|
||||
staked_support_amount_calc(TXO).label('staked_support_amount'),
|
||||
staked_support_count_calc(TXO).label('staked_support_count'),
|
||||
reposted_claim_count_calc(TXO).label('reposted_count'),
|
||||
TXO.c.signature, TXO.c.signature_digest,
|
||||
case([(
|
||||
TXO.c.channel_hash.isnot(None),
|
||||
select(channel_txo.c.public_key).select_from(channel_txo).where(
|
||||
(channel_txo.c.spent_height == 0) &
|
||||
(channel_txo.c.txo_type == TXO_TYPES['channel']) &
|
||||
(channel_txo.c.claim_hash == TXO.c.channel_hash)
|
||||
).limit(1).scalar_subquery()
|
||||
)]).label('channel_public_key')
|
||||
).where(
|
||||
where_unspent_txos(
|
||||
CLAIM_TYPE_CODES, blocks,
|
||||
missing_in_claims_table=missing_in_claims_table,
|
||||
missing_or_stale_in_claims_table=missing_or_stale_in_claims_table,
|
||||
)
|
||||
).select_from(TXO.join(TX))
|
||||
|
||||
|
||||
def row_to_claim_for_saving(row) -> Tuple[Output, dict]:
|
||||
return row_to_txo(row), {
|
||||
'staked_support_amount': int(row.staked_support_amount),
|
||||
'staked_support_count': int(row.staked_support_count),
|
||||
'reposted_count': int(row.reposted_count),
|
||||
'signature': row.signature,
|
||||
'signature_digest': row.signature_digest,
|
||||
'channel_public_key': row.channel_public_key
|
||||
}
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.claims.insert", "claims")
|
||||
def claims_insert(
|
||||
blocks: Tuple[int, int],
|
||||
missing_in_claims_table: bool,
|
||||
flush_size: int,
|
||||
p: ProgressContext
|
||||
):
|
||||
chain = get_or_initialize_lbrycrd(p.ctx)
|
||||
|
||||
p.start(
|
||||
count_unspent_txos(
|
||||
CLAIM_TYPE_CODES, blocks,
|
||||
missing_in_claims_table=missing_in_claims_table,
|
||||
), progress_id=blocks[0], label=make_label("add claims", blocks)
|
||||
)
|
||||
|
||||
with p.ctx.connect_streaming() as c:
|
||||
loader = p.ctx.get_bulk_loader()
|
||||
cursor = c.execute(select_claims_for_saving(
|
||||
blocks, missing_in_claims_table=missing_in_claims_table
|
||||
).order_by(TXO.c.claim_hash))
|
||||
for rows in cursor.partitions(900):
|
||||
claim_metadata = chain.db.sync_get_claim_metadata(
|
||||
claim_hashes=[row['claim_hash'] for row in rows]
|
||||
)
|
||||
i = 0
|
||||
for row in rows:
|
||||
metadata = claim_metadata[i] if i < len(claim_metadata) else {}
|
||||
if metadata and metadata['claim_hash'] == row.claim_hash:
|
||||
i += 1
|
||||
txo, extra = row_to_claim_for_saving(row)
|
||||
extra.update({
|
||||
'short_url': metadata.get('short_url'),
|
||||
'creation_height': metadata.get('creation_height'),
|
||||
'activation_height': metadata.get('activation_height'),
|
||||
'expiration_height': metadata.get('expiration_height'),
|
||||
'takeover_height': metadata.get('takeover_height'),
|
||||
})
|
||||
loader.add_claim(txo, **extra)
|
||||
if len(loader.claims) >= flush_size:
|
||||
p.add(loader.flush(Claim))
|
||||
p.add(loader.flush(Claim))
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.claims.indexes", "steps")
|
||||
def claims_constraints_and_indexes(p: ProgressContext):
|
||||
should_run = p.ctx.is_postgres and not p.ctx.pg_has_pk_constraint('claim')
|
||||
p.start(2 + len(pg_add_claim_and_tag_constraints_and_indexes))
|
||||
if should_run:
|
||||
p.ctx.execute_notx(text("VACUUM ANALYZE claim;"))
|
||||
p.step()
|
||||
if should_run:
|
||||
p.ctx.execute_notx(text("VACUUM ANALYZE tag;"))
|
||||
p.step()
|
||||
for constraint in pg_add_claim_and_tag_constraints_and_indexes:
|
||||
if should_run:
|
||||
p.ctx.execute(text(constraint))
|
||||
p.step()
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.claims.vacuum", "steps")
|
||||
def claims_vacuum(p: ProgressContext):
|
||||
p.start(2)
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM claim;"))
|
||||
p.step()
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM tag;"))
|
||||
p.step()
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.claims.update", "claims")
|
||||
def claims_update(blocks: Tuple[int, int], p: ProgressContext):
|
||||
p.start(
|
||||
count_unspent_txos(CLAIM_TYPE_CODES, blocks, missing_or_stale_in_claims_table=True),
|
||||
progress_id=blocks[0], label=make_label("mod claims", blocks)
|
||||
)
|
||||
with p.ctx.connect_streaming() as c:
|
||||
loader = p.ctx.get_bulk_loader()
|
||||
cursor = c.execute(select_claims_for_saving(
|
||||
blocks, missing_or_stale_in_claims_table=True
|
||||
))
|
||||
for row in cursor:
|
||||
txo, extra = row_to_claim_for_saving(row)
|
||||
loader.update_claim(txo, **extra)
|
||||
if len(loader.update_claims) >= 25:
|
||||
p.add(loader.flush(Claim))
|
||||
p.add(loader.flush(Claim))
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.claims.delete", "claims")
|
||||
def claims_delete(claims, p: ProgressContext):
|
||||
p.start(claims, label="del claims")
|
||||
deleted = p.ctx.execute(Claim.delete().where(where_abandoned_claims()))
|
||||
p.step(deleted.rowcount)
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.claims.takeovers", "claims")
|
||||
def update_takeovers(blocks: Tuple[int, int], takeovers, p: ProgressContext):
|
||||
p.start(takeovers, label=make_label("mod winner", blocks))
|
||||
chain = get_or_initialize_lbrycrd(p.ctx)
|
||||
with p.ctx.engine.begin() as c:
|
||||
for takeover in chain.db.sync_get_takeovers(start_height=blocks[0], end_height=blocks[-1]):
|
||||
update_claims = (
|
||||
Claim.update()
|
||||
.where(Claim.c.normalized == takeover['normalized'])
|
||||
.values(
|
||||
is_controlling=case(
|
||||
[(Claim.c.claim_hash == takeover['claim_hash'], True)],
|
||||
else_=False
|
||||
),
|
||||
takeover_height=case(
|
||||
[(Claim.c.claim_hash == takeover['claim_hash'], takeover['height'])],
|
||||
else_=None
|
||||
),
|
||||
activation_height=least(Claim.c.activation_height, takeover['height']),
|
||||
)
|
||||
)
|
||||
result = c.execute(update_claims)
|
||||
p.add(result.rowcount)
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.claims.stakes", "claims")
|
||||
def update_stakes(blocks: Tuple[int, int], claims: int, p: ProgressContext):
|
||||
p.start(claims)
|
||||
sql = (
|
||||
Claim.update()
|
||||
.where(where_claims_with_changed_supports(blocks))
|
||||
.values(
|
||||
staked_support_amount=staked_support_amount_calc(Claim),
|
||||
staked_support_count=staked_support_count_calc(Claim),
|
||||
)
|
||||
)
|
||||
result = p.ctx.execute(sql)
|
||||
p.step(result.rowcount)
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.claims.reposts", "claims")
|
||||
def update_reposts(blocks: Tuple[int, int], claims: int, p: ProgressContext):
|
||||
p.start(claims)
|
||||
sql = (
|
||||
Claim.update()
|
||||
.where(where_claims_with_changed_reposts(blocks))
|
||||
.values(reposted_count=reposted_claim_count_calc(Claim))
|
||||
)
|
||||
result = p.ctx.execute(sql)
|
||||
p.step(result.rowcount)
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.claims.invalidate", "claims")
|
||||
def update_stale_signatures(blocks: Tuple[int, int], claims: int, p: ProgressContext):
|
||||
p.start(claims)
|
||||
with p.ctx.connect_streaming() as c:
|
||||
loader = p.ctx.get_bulk_loader()
|
||||
stream = Claim.alias('stream')
|
||||
sql = (
|
||||
select_claims_for_saving(None)
|
||||
.where(TXO.c.claim_hash.in_(
|
||||
where_claims_with_stale_signatures(
|
||||
select(stream.c.claim_hash), blocks, stream
|
||||
)
|
||||
))
|
||||
)
|
||||
cursor = c.execute(sql)
|
||||
for row in cursor:
|
||||
txo, extra = row_to_claim_for_saving(row)
|
||||
loader.update_claim(txo, public_key_height=blocks[1], **extra)
|
||||
if len(loader.update_claims) >= 25:
|
||||
p.add(loader.flush(Claim))
|
||||
p.add(loader.flush(Claim))
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.claims.channels", "channels")
|
||||
def update_channel_stats(blocks: Tuple[int, int], initial_sync: int, p: ProgressContext):
|
||||
update_sql = Claim.update().values(
|
||||
signed_claim_count=channel_content_count_calc(Claim.alias('content')),
|
||||
signed_support_count=channel_content_count_calc(Support),
|
||||
)
|
||||
if initial_sync:
|
||||
p.start(p.ctx.fetchtotal(Claim.c.claim_type == TXO_TYPES['channel']), label="channel stats")
|
||||
update_sql = update_sql.where(Claim.c.claim_type == TXO_TYPES['channel'])
|
||||
elif blocks:
|
||||
p.start(count_channels_with_changed_content(blocks), label="channel stats")
|
||||
update_sql = update_sql.where(where_channels_with_changed_content(blocks))
|
||||
else:
|
||||
return
|
||||
result = p.ctx.execute(update_sql)
|
||||
if result.rowcount and p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM claim;"))
|
||||
p.step(result.rowcount)
|
||||
|
||||
|
||||
def select_reposts(channel_hashes, filter_type):
|
||||
return (
|
||||
select(Claim.c.reposted_claim_hash, filter_type, Claim.c.channel_hash).where(
|
||||
(Claim.c.channel_hash.in_(channel_hashes)) &
|
||||
(Claim.c.reposted_claim_hash.isnot(None))
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.claims.filters", "claim_filters")
|
||||
def update_claim_filters(resolve_censor_channel_hashes, search_censor_channel_hashes, p: ProgressContext):
|
||||
p.ctx.execute(CensoredClaim.delete())
|
||||
# order matters: first we insert the resolve filters; then the search ones.
|
||||
# a claim that's censored in resolve is automatically also censored in search results.
|
||||
p.ctx.execute(CensoredClaim.insert().from_select(
|
||||
['claim_hash', 'censor_type', 'censoring_channel_hash'],
|
||||
select_reposts(resolve_censor_channel_hashes, Censor.RESOLVE)
|
||||
))
|
||||
p.ctx.execute(p.ctx.insert_or_ignore(CensoredClaim).from_select(
|
||||
['claim_hash', 'censor_type', 'censoring_channel_hash'],
|
||||
select_reposts(search_censor_channel_hashes, Censor.SEARCH)
|
||||
))
|
||||
@@ -1,25 +0,0 @@
|
||||
from contextvars import ContextVar
|
||||
from lbry.db import query_context
|
||||
|
||||
from lbry.blockchain.lbrycrd import Lbrycrd
|
||||
|
||||
|
||||
_chain: ContextVar[Lbrycrd] = ContextVar('chain')
|
||||
|
||||
|
||||
def get_or_initialize_lbrycrd(ctx=None) -> Lbrycrd:
|
||||
chain = _chain.get(None)
|
||||
if chain is not None:
|
||||
return chain
|
||||
chain = Lbrycrd((ctx or query_context.context()).ledger)
|
||||
chain.db.sync_open()
|
||||
_chain.set(chain)
|
||||
return chain
|
||||
|
||||
|
||||
def uninitialize():
|
||||
chain = _chain.get(None)
|
||||
if chain is not None:
|
||||
chain.db.sync_close()
|
||||
chain.sync_run(chain.close_session())
|
||||
_chain.set(None)
|
||||
@@ -1,79 +0,0 @@
|
||||
from typing import Dict
|
||||
|
||||
|
||||
def split_range_into_10k_batches(start, end):
|
||||
batch = [start, end]
|
||||
batches = [batch]
|
||||
for block in range(start, end+1):
|
||||
if 0 < block != batch[0] and block % 10_000 == 0:
|
||||
batch = [block, block]
|
||||
batches.append(batch)
|
||||
else:
|
||||
batch[1] = block
|
||||
return batches
|
||||
|
||||
|
||||
class GroupFilter:
|
||||
"""
|
||||
Collects addresses into buckets of specific sizes defined by 10 raised to power of factor.
|
||||
eg. a factor of 2 (10**2) would create block buckets 100-199, 200-299, etc
|
||||
a factor of 3 (10**3) would create block buckets 1000-1999, 2000-2999, etc
|
||||
"""
|
||||
def __init__(self, start, end, factor):
|
||||
self.start = start
|
||||
self.end = end
|
||||
self.factor = factor
|
||||
self.resolution = resolution = 10**factor
|
||||
last_height_in_group, groups = resolution-1, {}
|
||||
for block in range(start, end+1):
|
||||
if block % resolution == last_height_in_group:
|
||||
groups[block-last_height_in_group] = set()
|
||||
self.last_height_in_group = last_height_in_group
|
||||
self.groups: Dict[int, set] = groups
|
||||
|
||||
@property
|
||||
def coverage(self):
|
||||
return list(self.groups.keys())
|
||||
|
||||
def add(self, height, addresses):
|
||||
group = self.groups.get(height - (height % self.resolution))
|
||||
if group is not None:
|
||||
group.update(addresses)
|
||||
|
||||
|
||||
class FilterBuilder:
|
||||
"""
|
||||
Creates filter groups, calculates the necessary block range to fulfill creation
|
||||
of filter groups and collects tx filters, block filters and group filters.
|
||||
"""
|
||||
def __init__(self, start, end):
|
||||
self.start = start
|
||||
self.end = end
|
||||
self.group_filters = [
|
||||
GroupFilter(start, end, 4),
|
||||
GroupFilter(start, end, 3),
|
||||
GroupFilter(start, end, 2),
|
||||
]
|
||||
self.start_tx_height, self.end_tx_height = self._calculate_tx_heights_for_query()
|
||||
self.tx_filters = []
|
||||
self.block_filters: Dict[int, set] = {}
|
||||
|
||||
def _calculate_tx_heights_for_query(self):
|
||||
for group_filter in self.group_filters:
|
||||
if group_filter.groups:
|
||||
return group_filter.coverage[0], self.end
|
||||
return self.start, self.end
|
||||
|
||||
@property
|
||||
def query_heights(self):
|
||||
return self.start_tx_height, self.end_tx_height
|
||||
|
||||
def add(self, tx_hash, height, addresses):
|
||||
if self.start <= height <= self.end:
|
||||
self.tx_filters.append((tx_hash, height, addresses))
|
||||
block_filter = self.block_filters.get(height)
|
||||
if block_filter is None:
|
||||
block_filter = self.block_filters[height] = set()
|
||||
block_filter.update(addresses)
|
||||
for group_filter in self.group_filters:
|
||||
group_filter.add(height, addresses)
|
||||
@@ -1,95 +0,0 @@
|
||||
import logging
|
||||
from typing import Tuple
|
||||
|
||||
from sqlalchemy import case, desc, text
|
||||
from sqlalchemy.future import select
|
||||
|
||||
from lbry.db.tables import TX, TXO, Support, pg_add_support_constraints_and_indexes
|
||||
from lbry.db.query_context import ProgressContext, event_emitter
|
||||
from lbry.db.queries import row_to_txo
|
||||
from lbry.db.constants import TXO_TYPES
|
||||
from lbry.db.queries.txio import (
|
||||
minimum_txo_columns,
|
||||
where_unspent_txos, where_abandoned_supports,
|
||||
count_unspent_txos,
|
||||
)
|
||||
|
||||
from .claims import make_label
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.supports.insert", "supports")
|
||||
def supports_insert(
|
||||
blocks: Tuple[int, int],
|
||||
missing_in_supports_table: bool,
|
||||
flush_size: int,
|
||||
p: ProgressContext
|
||||
):
|
||||
p.start(
|
||||
count_unspent_txos(
|
||||
TXO_TYPES['support'], blocks,
|
||||
missing_in_supports_table=missing_in_supports_table,
|
||||
), progress_id=blocks[0], label=make_label("add supprt", blocks)
|
||||
)
|
||||
channel_txo = TXO.alias('channel_txo')
|
||||
select_supports = select(
|
||||
*minimum_txo_columns, TXO.c.claim_hash,
|
||||
TXO.c.signature, TXO.c.signature_digest,
|
||||
case([(
|
||||
TXO.c.channel_hash.isnot(None),
|
||||
select(channel_txo.c.public_key).select_from(channel_txo).where(
|
||||
(channel_txo.c.txo_type == TXO_TYPES['channel']) &
|
||||
(channel_txo.c.claim_hash == TXO.c.channel_hash) &
|
||||
(channel_txo.c.height <= TXO.c.height)
|
||||
).order_by(desc(channel_txo.c.height)).limit(1).scalar_subquery()
|
||||
)]).label('channel_public_key'),
|
||||
).select_from(
|
||||
TXO.join(TX)
|
||||
).where(
|
||||
where_unspent_txos(
|
||||
TXO_TYPES['support'], blocks,
|
||||
missing_in_supports_table=missing_in_supports_table,
|
||||
)
|
||||
)
|
||||
with p.ctx.connect_streaming() as c:
|
||||
loader = p.ctx.get_bulk_loader()
|
||||
for row in c.execute(select_supports):
|
||||
txo = row_to_txo(row)
|
||||
loader.add_support(
|
||||
txo,
|
||||
signature=row.signature,
|
||||
signature_digest=row.signature_digest,
|
||||
channel_public_key=row.channel_public_key
|
||||
)
|
||||
if len(loader.supports) >= flush_size:
|
||||
p.add(loader.flush(Support))
|
||||
p.add(loader.flush(Support))
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.supports.delete", "supports")
|
||||
def supports_delete(supports, p: ProgressContext):
|
||||
p.start(supports, label="del supprt")
|
||||
deleted = p.ctx.execute(Support.delete().where(where_abandoned_supports()))
|
||||
p.step(deleted.rowcount)
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.supports.indexes", "steps")
|
||||
def supports_constraints_and_indexes(p: ProgressContext):
|
||||
p.start(1 + len(pg_add_support_constraints_and_indexes))
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM ANALYZE support;"))
|
||||
p.step()
|
||||
for constraint in pg_add_support_constraints_and_indexes:
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute(text(constraint))
|
||||
p.step()
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.supports.vacuum", "steps")
|
||||
def supports_vacuum(p: ProgressContext):
|
||||
p.start(1)
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM support;"))
|
||||
p.step()
|
||||
@@ -1,422 +0,0 @@
|
||||
import os
|
||||
import asyncio
|
||||
import logging
|
||||
from binascii import unhexlify
|
||||
from typing import Optional, Tuple, Set, List, Coroutine
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from lbry.db import Database, trending
|
||||
from lbry.db import queries as q
|
||||
from lbry.db.constants import TXO_TYPES, CLAIM_TYPE_CODES
|
||||
from lbry.db.query_context import Event, Progress
|
||||
from lbry.event import BroadcastSubscription, EventController
|
||||
from lbry.service.base import Sync, BlockEvent
|
||||
from lbry.blockchain.lbrycrd import Lbrycrd
|
||||
from lbry.error import LbrycrdEventSubscriptionError
|
||||
|
||||
from . import blocks as block_phase, claims as claim_phase, supports as support_phase
|
||||
from .context import uninitialize
|
||||
from .filter_builder import split_range_into_10k_batches
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
BLOCKS_INIT_EVENT = Event.add("blockchain.sync.blocks.init", "steps")
|
||||
BLOCKS_MAIN_EVENT = Event.add("blockchain.sync.blocks.main", "blocks", "txs")
|
||||
FILTER_INIT_EVENT = Event.add("blockchain.sync.filters.init", "steps")
|
||||
FILTER_MAIN_EVENT = Event.add("blockchain.sync.filters.main", "blocks")
|
||||
CLAIMS_INIT_EVENT = Event.add("blockchain.sync.claims.init", "steps")
|
||||
CLAIMS_MAIN_EVENT = Event.add("blockchain.sync.claims.main", "claims")
|
||||
TRENDS_INIT_EVENT = Event.add("blockchain.sync.trends.init", "steps")
|
||||
TRENDS_MAIN_EVENT = Event.add("blockchain.sync.trends.main", "blocks")
|
||||
SUPPORTS_INIT_EVENT = Event.add("blockchain.sync.supports.init", "steps")
|
||||
SUPPORTS_MAIN_EVENT = Event.add("blockchain.sync.supports.main", "supports")
|
||||
|
||||
|
||||
class BlockchainSync(Sync):
|
||||
|
||||
TX_FLUSH_SIZE = 25_000 # flush to db after processing this many TXs and update progress
|
||||
CLAIM_FLUSH_SIZE = 25_000 # flush to db after processing this many claims and update progress
|
||||
SUPPORT_FLUSH_SIZE = 25_000 # flush to db after processing this many supports and update progress
|
||||
FILTER_FLUSH_SIZE = 10_000 # flush to db after processing this many filters and update progress
|
||||
|
||||
def __init__(self, chain: Lbrycrd, db: Database):
|
||||
super().__init__(chain.ledger, db)
|
||||
self.chain = chain
|
||||
self.pid = os.getpid()
|
||||
self._on_block_controller = EventController()
|
||||
self.on_block = self._on_block_controller.stream
|
||||
self.conf.events.register("blockchain.block", self.on_block)
|
||||
self._on_mempool_controller = EventController()
|
||||
self.on_mempool = self._on_mempool_controller.stream
|
||||
self.on_block_hash_subscription: Optional[BroadcastSubscription] = None
|
||||
self.on_tx_hash_subscription: Optional[BroadcastSubscription] = None
|
||||
self.advance_loop_task: Optional[asyncio.Task] = None
|
||||
self.block_hash_event = asyncio.Event()
|
||||
self.tx_hash_event = asyncio.Event()
|
||||
self.mempool = []
|
||||
self.search_censor_channel_hashes = {
|
||||
unhexlify(channel_id)[::-1] for channel_id in self.conf.search_censor_channel_ids
|
||||
}
|
||||
self.resolve_censor_channel_hashes = {
|
||||
unhexlify(channel_id)[::-1] for channel_id in self.conf.resolve_censor_channel_ids
|
||||
}
|
||||
|
||||
async def wait_for_chain_ready(self):
|
||||
while True:
|
||||
try:
|
||||
return await self.chain.ensure_subscribable()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except LbrycrdEventSubscriptionError as e:
|
||||
log.warning(
|
||||
"Lbrycrd is misconfigured. Please double check if"
|
||||
" zmqpubhashblock is properly set on lbrycrd.conf"
|
||||
)
|
||||
raise
|
||||
except Exception as e:
|
||||
log.warning("Blockchain not ready, waiting for it: %s", str(e))
|
||||
await asyncio.sleep(1)
|
||||
|
||||
async def start(self):
|
||||
self.db.stop_event.clear()
|
||||
await self.wait_for_chain_ready()
|
||||
self.advance_loop_task = asyncio.create_task(self.advance())
|
||||
await self.advance_loop_task
|
||||
await self.chain.subscribe()
|
||||
self.advance_loop_task = asyncio.create_task(self.advance_loop())
|
||||
self.on_block_hash_subscription = self.chain.on_block_hash.listen(
|
||||
lambda e: self.block_hash_event.set()
|
||||
)
|
||||
self.on_tx_hash_subscription = self.chain.on_tx_hash.listen(
|
||||
lambda e: self.tx_hash_event.set()
|
||||
)
|
||||
|
||||
async def stop(self):
|
||||
self.chain.unsubscribe()
|
||||
self.db.stop_event.set()
|
||||
for subscription in (
|
||||
self.on_block_hash_subscription,
|
||||
self.on_tx_hash_subscription,
|
||||
self.advance_loop_task
|
||||
):
|
||||
if subscription is not None:
|
||||
subscription.cancel()
|
||||
if isinstance(self.db.executor, ThreadPoolExecutor):
|
||||
await self.db.run(uninitialize)
|
||||
|
||||
async def run_tasks(self, tasks: List[Coroutine]) -> Optional[Set[asyncio.Future]]:
|
||||
done, pending = await asyncio.wait(
|
||||
tasks, return_when=asyncio.FIRST_EXCEPTION
|
||||
)
|
||||
if pending:
|
||||
self.db.stop_event.set()
|
||||
for future in pending:
|
||||
future.cancel()
|
||||
for future in done:
|
||||
future.result()
|
||||
return
|
||||
return done
|
||||
|
||||
async def get_block_headers(self, start_height: int, end_height: int = None):
|
||||
return await self.db.get_block_headers(start_height, end_height)
|
||||
|
||||
async def get_best_block_height(self) -> int:
|
||||
return await self.db.get_best_block_height()
|
||||
|
||||
async def get_best_block_height_for_file(self, file_number) -> int:
|
||||
return await self.db.run(
|
||||
block_phase.get_best_block_height_for_file, file_number
|
||||
)
|
||||
|
||||
async def sync_blocks(self) -> Optional[Tuple[int, int]]:
|
||||
tasks = []
|
||||
starting_height = None
|
||||
tx_count = block_count = 0
|
||||
with Progress(self.db.message_queue, BLOCKS_INIT_EVENT) as p:
|
||||
ending_height = await self.chain.db.get_best_height()
|
||||
for chain_file in p.iter(await self.chain.db.get_block_files()):
|
||||
# block files may be read and saved out of order, need to check
|
||||
# each file individually to see if we have missing blocks
|
||||
our_best_file_height = await self.get_best_block_height_for_file(
|
||||
chain_file['file_number']
|
||||
)
|
||||
if our_best_file_height == chain_file['best_height']:
|
||||
# we have all blocks in this file, skipping
|
||||
continue
|
||||
if -1 < our_best_file_height < chain_file['best_height']:
|
||||
# we have some blocks, need to figure out what we're missing
|
||||
# call get_block_files again limited to this file and current_height
|
||||
chain_file = (await self.chain.db.get_block_files(
|
||||
file_number=chain_file['file_number'], start_height=our_best_file_height+1,
|
||||
))[0]
|
||||
tx_count += chain_file['txs']
|
||||
block_count += chain_file['blocks']
|
||||
file_start_height = chain_file['start_height']
|
||||
starting_height = min(
|
||||
file_start_height if starting_height is None else starting_height,
|
||||
file_start_height
|
||||
)
|
||||
tasks.append(self.db.run(
|
||||
block_phase.sync_block_file, chain_file['file_number'], file_start_height,
|
||||
chain_file['txs'], self.TX_FLUSH_SIZE
|
||||
))
|
||||
with Progress(self.db.message_queue, BLOCKS_MAIN_EVENT) as p:
|
||||
p.start(block_count, tx_count, extra={
|
||||
"starting_height": starting_height,
|
||||
"ending_height": ending_height,
|
||||
"files": len(tasks),
|
||||
"claims": await self.chain.db.get_claim_metadata_count(starting_height, ending_height),
|
||||
"supports": await self.chain.db.get_support_metadata_count(starting_height, ending_height),
|
||||
})
|
||||
completed = await self.run_tasks(tasks)
|
||||
if completed:
|
||||
if starting_height == 0:
|
||||
await self.db.run(block_phase.blocks_constraints_and_indexes)
|
||||
else:
|
||||
await self.db.run(block_phase.blocks_vacuum)
|
||||
best_height_processed = max(f.result() for f in completed)
|
||||
return starting_height, best_height_processed
|
||||
|
||||
async def sync_filters(self):
|
||||
with Progress(self.db.message_queue, FILTER_INIT_EVENT) as p:
|
||||
p.start(2)
|
||||
initial_sync = not await self.db.has_filters()
|
||||
p.step()
|
||||
if initial_sync:
|
||||
blocks = [0, await self.db.get_best_block_height()]
|
||||
else:
|
||||
blocks = await self.db.run(block_phase.get_block_range_without_filters)
|
||||
if blocks != (-1, -1):
|
||||
batches = split_range_into_10k_batches(*blocks)
|
||||
p.step()
|
||||
else:
|
||||
p.step()
|
||||
return
|
||||
with Progress(self.db.message_queue, FILTER_MAIN_EVENT) as p:
|
||||
p.start((blocks[1]-blocks[0])+1)
|
||||
await self.run_tasks([
|
||||
self.db.run(block_phase.sync_filters, *batch) for batch in batches
|
||||
])
|
||||
if initial_sync:
|
||||
await self.db.run(block_phase.filters_constraints_and_indexes)
|
||||
else:
|
||||
await self.db.run(block_phase.filters_vacuum)
|
||||
|
||||
async def sync_spends(self, blocks_added):
|
||||
if blocks_added:
|
||||
await self.db.run(block_phase.sync_spends, blocks_added[0] == 0)
|
||||
|
||||
async def count_unspent_txos(
|
||||
self,
|
||||
txo_types: Tuple[int, ...],
|
||||
blocks: Tuple[int, int] = None,
|
||||
missing_in_supports_table: bool = False,
|
||||
missing_in_claims_table: bool = False,
|
||||
missing_or_stale_in_claims_table: bool = False,
|
||||
) -> int:
|
||||
return await self.db.run(
|
||||
q.count_unspent_txos, txo_types, blocks,
|
||||
missing_in_supports_table,
|
||||
missing_in_claims_table,
|
||||
missing_or_stale_in_claims_table,
|
||||
)
|
||||
|
||||
async def distribute_unspent_txos(
|
||||
self,
|
||||
txo_types: Tuple[int, ...],
|
||||
blocks: Tuple[int, int] = None,
|
||||
missing_in_supports_table: bool = False,
|
||||
missing_in_claims_table: bool = False,
|
||||
missing_or_stale_in_claims_table: bool = False,
|
||||
) -> int:
|
||||
return await self.db.run(
|
||||
q.distribute_unspent_txos, txo_types, blocks,
|
||||
missing_in_supports_table,
|
||||
missing_in_claims_table,
|
||||
missing_or_stale_in_claims_table,
|
||||
self.db.workers
|
||||
)
|
||||
|
||||
async def count_abandoned_supports(self) -> int:
|
||||
return await self.db.run(q.count_abandoned_supports)
|
||||
|
||||
async def count_abandoned_claims(self) -> int:
|
||||
return await self.db.run(q.count_abandoned_claims)
|
||||
|
||||
async def count_claims_with_changed_supports(self, blocks) -> int:
|
||||
return await self.db.run(q.count_claims_with_changed_supports, blocks)
|
||||
|
||||
async def count_claims_with_changed_reposts(self, blocks) -> int:
|
||||
return await self.db.run(q.count_claims_with_changed_reposts, blocks)
|
||||
|
||||
async def count_claims_with_stale_signatures(self, blocks) -> int:
|
||||
return await self.db.run(q.count_claims_with_stale_signatures, blocks)
|
||||
|
||||
async def count_channels_with_changed_content(self, blocks) -> int:
|
||||
return await self.db.run(q.count_channels_with_changed_content, blocks)
|
||||
|
||||
async def count_takeovers(self, blocks) -> int:
|
||||
return await self.chain.db.get_takeover_count(
|
||||
start_height=blocks[0], end_height=blocks[-1]
|
||||
)
|
||||
|
||||
async def sync_claims(self, blocks) -> bool:
|
||||
delete_claims = takeovers = claims_with_changed_supports =\
|
||||
claims_with_changed_reposts = claims_with_stale_signatures = 0
|
||||
initial_sync = not await self.db.has_claims()
|
||||
with Progress(self.db.message_queue, CLAIMS_INIT_EVENT) as p:
|
||||
if initial_sync:
|
||||
total, batches = await self.distribute_unspent_txos(CLAIM_TYPE_CODES)
|
||||
elif blocks:
|
||||
p.start(6)
|
||||
# 1. content claims to be inserted or updated
|
||||
total = await self.count_unspent_txos(
|
||||
CLAIM_TYPE_CODES, blocks, missing_or_stale_in_claims_table=True
|
||||
)
|
||||
batches = [blocks] if total else []
|
||||
p.step()
|
||||
# 2. claims to be deleted
|
||||
delete_claims = await self.count_abandoned_claims()
|
||||
total += delete_claims
|
||||
p.step()
|
||||
# 3. claims to be updated with new support totals
|
||||
claims_with_changed_supports = await self.count_claims_with_changed_supports(blocks)
|
||||
total += claims_with_changed_supports
|
||||
p.step()
|
||||
# 4. claims to be updated with new repost totals
|
||||
claims_with_changed_reposts = await self.count_claims_with_changed_reposts(blocks)
|
||||
total += claims_with_changed_reposts
|
||||
p.step()
|
||||
# 5. claims to be updated due to name takeovers
|
||||
takeovers = await self.count_takeovers(blocks)
|
||||
total += takeovers
|
||||
p.step()
|
||||
# 6. claims where channel signature changed and claim was not re-signed in time
|
||||
claims_with_stale_signatures = await self.count_claims_with_stale_signatures(blocks)
|
||||
total += claims_with_stale_signatures
|
||||
p.step()
|
||||
else:
|
||||
return initial_sync
|
||||
with Progress(self.db.message_queue, CLAIMS_MAIN_EVENT) as p:
|
||||
p.start(total)
|
||||
if batches:
|
||||
await self.run_tasks([
|
||||
self.db.run(claim_phase.claims_insert, batch, not initial_sync, self.CLAIM_FLUSH_SIZE)
|
||||
for batch in batches
|
||||
])
|
||||
if not initial_sync:
|
||||
await self.run_tasks([
|
||||
self.db.run(claim_phase.claims_update, batch) for batch in batches
|
||||
])
|
||||
if delete_claims:
|
||||
await self.db.run(claim_phase.claims_delete, delete_claims)
|
||||
if takeovers:
|
||||
await self.db.run(claim_phase.update_takeovers, blocks, takeovers)
|
||||
if claims_with_changed_supports:
|
||||
await self.db.run(claim_phase.update_stakes, blocks, claims_with_changed_supports)
|
||||
if claims_with_changed_reposts:
|
||||
await self.db.run(claim_phase.update_reposts, blocks, claims_with_changed_reposts)
|
||||
if claims_with_stale_signatures:
|
||||
await self.db.run(claim_phase.update_stale_signatures, blocks, claims_with_stale_signatures)
|
||||
if initial_sync:
|
||||
await self.db.run(claim_phase.claims_constraints_and_indexes)
|
||||
else:
|
||||
await self.db.run(claim_phase.claims_vacuum)
|
||||
return initial_sync
|
||||
|
||||
async def sync_supports(self, blocks):
|
||||
delete_supports = 0
|
||||
initial_sync = not await self.db.has_supports()
|
||||
with Progress(self.db.message_queue, SUPPORTS_INIT_EVENT) as p:
|
||||
if initial_sync:
|
||||
total, support_batches = await self.distribute_unspent_txos(TXO_TYPES['support'])
|
||||
elif blocks:
|
||||
p.start(2)
|
||||
# 1. supports to be inserted
|
||||
total = await self.count_unspent_txos(
|
||||
TXO_TYPES['support'], blocks, missing_in_supports_table=True
|
||||
)
|
||||
support_batches = [blocks] if total else []
|
||||
p.step()
|
||||
# 2. supports to be deleted
|
||||
delete_supports = await self.count_abandoned_supports()
|
||||
total += delete_supports
|
||||
p.step()
|
||||
else:
|
||||
return
|
||||
with Progress(self.db.message_queue, SUPPORTS_MAIN_EVENT) as p:
|
||||
p.start(total)
|
||||
if support_batches:
|
||||
await self.run_tasks([
|
||||
self.db.run(
|
||||
support_phase.supports_insert, batch, not initial_sync, self.SUPPORT_FLUSH_SIZE
|
||||
) for batch in support_batches
|
||||
])
|
||||
if delete_supports:
|
||||
await self.db.run(support_phase.supports_delete, delete_supports)
|
||||
if initial_sync:
|
||||
await self.db.run(support_phase.supports_constraints_and_indexes)
|
||||
else:
|
||||
await self.db.run(support_phase.supports_vacuum)
|
||||
|
||||
async def sync_channel_stats(self, blocks, initial_sync):
|
||||
await self.db.run(claim_phase.update_channel_stats, blocks, initial_sync)
|
||||
|
||||
async def sync_trends(self):
|
||||
ending_height = await self.chain.db.get_best_height()
|
||||
if ending_height is not None:
|
||||
await self.db.run(trending.calculate_trending, ending_height)
|
||||
|
||||
async def sync_claim_filtering(self):
|
||||
await self.db.run(
|
||||
claim_phase.update_claim_filters,
|
||||
self.resolve_censor_channel_hashes,
|
||||
self.search_censor_channel_hashes
|
||||
)
|
||||
|
||||
async def advance(self):
|
||||
blocks_added = await self.sync_blocks()
|
||||
await self.sync_spends(blocks_added)
|
||||
await self.sync_filters()
|
||||
initial_claim_sync = await self.sync_claims(blocks_added)
|
||||
await self.sync_supports(blocks_added)
|
||||
await self.sync_channel_stats(blocks_added, initial_claim_sync)
|
||||
await self.sync_trends()
|
||||
await self.sync_claim_filtering()
|
||||
if blocks_added:
|
||||
await self._on_block_controller.add(BlockEvent(blocks_added[-1]))
|
||||
|
||||
async def sync_mempool(self):
|
||||
added = await self.db.run(block_phase.sync_mempool)
|
||||
await self.sync_spends([-1])
|
||||
await self.db.run(claim_phase.claims_insert, [-1, -1], True, self.CLAIM_FLUSH_SIZE)
|
||||
await self.db.run(claim_phase.claims_update, [-1, -1])
|
||||
await self.db.run(claim_phase.claims_vacuum)
|
||||
self.mempool.extend(added)
|
||||
await self._on_mempool_controller.add(added)
|
||||
|
||||
async def clear_mempool(self):
|
||||
self.mempool.clear()
|
||||
await self.db.run(block_phase.clear_mempool)
|
||||
|
||||
async def advance_loop(self):
|
||||
while True:
|
||||
try:
|
||||
await asyncio.wait([
|
||||
self.tx_hash_event.wait(),
|
||||
self.block_hash_event.wait(),
|
||||
], return_when=asyncio.FIRST_COMPLETED)
|
||||
if self.block_hash_event.is_set():
|
||||
self.block_hash_event.clear()
|
||||
#await self.clear_mempool()
|
||||
await self.advance()
|
||||
self.tx_hash_event.clear()
|
||||
#await self.sync_mempool()
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
except Exception as e:
|
||||
log.exception(e)
|
||||
await self.stop()
|
||||
|
||||
async def rewind(self, height):
|
||||
await self.db.run(block_phase.rewind, height)
|
||||
+194
-139
@@ -1,23 +1,21 @@
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import typing
|
||||
import logging
|
||||
from typing import List, Dict, Tuple, Union, TypeVar, Generic, Optional
|
||||
from argparse import ArgumentParser
|
||||
from contextlib import contextmanager
|
||||
from typing import Tuple
|
||||
|
||||
from appdirs import user_data_dir, user_config_dir
|
||||
import yaml
|
||||
from lbry.utils.dirs import user_data_dir, user_download_dir
|
||||
from lbry.error import InvalidCurrencyError
|
||||
from lbry.dht import constants
|
||||
from lbry.wallet.coinselection import COIN_SELECTION_STRATEGIES
|
||||
from lbry.event import EventRegistry
|
||||
from lbry.wallet.coinselection import STRATEGIES
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
NOT_SET = type('NOT_SET', (object,), {}) # pylint: disable=invalid-name
|
||||
T = typing.TypeVar('T')
|
||||
T = TypeVar('T')
|
||||
|
||||
CURRENCIES = {
|
||||
'BTC': {'type': 'crypto'},
|
||||
@@ -26,11 +24,11 @@ CURRENCIES = {
|
||||
}
|
||||
|
||||
|
||||
class Setting(typing.Generic[T]):
|
||||
class Setting(Generic[T]):
|
||||
|
||||
def __init__(self, doc: str, default: typing.Optional[T] = None,
|
||||
previous_names: typing.Optional[typing.List[str]] = None,
|
||||
metavar: typing.Optional[str] = None):
|
||||
def __init__(self, doc: str, default: Optional[T] = None,
|
||||
previous_names: Optional[List[str]] = None,
|
||||
metavar: Optional[str] = None):
|
||||
self.doc = doc
|
||||
self.default = default
|
||||
self.previous_names = previous_names or []
|
||||
@@ -47,7 +45,7 @@ class Setting(typing.Generic[T]):
|
||||
def no_cli_name(self):
|
||||
return f"--no-{self.name.replace('_', '-')}"
|
||||
|
||||
def __get__(self, obj: typing.Optional['BaseConfig'], owner) -> T:
|
||||
def __get__(self, obj: Optional['BaseConfig'], owner) -> T:
|
||||
if obj is None:
|
||||
return self
|
||||
for location in obj.search_order:
|
||||
@@ -55,7 +53,7 @@ class Setting(typing.Generic[T]):
|
||||
return location[self.name]
|
||||
return self.default
|
||||
|
||||
def __set__(self, obj: 'BaseConfig', val: typing.Union[T, NOT_SET]):
|
||||
def __set__(self, obj: 'BaseConfig', val: Union[T, NOT_SET]):
|
||||
if val == NOT_SET:
|
||||
for location in obj.modify_order:
|
||||
if self.name in location:
|
||||
@@ -65,6 +63,18 @@ class Setting(typing.Generic[T]):
|
||||
for location in obj.modify_order:
|
||||
location[self.name] = val
|
||||
|
||||
def is_set(self, obj: 'BaseConfig') -> bool:
|
||||
for location in obj.search_order:
|
||||
if self.name in location:
|
||||
return True
|
||||
return False
|
||||
|
||||
def is_set_to_default(self, obj: 'BaseConfig') -> bool:
|
||||
for location in obj.search_order:
|
||||
if self.name in location:
|
||||
return location[self.name] == self.default
|
||||
return False
|
||||
|
||||
def validate(self, value):
|
||||
raise NotImplementedError()
|
||||
|
||||
@@ -89,7 +99,7 @@ class String(Setting[str]):
|
||||
f"Setting '{self.name}' must be a string."
|
||||
|
||||
# TODO: removes this after pylint starts to understand generics
|
||||
def __get__(self, obj: typing.Optional['BaseConfig'], owner) -> str: # pylint: disable=useless-super-delegation
|
||||
def __get__(self, obj: Optional['BaseConfig'], owner) -> str: # pylint: disable=useless-super-delegation
|
||||
return super().__get__(obj, owner)
|
||||
|
||||
|
||||
@@ -193,7 +203,7 @@ class MaxKeyFee(Setting[dict]):
|
||||
)
|
||||
parser.add_argument(
|
||||
self.no_cli_name,
|
||||
help="Disable maximum key fee check.",
|
||||
help=f"Disable maximum key fee check.",
|
||||
dest=self.name,
|
||||
const=None,
|
||||
action="store_const",
|
||||
@@ -202,7 +212,7 @@ class MaxKeyFee(Setting[dict]):
|
||||
|
||||
|
||||
class StringChoice(String):
|
||||
def __init__(self, doc: str, valid_values: typing.List[str], default: str, *args, **kwargs):
|
||||
def __init__(self, doc: str, valid_values: List[str], default: str, *args, **kwargs):
|
||||
super().__init__(doc, default, *args, **kwargs)
|
||||
if not valid_values:
|
||||
raise ValueError("No valid values provided")
|
||||
@@ -275,17 +285,95 @@ class Strings(ListSetting):
|
||||
f"'{self.name}' must be a string."
|
||||
|
||||
|
||||
class KnownHubsList:
|
||||
|
||||
def __init__(self, config: 'Config' = None, file_name: str = 'known_hubs.yml'):
|
||||
self.file_name = file_name
|
||||
self.path = os.path.join(config.wallet_dir, self.file_name) if config else None
|
||||
self.hubs: Dict[Tuple[str, int], Dict] = {}
|
||||
if self.exists:
|
||||
self.load()
|
||||
|
||||
@property
|
||||
def exists(self):
|
||||
return self.path and os.path.exists(self.path)
|
||||
|
||||
@property
|
||||
def serialized(self) -> Dict[str, Dict]:
|
||||
return {f"{host}:{port}": details for (host, port), details in self.hubs.items()}
|
||||
|
||||
def filter(self, match_none=False, **kwargs):
|
||||
if not kwargs:
|
||||
return self.hubs
|
||||
result = {}
|
||||
for hub, details in self.hubs.items():
|
||||
for key, constraint in kwargs.items():
|
||||
value = details.get(key)
|
||||
if value == constraint or (match_none and value is None):
|
||||
result[hub] = details
|
||||
break
|
||||
return result
|
||||
|
||||
def load(self):
|
||||
if self.path:
|
||||
with open(self.path, 'r') as known_hubs_file:
|
||||
raw = known_hubs_file.read()
|
||||
for hub, details in yaml.safe_load(raw).items():
|
||||
self.set(hub, details)
|
||||
|
||||
def save(self):
|
||||
if self.path:
|
||||
with open(self.path, 'w') as known_hubs_file:
|
||||
known_hubs_file.write(yaml.safe_dump(self.serialized, default_flow_style=False))
|
||||
|
||||
def set(self, hub: str, details: Dict):
|
||||
if hub and hub.count(':') == 1:
|
||||
host, port = hub.split(':')
|
||||
hub_parts = (host, int(port))
|
||||
if hub_parts not in self.hubs:
|
||||
self.hubs[hub_parts] = details
|
||||
return hub
|
||||
|
||||
def add_hubs(self, hubs: List[str]):
|
||||
added = False
|
||||
for hub in hubs:
|
||||
if self.set(hub, {}) is not None:
|
||||
added = True
|
||||
return added
|
||||
|
||||
def items(self):
|
||||
return self.hubs.items()
|
||||
|
||||
def __bool__(self):
|
||||
return len(self) > 0
|
||||
|
||||
def __len__(self):
|
||||
return self.hubs.__len__()
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self.hubs)
|
||||
|
||||
|
||||
class EnvironmentAccess:
|
||||
PREFIX = 'LBRY_'
|
||||
|
||||
def __init__(self, environ: dict):
|
||||
self.environ = environ
|
||||
def __init__(self, config: 'BaseConfig', environ: dict):
|
||||
self.configuration = config
|
||||
self.data = {}
|
||||
if environ:
|
||||
self.load(environ)
|
||||
|
||||
def load(self, environ):
|
||||
for setting in self.configuration.get_settings():
|
||||
value = environ.get(f'{self.PREFIX}{setting.name.upper()}', NOT_SET)
|
||||
if value != NOT_SET and not (isinstance(setting, ListSetting) and value is None):
|
||||
self.data[setting.name] = setting.deserialize(value)
|
||||
|
||||
def __contains__(self, item: str):
|
||||
return f'{self.PREFIX}{item.upper()}' in self.environ
|
||||
return item in self.data
|
||||
|
||||
def __getitem__(self, item: str):
|
||||
return self.environ[f'{self.PREFIX}{item.upper()}']
|
||||
return self.data[item]
|
||||
|
||||
|
||||
class ArgumentAccess:
|
||||
@@ -326,7 +414,7 @@ class ConfigFileAccess:
|
||||
cls = type(self.configuration)
|
||||
with open(self.path, 'r') as config_file:
|
||||
raw = config_file.read()
|
||||
serialized = yaml.full_load(raw) or {}
|
||||
serialized = yaml.safe_load(raw) or {}
|
||||
for key, value in serialized.items():
|
||||
attr = getattr(cls, key, None)
|
||||
if attr is None:
|
||||
@@ -370,7 +458,7 @@ class ConfigFileAccess:
|
||||
del self.data[key]
|
||||
|
||||
|
||||
TBC = typing.TypeVar('TBC', bound='BaseConfig')
|
||||
TBC = TypeVar('TBC', bound='BaseConfig')
|
||||
|
||||
|
||||
class BaseConfig:
|
||||
@@ -383,13 +471,8 @@ class BaseConfig:
|
||||
self.environment = {} # from environment variables
|
||||
self.persisted = {} # from config file
|
||||
self._updating_config = False
|
||||
self.events = EventRegistry()
|
||||
self.set(**kwargs)
|
||||
|
||||
def set(self, **kwargs):
|
||||
for key, value in kwargs.items():
|
||||
setattr(self, key, value)
|
||||
return self
|
||||
|
||||
@contextmanager
|
||||
def update_config(self):
|
||||
@@ -449,7 +532,7 @@ class BaseConfig:
|
||||
self.arguments = ArgumentAccess(self, args)
|
||||
|
||||
def set_environment(self, environ=None):
|
||||
self.environment = EnvironmentAccess(environ or os.environ)
|
||||
self.environment = EnvironmentAccess(self, environ or os.environ)
|
||||
|
||||
def set_persisted(self, config_file_path=None):
|
||||
if config_file_path is None:
|
||||
@@ -475,16 +558,16 @@ class TranscodeConfig(BaseConfig):
|
||||
'', previous_names=['ffmpeg_folder'])
|
||||
video_encoder = String('FFmpeg codec and parameters for the video encoding. '
|
||||
'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental',
|
||||
'libx264 -crf 21 -preset faster -pix_fmt yuv420p')
|
||||
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 8400000)
|
||||
'libx264 -crf 24 -preset faster -pix_fmt yuv420p')
|
||||
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 5_000_000)
|
||||
video_scaler = String('FFmpeg scaling parameters for reducing bitrate. '
|
||||
'Example: -vf "scale=-2:720,fps=24" -maxrate 5M -bufsize 3M',
|
||||
r'-vf "scale=if(gte(iw\,ih)\,min(1920\,iw)\,-2):if(lt(iw\,ih)\,min(1920\,ih)\,-2)" '
|
||||
r'-maxrate 8400K -bufsize 5000K')
|
||||
r'-maxrate 5500K -bufsize 5000K')
|
||||
audio_encoder = String('FFmpeg codec and parameters for the audio encoding. '
|
||||
'Example: libopus -b:a 128k',
|
||||
'aac -b:a 160k')
|
||||
volume_filter = String('FFmpeg filter for audio normalization.', '-af loudnorm')
|
||||
volume_filter = String('FFmpeg filter for audio normalization. Exmple: -af loudnorm', '')
|
||||
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', 240)
|
||||
|
||||
|
||||
@@ -506,30 +589,22 @@ class CLIConfig(TranscodeConfig):
|
||||
|
||||
|
||||
class Config(CLIConfig):
|
||||
db_url = String("Database connection URL, uses a local file based SQLite by default.")
|
||||
workers = Integer(
|
||||
"Multiprocessing, specify number of worker processes lbrynet can start (including main process)."
|
||||
" (-1: threads only, 0: equal to number of CPUs, >1: specific number of processes)", -1
|
||||
)
|
||||
console = StringChoice(
|
||||
"Basic text console output or advanced colored output with progress bars.",
|
||||
["basic", "advanced", "none"], "advanced"
|
||||
)
|
||||
|
||||
jurisdiction = String("Limit interactions to wallet server in this jurisdiction.")
|
||||
|
||||
# directories
|
||||
download_dir = Path("Directory to store downloaded files.", metavar='DIR')
|
||||
data_dir = Path("Main directory containing blobs, wallets and blockchain data.", metavar='DIR')
|
||||
blob_dir = Path("Directory to store blobs (default: 'data_dir'/blobs).", metavar='DIR')
|
||||
wallet_dir = Path("Directory to store wallets (default: 'data_dir'/wallets).", metavar='DIR')
|
||||
wallet_storage = StringChoice("Wallet storage mode.", ["file", "database"], "file")
|
||||
data_dir = Path("Directory path to store blobs.", metavar='DIR')
|
||||
download_dir = Path(
|
||||
"Directory path to place assembled files downloaded from LBRY.",
|
||||
previous_names=['download_directory'], metavar='DIR'
|
||||
)
|
||||
wallet_dir = Path(
|
||||
"Directory containing a 'wallets' subdirectory with 'default_wallet' file.",
|
||||
previous_names=['lbryum_wallet_dir'], metavar='DIR'
|
||||
)
|
||||
wallets = Strings(
|
||||
"Wallet files in 'wallet_dir' to load at startup.", ['default_wallet']
|
||||
)
|
||||
create_default_wallet = Toggle(
|
||||
"Create an initial wallet if it does not exist on startup.", True
|
||||
)
|
||||
create_default_account = Toggle(
|
||||
"Create an initial account if it does not exist in the default wallet.", True
|
||||
"Wallet files in 'wallet_dir' to load at startup.",
|
||||
['default_wallet']
|
||||
)
|
||||
|
||||
# network
|
||||
@@ -553,6 +628,7 @@ class Config(CLIConfig):
|
||||
# protocol timeouts
|
||||
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
|
||||
blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
|
||||
hub_timeout = Float("Timeout when making a hub request", 30.0)
|
||||
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0)
|
||||
node_rpc_timeout = Float("Timeout when making a DHT request", constants.RPC_TIMEOUT)
|
||||
|
||||
@@ -574,6 +650,7 @@ class Config(CLIConfig):
|
||||
"Maximum number of peers to connect to while downloading a blob", 4,
|
||||
previous_names=['max_connections_per_stream']
|
||||
)
|
||||
concurrent_hub_requests = Integer("Maximum number of concurrent hub requests", 32)
|
||||
fixed_peer_delay = Float(
|
||||
"Amount of seconds before adding the reflector servers as potential peers to download from in case dht"
|
||||
"peers are not found or are slow", 2.0
|
||||
@@ -594,11 +671,16 @@ class Config(CLIConfig):
|
||||
)
|
||||
|
||||
# servers
|
||||
reflector_servers = Servers("Reflector re-hosting servers", [
|
||||
reflector_servers = Servers("Reflector re-hosting servers for mirroring publishes", [
|
||||
('reflector.lbry.com', 5566)
|
||||
])
|
||||
full_nodes = Servers("Full blockchain nodes", [
|
||||
('sdk.lbry.tech', 5278),
|
||||
|
||||
fixed_peers = Servers("Fixed peers to fall back to if none are found on P2P for a blob", [
|
||||
('cdn.reflector.lbry.com', 5567)
|
||||
])
|
||||
|
||||
lbryum_servers = Servers("SPV wallet servers", [
|
||||
('spv11.lbry.com', 50001),
|
||||
('spv12.lbry.com', 50001),
|
||||
('spv13.lbry.com', 50001),
|
||||
('spv14.lbry.com', 50001),
|
||||
@@ -615,19 +697,10 @@ class Config(CLIConfig):
|
||||
('lbrynet4.lbry.com', 4444) # ASIA
|
||||
])
|
||||
|
||||
comment_server = String("Comment server API URL", "https://comments.lbry.com/api")
|
||||
comment_server = String("Comment server API URL", "https://comments.lbry.com/api/v2")
|
||||
|
||||
# blockchain
|
||||
blockchain = StringChoice("Blockchain network type.", ["mainnet", "regtest", "testnet"], "mainnet")
|
||||
lbrycrd_rpc_user = String("Username for connecting to lbrycrd.", "rpcuser")
|
||||
lbrycrd_rpc_pass = String("Password for connecting to lbrycrd.", "rpcpassword")
|
||||
lbrycrd_rpc_host = String("Hostname for connecting to lbrycrd.", "localhost")
|
||||
lbrycrd_rpc_port = Integer("Port for connecting to lbrycrd.", 9245)
|
||||
lbrycrd_peer_port = Integer("Peer port for lbrycrd.", 9246)
|
||||
lbrycrd_zmq = String("ZMQ events address.")
|
||||
lbrycrd_dir = Path("Directory containing lbrycrd data.", metavar='DIR')
|
||||
search_censor_channel_ids = Strings("List of channel ids for filtering out search results.", [])
|
||||
resolve_censor_channel_ids = Strings("List of channel ids for filtering out resolve results.", [])
|
||||
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
|
||||
|
||||
# daemon
|
||||
save_files = Toggle("Save downloaded files when calling `get` by default", True)
|
||||
@@ -637,6 +710,9 @@ class Config(CLIConfig):
|
||||
previous_names=['upload_log', 'upload_log', 'share_debug_info']
|
||||
)
|
||||
track_bandwidth = Toggle("Track bandwidth usage", True)
|
||||
allowed_origin = String(
|
||||
"Allowed `Origin` header value for API request (sent by browser), use * to allow "
|
||||
"all hosts; default is to only allow API requests with no `Origin` value.", "")
|
||||
|
||||
# media server
|
||||
streaming_server = String('Host name and port to serve streaming media over range requests',
|
||||
@@ -646,8 +722,9 @@ class Config(CLIConfig):
|
||||
|
||||
coin_selection_strategy = StringChoice(
|
||||
"Strategy to use when selecting UTXOs for a transaction",
|
||||
COIN_SELECTION_STRATEGIES, "standard")
|
||||
STRATEGIES, "standard")
|
||||
|
||||
transaction_cache_size = Integer("Transaction cache size", 2 ** 17)
|
||||
save_resolved_claims = Toggle(
|
||||
"Save content claims to the database when they are resolved to keep file_list up to date, "
|
||||
"only disable this if file_x commands are not needed", True
|
||||
@@ -661,18 +738,10 @@ class Config(CLIConfig):
|
||||
def streaming_port(self):
|
||||
return int(self.streaming_server.split(':')[1])
|
||||
|
||||
@classmethod
|
||||
def with_null_dir(cls):
|
||||
return cls.with_same_dir('/dev/null')
|
||||
|
||||
@classmethod
|
||||
def with_same_dir(cls, same_dir):
|
||||
return cls(
|
||||
data_dir=same_dir,
|
||||
download_dir=same_dir,
|
||||
wallet_dir=same_dir,
|
||||
lbrycrd_dir=same_dir,
|
||||
)
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.set_default_paths()
|
||||
self.known_hubs = KnownHubsList(self)
|
||||
|
||||
def set_default_paths(self):
|
||||
if 'darwin' in sys.platform.lower():
|
||||
@@ -684,76 +753,62 @@ class Config(CLIConfig):
|
||||
else:
|
||||
return
|
||||
cls = type(self)
|
||||
cls.data_dir.default, cls.wallet_dir.default,\
|
||||
cls.blob_dir.default, cls.download_dir.default = get_directories()
|
||||
old_settings_file = os.path.join(self.data_dir, 'daemon_settings.yml')
|
||||
if os.path.exists(old_settings_file):
|
||||
cls.config.default = old_settings_file
|
||||
else:
|
||||
cls.config.default = os.path.join(self.data_dir, 'settings.yml')
|
||||
if self.data_dir != cls.data_dir.default:
|
||||
cls.blob_dir.default = os.path.join(self.data_dir, 'blobs')
|
||||
cls.wallet_dir.default = os.path.join(self.data_dir, 'wallets')
|
||||
cls.data_dir.default, cls.wallet_dir.default, cls.download_dir.default = get_directories()
|
||||
cls.config.default = os.path.join(
|
||||
self.data_dir, 'daemon_settings.yml'
|
||||
)
|
||||
|
||||
@property
|
||||
def log_file_path(self):
|
||||
return os.path.join(self.data_dir, 'daemon.log')
|
||||
|
||||
@property
|
||||
def db_url_or_default(self):
|
||||
if self.db_url:
|
||||
return self.db_url
|
||||
return 'sqlite:///'+os.path.join(self.data_dir, f'{self.blockchain}.db')
|
||||
return os.path.join(self.data_dir, 'lbrynet.log')
|
||||
|
||||
|
||||
def get_windows_directories() -> Tuple[str, str, str, str]:
|
||||
# very old
|
||||
data_dir = user_data_dir('lbrynet', roaming=True)
|
||||
blob_dir = os.path.join(data_dir, 'blobfiles')
|
||||
wallet_dir = os.path.join(user_data_dir('lbryum', roaming=True), 'wallets')
|
||||
if os.path.isdir(blob_dir) or os.path.isdir(wallet_dir):
|
||||
return data_dir, wallet_dir, blob_dir, user_download_dir()
|
||||
def get_windows_directories() -> Tuple[str, str, str]:
|
||||
from lbry.winpaths import get_path, FOLDERID, UserHandle, \
|
||||
PathNotFoundException # pylint: disable=import-outside-toplevel
|
||||
|
||||
try:
|
||||
download_dir = get_path(FOLDERID.Downloads, UserHandle.current)
|
||||
except PathNotFoundException:
|
||||
download_dir = os.getcwd()
|
||||
|
||||
# old
|
||||
appdata = get_path(FOLDERID.RoamingAppData, UserHandle.current)
|
||||
data_dir = os.path.join(appdata, 'lbrynet')
|
||||
lbryum_dir = os.path.join(appdata, 'lbryum')
|
||||
if os.path.isdir(data_dir) or os.path.isdir(lbryum_dir):
|
||||
return data_dir, lbryum_dir, download_dir
|
||||
|
||||
# new
|
||||
data_dir = user_data_dir('lbrynet', 'lbry')
|
||||
blob_dir = os.path.join(data_dir, 'blobfiles')
|
||||
wallet_dir = os.path.join(user_data_dir('lbryum', 'lbry'), 'wallets')
|
||||
if os.path.isdir(blob_dir) and os.path.isdir(wallet_dir):
|
||||
return data_dir, wallet_dir, blob_dir, user_download_dir()
|
||||
# new
|
||||
return get_universal_directories()
|
||||
lbryum_dir = user_data_dir('lbryum', 'lbry')
|
||||
return data_dir, lbryum_dir, download_dir
|
||||
|
||||
|
||||
def get_darwin_directories() -> Tuple[str, str, str, str]:
|
||||
def get_darwin_directories() -> Tuple[str, str, str]:
|
||||
data_dir = user_data_dir('LBRY')
|
||||
blob_dir = os.path.join(data_dir, 'blobfiles')
|
||||
wallet_dir = os.path.expanduser('~/.lbryum/wallets')
|
||||
if os.path.isdir(blob_dir) or os.path.isdir(wallet_dir):
|
||||
return data_dir, wallet_dir, blob_dir, user_download_dir()
|
||||
return get_universal_directories()
|
||||
lbryum_dir = os.path.expanduser('~/.lbryum')
|
||||
download_dir = os.path.expanduser('~/Downloads')
|
||||
return data_dir, lbryum_dir, download_dir
|
||||
|
||||
|
||||
def get_linux_directories() -> Tuple[str, str, str, str]:
|
||||
# very old
|
||||
data_dir = os.path.expanduser('~/.lbrynet')
|
||||
blob_dir = os.path.join(data_dir, 'blobfiles')
|
||||
wallet_dir = os.path.join(os.path.expanduser('~/.lbryum'), 'wallets')
|
||||
if os.path.isdir(blob_dir) or os.path.isdir(wallet_dir):
|
||||
return data_dir, wallet_dir, blob_dir, user_download_dir()
|
||||
def get_linux_directories() -> Tuple[str, str, str]:
|
||||
try:
|
||||
with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg:
|
||||
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read())
|
||||
if down_dir:
|
||||
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir.group(1))
|
||||
download_dir = re.sub('\"', '', down_dir)
|
||||
except OSError:
|
||||
download_dir = os.getenv('XDG_DOWNLOAD_DIR')
|
||||
if not download_dir:
|
||||
download_dir = os.path.expanduser('~/Downloads')
|
||||
|
||||
# old
|
||||
data_dir = user_data_dir('lbry/lbrynet')
|
||||
blob_dir = os.path.join(data_dir, 'blobfiles')
|
||||
wallet_dir = user_data_dir('lbry/lbryum/wallets')
|
||||
if os.path.isdir(blob_dir) or os.path.isdir(wallet_dir):
|
||||
return data_dir, wallet_dir, blob_dir, user_download_dir()
|
||||
data_dir = os.path.expanduser('~/.lbrynet')
|
||||
lbryum_dir = os.path.expanduser('~/.lbryum')
|
||||
if os.path.isdir(data_dir) or os.path.isdir(lbryum_dir):
|
||||
return data_dir, lbryum_dir, download_dir
|
||||
|
||||
# new
|
||||
return get_universal_directories()
|
||||
|
||||
|
||||
def get_universal_directories() -> Tuple[str, str, str, str]:
|
||||
lbrynet_dir = user_data_dir('lbrynet', 'LBRY')
|
||||
return (
|
||||
lbrynet_dir,
|
||||
os.path.join(lbrynet_dir, 'wallets'),
|
||||
os.path.join(lbrynet_dir, 'blobs'),
|
||||
user_download_dir()
|
||||
)
|
||||
return user_data_dir('lbry/lbrynet'), user_data_dir('lbry/lbryum'), download_dir
|
||||
-496
@@ -1,496 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import itertools
|
||||
import logging
|
||||
from typing import Dict, Any, Type
|
||||
from tempfile import TemporaryFile
|
||||
|
||||
from tqdm.std import tqdm, Bar
|
||||
from tqdm.utils import FormatReplace, _unicode, disp_len, disp_trim, _is_ascii
|
||||
|
||||
from lbry import __version__
|
||||
from lbry.service.base import Service
|
||||
from lbry.service.full_node import FullNode
|
||||
from lbry.service.light_client import LightClient
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RedirectOutput:
|
||||
|
||||
silence_lines = [
|
||||
b'libprotobuf ERROR google/protobuf/wire_format_lite.cc:626',
|
||||
]
|
||||
|
||||
def __init__(self, stream_type: str):
|
||||
assert stream_type in ('stderr', 'stdout')
|
||||
self.stream_type = stream_type
|
||||
self.stream_no = getattr(sys, stream_type).fileno()
|
||||
self.last_flush = time.time()
|
||||
self.last_read = 0
|
||||
self.backup = None
|
||||
self.file = None
|
||||
|
||||
def __enter__(self):
|
||||
self.backup = os.dup(self.stream_no)
|
||||
setattr(sys, self.stream_type, os.fdopen(self.backup, 'w'))
|
||||
self.file = TemporaryFile()
|
||||
self.backup = os.dup2(self.file.fileno(), self.stream_no)
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.file.close()
|
||||
os.dup2(self.backup, self.stream_no)
|
||||
os.close(self.backup)
|
||||
setattr(sys, self.stream_type, os.fdopen(self.stream_no, 'w'))
|
||||
|
||||
def capture(self):
|
||||
self.__enter__()
|
||||
|
||||
def release(self):
|
||||
self.__exit__(None, None, None)
|
||||
|
||||
def flush(self, writer, force=False):
|
||||
if not force and (time.time() - self.last_flush) < 5:
|
||||
return
|
||||
self.file.seek(self.last_read)
|
||||
for line in self.file.readlines():
|
||||
silence = False
|
||||
for bad_line in self.silence_lines:
|
||||
if bad_line in line:
|
||||
silence = True
|
||||
break
|
||||
if not silence:
|
||||
writer(line.decode().rstrip())
|
||||
self.last_read = self.file.tell()
|
||||
self.last_flush = time.time()
|
||||
|
||||
|
||||
class Console:
|
||||
|
||||
def __init__(self, service: Service):
|
||||
self.service = service
|
||||
|
||||
def starting(self):
|
||||
pass
|
||||
|
||||
def stopping(self):
|
||||
pass
|
||||
|
||||
|
||||
class Basic(Console):
|
||||
|
||||
def __init__(self, service: Service):
|
||||
super().__init__(service)
|
||||
self.service.sync.on_progress.listen(self.on_sync_progress)
|
||||
self.tasks = {}
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s")
|
||||
|
||||
def starting(self):
|
||||
conf = self.service.conf
|
||||
s = [f'LBRY v{__version__}']
|
||||
if isinstance(self.service, FullNode):
|
||||
s.append('Full Node')
|
||||
elif isinstance(self.service, LightClient):
|
||||
s.append('Light Client')
|
||||
if conf.workers == -1:
|
||||
s.append('Threads Only')
|
||||
else:
|
||||
workers = os.cpu_count() if conf.workers == 0 else conf.workers
|
||||
s.append(f'{workers} Worker' if workers == 1 else f'{workers} Workers')
|
||||
s.append(f'({os.cpu_count()} CPUs available)')
|
||||
log.info(' '.join(s))
|
||||
|
||||
def stopping(self):
|
||||
log.info('exiting')
|
||||
|
||||
@staticmethod
|
||||
def maybe_log_progress(event, done, total, last):
|
||||
if done == 0:
|
||||
log.info("%s 0%%", event)
|
||||
return 0
|
||||
elif done == total:
|
||||
log.info("%s 100%%", event)
|
||||
return 1
|
||||
else:
|
||||
percent = done/total
|
||||
if percent >= 0.25 > last:
|
||||
log.info("%s 25%%", event)
|
||||
return 0.25
|
||||
elif percent >= 0.50 > last:
|
||||
log.info("%s 50%%", event)
|
||||
return 0.50
|
||||
elif percent >= 0.75 > last:
|
||||
log.info("%s 75%%", event)
|
||||
return 0.75
|
||||
return last
|
||||
|
||||
def on_sync_progress(self, event):
|
||||
e, data = event["event"], event["data"]
|
||||
name, current, total, last = e, data['done'][0], 0, 0
|
||||
if not e.endswith("init") and not e.endswith("main") and not e.endswith("indexes"):
|
||||
name = f"{e}#{data['id']}"
|
||||
if "total" in data:
|
||||
total, last = self.tasks[name] = (data["total"][0], last)
|
||||
elif name in self.tasks:
|
||||
total, last = self.tasks[name]
|
||||
elif total == 0:
|
||||
return
|
||||
progress_status = (total, self.maybe_log_progress(name, current, total, last))
|
||||
if progress_status[1] == 1:
|
||||
del self.tasks[name]
|
||||
else:
|
||||
self.tasks[name] = progress_status
|
||||
|
||||
|
||||
class Bar2(Bar):
|
||||
|
||||
def __init__(self, frac, default_len=10, charset=None):
|
||||
super().__init__(frac[0], default_len, charset)
|
||||
self.frac2 = frac[1]
|
||||
|
||||
def __format__(self, format_spec):
|
||||
width = self.default_len
|
||||
row1 = (1,)*int(self.frac2 * width * 2)
|
||||
row2 = (2,)*int(self.frac * width * 2)
|
||||
fill = []
|
||||
for one, two, _ in itertools.zip_longest(row1, row2, range(width*2)):
|
||||
fill.append((one or 0)+(two or 0))
|
||||
bar = []
|
||||
for i in range(0, width*2, 2):
|
||||
if fill[i] == 1:
|
||||
if fill[i+1] == 1:
|
||||
bar.append('▀')
|
||||
else:
|
||||
bar.append('▘')
|
||||
elif fill[i] == 2:
|
||||
if fill[i+1] == 2:
|
||||
bar.append('▄')
|
||||
else:
|
||||
bar.append('▖')
|
||||
elif fill[i] == 3:
|
||||
if fill[i+1] == 1:
|
||||
bar.append('▛')
|
||||
elif fill[i+1] == 2:
|
||||
bar.append('▙')
|
||||
elif fill[i+1] == 3:
|
||||
bar.append('█')
|
||||
else:
|
||||
bar.append('▌')
|
||||
else:
|
||||
bar.append(' ')
|
||||
return ''.join(bar)
|
||||
|
||||
|
||||
class tqdm2(tqdm): # pylint: disable=invalid-name
|
||||
|
||||
def __init__(self, initial=(0, 0), unit=('it', 'it'), total=(None, None), **kwargs):
|
||||
self.n2 = self.last_print_n2 = initial[1] # pylint: disable=invalid-name
|
||||
self.unit2 = unit[1]
|
||||
self.total2 = total[1]
|
||||
super().__init__(initial=initial[0], unit=unit[0], total=total[0], **kwargs)
|
||||
|
||||
@property
|
||||
def format_dict(self):
|
||||
d = super().format_dict
|
||||
d.update({
|
||||
'n2': self.n2,
|
||||
'unit2': self.unit2,
|
||||
'total2': self.total2,
|
||||
})
|
||||
return d
|
||||
|
||||
def update(self, n=(1, 1)):
|
||||
if self.disable:
|
||||
return
|
||||
last_last_print_t = self.last_print_t
|
||||
self.n2 += n[1]
|
||||
super().update(n[0])
|
||||
if last_last_print_t != self.last_print_t:
|
||||
self.last_print_n2 = self.n2
|
||||
|
||||
@staticmethod
|
||||
def format_meter(
|
||||
n, total, elapsed, ncols=None, prefix='', ascii=False, # pylint: disable=redefined-builtin
|
||||
unit='it', unit_scale=False, rate=None, bar_format=None,
|
||||
postfix=None, unit_divisor=1000, initial=0, **extra_kwargs
|
||||
):
|
||||
|
||||
# sanity check: total
|
||||
if total and n >= (total + 0.5): # allow float imprecision (#849)
|
||||
total = None
|
||||
|
||||
# apply custom scale if necessary
|
||||
if unit_scale and unit_scale not in (True, 1):
|
||||
if total:
|
||||
total *= unit_scale
|
||||
n *= unit_scale
|
||||
if rate:
|
||||
rate *= unit_scale # by default rate = 1 / self.avg_time
|
||||
unit_scale = False
|
||||
|
||||
elapsed_str = tqdm.format_interval(elapsed)
|
||||
|
||||
# if unspecified, attempt to use rate = average speed
|
||||
# (we allow manual override since predicting time is an arcane art)
|
||||
if rate is None and elapsed:
|
||||
rate = n / elapsed
|
||||
inv_rate = 1 / rate if rate else None
|
||||
format_sizeof = tqdm.format_sizeof
|
||||
rate_noinv_fmt = ((format_sizeof(rate) if unit_scale else
|
||||
'{0:5.2f}'.format(rate))
|
||||
if rate else '?') + unit + '/s'
|
||||
rate_inv_fmt = ((format_sizeof(inv_rate) if unit_scale else
|
||||
'{0:5.2f}'.format(inv_rate))
|
||||
if inv_rate else '?') + 's/' + unit
|
||||
rate_fmt = rate_inv_fmt if inv_rate and inv_rate > 1 else rate_noinv_fmt
|
||||
|
||||
if unit_scale:
|
||||
n_fmt = format_sizeof(n, divisor=unit_divisor)
|
||||
total_fmt = format_sizeof(total, divisor=unit_divisor) \
|
||||
if total is not None else '?'
|
||||
else:
|
||||
n_fmt = str(n)
|
||||
total_fmt = str(total) if total is not None else '?'
|
||||
|
||||
try:
|
||||
postfix = ', ' + postfix if postfix else ''
|
||||
except TypeError:
|
||||
pass
|
||||
|
||||
remaining = (total - n) / rate if rate and total else 0
|
||||
remaining_str = tqdm.format_interval(remaining) if rate else '?'
|
||||
|
||||
# format the stats displayed to the left and right sides of the bar
|
||||
if prefix:
|
||||
# old prefix setup work around
|
||||
bool_prefix_colon_already = (prefix[-2:] == ": ")
|
||||
l_bar = prefix if bool_prefix_colon_already else prefix + ": "
|
||||
else:
|
||||
l_bar = ''
|
||||
|
||||
r_bar = '| {0}/{1} [{2}<{3}, {4}{5}]'.format(
|
||||
n_fmt, total_fmt, elapsed_str, remaining_str, rate_fmt, postfix)
|
||||
|
||||
# Custom bar formatting
|
||||
# Populate a dict with all available progress indicators
|
||||
format_dict = dict(
|
||||
# slight extension of self.format_dict
|
||||
n=n, n_fmt=n_fmt, total=total, total_fmt=total_fmt,
|
||||
elapsed=elapsed_str, elapsed_s=elapsed,
|
||||
ncols=ncols, desc=prefix or '', unit=unit,
|
||||
rate=inv_rate if inv_rate and inv_rate > 1 else rate,
|
||||
rate_fmt=rate_fmt, rate_noinv=rate,
|
||||
rate_noinv_fmt=rate_noinv_fmt, rate_inv=inv_rate,
|
||||
rate_inv_fmt=rate_inv_fmt,
|
||||
postfix=postfix, unit_divisor=unit_divisor,
|
||||
# plus more useful definitions
|
||||
remaining=remaining_str, remaining_s=remaining,
|
||||
l_bar=l_bar, r_bar=r_bar,
|
||||
**extra_kwargs)
|
||||
|
||||
# total is known: we can predict some stats
|
||||
if total:
|
||||
n2, total2 = extra_kwargs['n2'], extra_kwargs['total2'] # pylint: disable=invalid-name
|
||||
|
||||
# fractional and percentage progress
|
||||
frac = n / total
|
||||
frac2 = n2 / total2
|
||||
percentage = frac * 100
|
||||
|
||||
l_bar += '{0:3.0f}%|'.format(percentage)
|
||||
|
||||
if ncols == 0:
|
||||
return l_bar[:-1] + r_bar[1:]
|
||||
|
||||
format_dict.update(l_bar=l_bar)
|
||||
if bar_format:
|
||||
format_dict.update(percentage=percentage)
|
||||
|
||||
# auto-remove colon for empty `desc`
|
||||
if not prefix:
|
||||
bar_format = bar_format.replace("{desc}: ", '')
|
||||
else:
|
||||
bar_format = "{l_bar}{bar}{r_bar}"
|
||||
|
||||
full_bar = FormatReplace()
|
||||
try:
|
||||
nobar = bar_format.format(bar=full_bar, **format_dict)
|
||||
except UnicodeEncodeError:
|
||||
bar_format = _unicode(bar_format)
|
||||
nobar = bar_format.format(bar=full_bar, **format_dict)
|
||||
if not full_bar.format_called:
|
||||
# no {bar}, we can just format and return
|
||||
return nobar
|
||||
|
||||
# Formatting progress bar space available for bar's display
|
||||
full_bar = Bar2(
|
||||
(frac, frac2),
|
||||
max(1, ncols - disp_len(nobar))
|
||||
if ncols else 10,
|
||||
charset=Bar2.ASCII if ascii is True else ascii or Bar2.UTF)
|
||||
if not _is_ascii(full_bar.charset) and _is_ascii(bar_format):
|
||||
bar_format = _unicode(bar_format)
|
||||
res = bar_format.format(bar=full_bar, **format_dict)
|
||||
return disp_trim(res, ncols) if ncols else res
|
||||
|
||||
elif bar_format:
|
||||
# user-specified bar_format but no total
|
||||
l_bar += '|'
|
||||
format_dict.update(l_bar=l_bar, percentage=0)
|
||||
full_bar = FormatReplace()
|
||||
nobar = bar_format.format(bar=full_bar, **format_dict)
|
||||
if not full_bar.format_called:
|
||||
return nobar
|
||||
full_bar = Bar2(
|
||||
(0, 0),
|
||||
max(1, ncols - disp_len(nobar))
|
||||
if ncols else 10,
|
||||
charset=Bar2.BLANK)
|
||||
res = bar_format.format(bar=full_bar, **format_dict)
|
||||
return disp_trim(res, ncols) if ncols else res
|
||||
else:
|
||||
# no total: no progressbar, ETA, just progress stats
|
||||
return ((prefix + ": ") if prefix else '') + \
|
||||
'{0}{1} [{2}, {3}{4}]'.format(
|
||||
n_fmt, unit, elapsed_str, rate_fmt, postfix)
|
||||
|
||||
|
||||
class Advanced(Basic):
|
||||
|
||||
FORMAT = '{l_bar}{bar}| {n_fmt:>8}/{total_fmt:>8} [{elapsed:>7}<{remaining:>8}, {rate_fmt:>17}]'
|
||||
|
||||
def __init__(self, service: Service):
|
||||
super().__init__(service)
|
||||
self.bars: Dict[Any, tqdm] = {}
|
||||
self.stderr = RedirectOutput('stderr')
|
||||
|
||||
def starting(self):
|
||||
self.stderr.capture()
|
||||
super().starting()
|
||||
|
||||
def stopping(self):
|
||||
for bar in self.bars.values():
|
||||
bar.close()
|
||||
super().stopping()
|
||||
#self.stderr.flush(self.bars['read'].write, True)
|
||||
#self.stderr.release()
|
||||
|
||||
def get_or_create_bar(self, name, desc, units, totals, leave=False, bar_format=None, postfix=None, position=None):
|
||||
bar = self.bars.get(name)
|
||||
if bar is None:
|
||||
if len(units) == 2:
|
||||
bar = self.bars[name] = tqdm2(
|
||||
desc=desc, unit=units, total=totals,
|
||||
bar_format=bar_format or self.FORMAT, leave=leave,
|
||||
postfix=postfix, position=position
|
||||
)
|
||||
else:
|
||||
bar = self.bars[name] = tqdm(
|
||||
desc=desc, unit=units[0], total=totals[0],
|
||||
bar_format=bar_format or self.FORMAT, leave=leave,
|
||||
postfix=postfix, position=position
|
||||
)
|
||||
return bar
|
||||
|
||||
def sync_init(self, name, d):
|
||||
bar_name = f"{name}#{d['id']}"
|
||||
bar = self.bars.get(bar_name)
|
||||
if bar is None:
|
||||
label = d.get('label', name[-11:])
|
||||
self.get_or_create_bar(bar_name, label, d['units'], d['total'], True)
|
||||
else:
|
||||
if d['done'][0] != -1:
|
||||
bar.update(d['done'][0] - bar.n)
|
||||
if d['done'][0] == -1 or d['done'][0] == bar.total:
|
||||
bar.close()
|
||||
self.bars.pop(bar_name)
|
||||
|
||||
def sync_main(self, name, d):
|
||||
bar = self.bars.get(name)
|
||||
if bar is None:
|
||||
label = d.get('label', name[-11:])
|
||||
self.get_or_create_bar(name, label, d['units'], d['total'], True)
|
||||
#self.last_stats = f"{d['txs']:,d} txs, {d['claims']:,d} claims and {d['supports']:,d} supports"
|
||||
#self.get_or_create_bar("read", "├─ blocks read", "blocks", d['blocks'], True)
|
||||
#self.get_or_create_bar("save", "└─┬ txs saved", "txs", d['txs'], True)
|
||||
else:
|
||||
if d['done'] == (-1,)*len(d['done']):
|
||||
base_name = name[:name.rindex('.')]
|
||||
for child_name, child_bar in self.bars.items():
|
||||
if child_name.startswith(base_name):
|
||||
child_bar.close()
|
||||
bar.close()
|
||||
self.bars.pop(name)
|
||||
else:
|
||||
if len(d['done']) == 2:
|
||||
bar.update((d['done'][0]-bar.n, d['done'][1]-bar.n2))
|
||||
else:
|
||||
bar.update(d['done'][0]-bar.n)
|
||||
|
||||
def sync_task(self, name, d):
|
||||
bar_name = f"{name}#{d['id']}"
|
||||
bar = self.bars.get(bar_name)
|
||||
if bar is None:
|
||||
#assert d['done'][0] == 0
|
||||
label = d.get('label', name[-11:])
|
||||
self.get_or_create_bar(
|
||||
f"{name}#{d['id']}", label, d['units'], d['total'],
|
||||
name.split('.')[-1] not in ('insert', 'update', 'file')
|
||||
)
|
||||
else:
|
||||
if d['done'][0] != -1:
|
||||
main_bar_name = f"{name[:name.rindex('.')]}.main"
|
||||
if len(d['done']) > 1:
|
||||
diff = tuple(a-b for a, b in zip(d['done'], (bar.n, bar.n2)))
|
||||
else:
|
||||
diff = d['done'][0] - bar.n
|
||||
if main_bar_name != name:
|
||||
main_bar = self.bars.get(main_bar_name)
|
||||
if main_bar and main_bar.unit == bar.unit:
|
||||
main_bar.update(diff)
|
||||
bar.update(diff)
|
||||
if d['done'][0] == -1 or d['done'][0] == bar.total:
|
||||
bar.close()
|
||||
self.bars.pop(bar_name)
|
||||
|
||||
def update_other_bars(self, e, d):
|
||||
if d['total'] == 0:
|
||||
return
|
||||
bar = self.bars.get(e)
|
||||
if not bar:
|
||||
name = (
|
||||
' '.join(e.split('.')[-2:])
|
||||
.replace('support', 'suprt')
|
||||
.replace('channels', 'chanls')
|
||||
.replace('signatures', 'sigs')
|
||||
)
|
||||
bar = self.get_or_create_bar(e, f"├─ {name:>12}", d['unit'], d['total'], True)
|
||||
diff = d['step']-bar.n
|
||||
bar.update(diff)
|
||||
#if d['step'] == d['total']:
|
||||
#bar.close()
|
||||
|
||||
def on_sync_progress(self, event):
|
||||
e, d = event['event'], event.get('data', {})
|
||||
if e.endswith(".init"):
|
||||
self.sync_init(e, d)
|
||||
elif e.endswith(".main"):
|
||||
self.sync_main(e, d)
|
||||
else:
|
||||
self.sync_task(e, d)
|
||||
|
||||
# if e.endswith("sync.start"):
|
||||
# self.sync_start(d)
|
||||
# self.stderr.flush(self.bars['read'].write)
|
||||
# elif e.endswith("sync.complete"):
|
||||
# self.stderr.flush(self.bars['read'].write, True)
|
||||
# self.sync_complete()
|
||||
# else:
|
||||
# self.stderr.flush(self.bars['read'].write)
|
||||
# self.update_progress(e, d)
|
||||
|
||||
|
||||
def console_class_from_name(name) -> Type[Console]:
|
||||
return {'basic': Basic, 'advanced': Advanced}.get(name, Console)
|
||||
@@ -1,8 +1,2 @@
|
||||
DEFAULT_PAGE_SIZE = 20
|
||||
|
||||
NULL_HASH32 = b'\x00'*32
|
||||
|
||||
CENT = 1000000
|
||||
COIN = 100*CENT
|
||||
|
||||
INVALIDATED_SIGNATURE_GRACE_PERIOD = 50
|
||||
+4
-4
@@ -36,12 +36,12 @@ def hash160(x):
|
||||
return ripemd160(sha256(x))
|
||||
|
||||
|
||||
def hash_to_hex_str(x: bytes) -> str:
|
||||
def hash_to_hex_str(x):
|
||||
""" Convert a big-endian binary hash to displayed hex string.
|
||||
Display form of a binary hash is reversed and converted to hex. """
|
||||
return hexlify(x[::-1])
|
||||
return hexlify(reversed(x))
|
||||
|
||||
|
||||
def hex_str_to_hash(x: str) -> bytes:
|
||||
def hex_str_to_hash(x):
|
||||
""" Convert a displayed hex string to a binary hash. """
|
||||
return unhexlify(x)[::-1]
|
||||
return reversed(unhexlify(x))
|
||||
@@ -1,5 +0,0 @@
|
||||
from .database import Database, Result
|
||||
from .constants import (
|
||||
TXO_TYPES, SPENDABLE_TYPE_CODES,
|
||||
CLAIM_TYPE_CODES, CLAIM_TYPE_NAMES
|
||||
)
|
||||
@@ -1,73 +0,0 @@
|
||||
MAX_QUERY_VARIABLES = 900
|
||||
|
||||
TXO_TYPES = {
|
||||
"other": 0,
|
||||
"stream": 1,
|
||||
"channel": 2,
|
||||
"support": 3,
|
||||
"purchase": 4,
|
||||
"collection": 5,
|
||||
"repost": 6,
|
||||
}
|
||||
|
||||
CLAIM_TYPE_NAMES = [
|
||||
'stream',
|
||||
'channel',
|
||||
'collection',
|
||||
'repost',
|
||||
]
|
||||
|
||||
CONTENT_TYPE_NAMES = [
|
||||
name for name in CLAIM_TYPE_NAMES if name != "channel"
|
||||
]
|
||||
|
||||
CLAIM_TYPE_CODES = [
|
||||
TXO_TYPES[name] for name in CLAIM_TYPE_NAMES
|
||||
]
|
||||
|
||||
CONTENT_TYPE_CODES = [
|
||||
TXO_TYPES[name] for name in CONTENT_TYPE_NAMES
|
||||
]
|
||||
|
||||
SPENDABLE_TYPE_CODES = [
|
||||
TXO_TYPES['other'],
|
||||
TXO_TYPES['purchase']
|
||||
]
|
||||
|
||||
STREAM_TYPES = {
|
||||
'video': 1,
|
||||
'audio': 2,
|
||||
'image': 3,
|
||||
'document': 4,
|
||||
'binary': 5,
|
||||
'model': 6
|
||||
}
|
||||
|
||||
MATURE_TAGS = (
|
||||
'nsfw', 'porn', 'xxx', 'mature', 'adult', 'sex'
|
||||
)
|
||||
|
||||
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
|
||||
|
||||
SEARCH_INTEGER_PARAMS = {
|
||||
'height', 'creation_height', 'activation_height', 'expiration_height',
|
||||
'timestamp', 'creation_timestamp', 'duration', 'release_time', 'fee_amount',
|
||||
'tx_position', 'channel_join', 'reposted',
|
||||
'amount', 'staked_amount', 'support_amount',
|
||||
'trend_group', 'trend_mixed', 'trend_local', 'trend_global',
|
||||
}
|
||||
|
||||
SEARCH_PARAMS = {
|
||||
'name', 'text', 'claim_id', 'claim_ids', 'txid', 'nout', 'channel', 'channel_ids', 'not_channel_ids',
|
||||
'public_key_id', 'claim_type', 'stream_types', 'media_types', 'fee_currency',
|
||||
'has_channel_signature', 'signature_valid',
|
||||
'any_tags', 'all_tags', 'not_tags', 'reposted_claim_id',
|
||||
'any_locations', 'all_locations', 'not_locations',
|
||||
'any_languages', 'all_languages', 'not_languages',
|
||||
'is_controlling', 'limit', 'offset', 'order_by',
|
||||
'no_totals',
|
||||
} | SEARCH_INTEGER_PARAMS
|
||||
|
||||
SEARCH_ORDER_FIELDS = {
|
||||
'name', 'claim_hash', 'claim_id'
|
||||
} | SEARCH_INTEGER_PARAMS
|
||||
@@ -1,402 +0,0 @@
|
||||
import os
|
||||
import asyncio
|
||||
import tempfile
|
||||
import multiprocessing as mp
|
||||
from binascii import hexlify
|
||||
from typing import List, Optional, Iterable, Iterator, TypeVar, Generic, TYPE_CHECKING, Dict, Tuple
|
||||
from concurrent.futures import Executor, ThreadPoolExecutor, ProcessPoolExecutor
|
||||
from functools import partial
|
||||
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
from lbry.event import EventController
|
||||
from lbry.crypto.bip32 import PubKey
|
||||
from lbry.blockchain.transaction import Transaction, Output
|
||||
from .constants import TXO_TYPES, CLAIM_TYPE_CODES
|
||||
from .query_context import initialize, uninitialize, ProgressPublisher
|
||||
from . import queries as q
|
||||
from . import sync
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from lbry.blockchain.ledger import Ledger
|
||||
|
||||
|
||||
def clean_wallet_account_ids(constraints):
|
||||
wallet = constraints.pop('wallet', None)
|
||||
account = constraints.pop('account', None)
|
||||
accounts = constraints.pop('accounts', [])
|
||||
if account and not accounts:
|
||||
accounts = [account]
|
||||
if wallet:
|
||||
constraints['wallet_account_ids'] = [account.id for account in wallet.accounts]
|
||||
if not accounts:
|
||||
accounts = wallet.accounts
|
||||
if accounts:
|
||||
constraints['account_ids'] = [account.id for account in accounts]
|
||||
|
||||
|
||||
async def add_channel_keys_to_txo_results(accounts: List, txos: Iterable[Output]):
|
||||
sub_channels = set()
|
||||
for txo in txos:
|
||||
if txo.is_claim and txo.claim.is_channel:
|
||||
for account in accounts:
|
||||
private_key = await account.get_channel_private_key(
|
||||
txo.claim.channel.public_key_bytes
|
||||
)
|
||||
if private_key:
|
||||
txo.private_key = private_key
|
||||
break
|
||||
if txo.channel is not None:
|
||||
sub_channels.add(txo.channel)
|
||||
if sub_channels:
|
||||
await add_channel_keys_to_txo_results(accounts, sub_channels)
|
||||
|
||||
ResultType = TypeVar('ResultType')
|
||||
|
||||
|
||||
class Result(Generic[ResultType]):
|
||||
|
||||
__slots__ = 'rows', 'total', 'censor'
|
||||
|
||||
def __init__(self, rows: List[ResultType], total, censor=None):
|
||||
self.rows = rows
|
||||
self.total = total
|
||||
self.censor = censor
|
||||
|
||||
def __getitem__(self, item: int) -> ResultType:
|
||||
return self.rows[item]
|
||||
|
||||
def __iter__(self) -> Iterator[ResultType]:
|
||||
return iter(self.rows)
|
||||
|
||||
def __len__(self):
|
||||
return len(self.rows)
|
||||
|
||||
def __repr__(self):
|
||||
return repr(self.rows)
|
||||
|
||||
|
||||
class Database:
|
||||
|
||||
def __init__(self, ledger: 'Ledger'):
|
||||
self.url = ledger.conf.db_url_or_default
|
||||
self.ledger = ledger
|
||||
self.workers = self._normalize_worker_processes(ledger.conf.workers)
|
||||
self.executor: Optional[Executor] = None
|
||||
self.message_queue = mp.Queue()
|
||||
self.stop_event = mp.Event()
|
||||
self._on_progress_controller = EventController()
|
||||
self.on_progress = self._on_progress_controller.stream
|
||||
self.progress_publisher = ProgressPublisher(
|
||||
self.message_queue, self._on_progress_controller
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_worker_processes(workers):
|
||||
if workers == 0:
|
||||
return os.cpu_count()
|
||||
elif workers > 0:
|
||||
return workers
|
||||
return 1
|
||||
|
||||
@classmethod
|
||||
def temp_from_url_regtest(cls, db_url, lbrycrd_config=None):
|
||||
from lbry import Config, RegTestLedger # pylint: disable=import-outside-toplevel
|
||||
directory = tempfile.mkdtemp()
|
||||
if lbrycrd_config:
|
||||
conf = lbrycrd_config
|
||||
conf.data_dir = directory
|
||||
conf.download_dir = directory
|
||||
conf.wallet_dir = directory
|
||||
else:
|
||||
conf = Config.with_same_dir(directory)
|
||||
conf.set(blockchain="regtest", db_url=db_url)
|
||||
ledger = RegTestLedger(conf)
|
||||
return cls(ledger)
|
||||
|
||||
@classmethod
|
||||
def temp_sqlite_regtest(cls, lbrycrd_config=None):
|
||||
from lbry import Config, RegTestLedger # pylint: disable=import-outside-toplevel
|
||||
directory = tempfile.mkdtemp()
|
||||
if lbrycrd_config:
|
||||
conf = lbrycrd_config
|
||||
conf.data_dir = directory
|
||||
conf.download_dir = directory
|
||||
conf.wallet_dir = directory
|
||||
else:
|
||||
conf = Config.with_same_dir(directory).set(blockchain="regtest")
|
||||
ledger = RegTestLedger(conf)
|
||||
return cls(ledger)
|
||||
|
||||
@classmethod
|
||||
def temp_sqlite(cls):
|
||||
from lbry import Config, Ledger # pylint: disable=import-outside-toplevel
|
||||
conf = Config.with_same_dir(tempfile.mkdtemp())
|
||||
return cls(Ledger(conf))
|
||||
|
||||
@classmethod
|
||||
def from_url(cls, db_url):
|
||||
from lbry import Config, Ledger # pylint: disable=import-outside-toplevel
|
||||
return cls(Ledger(Config.with_null_dir().set(db_url=db_url)))
|
||||
|
||||
@classmethod
|
||||
def in_memory(cls):
|
||||
return cls.from_url('sqlite:///:memory:')
|
||||
|
||||
def sync_create(self, name):
|
||||
engine = create_engine(self.url)
|
||||
db = engine.connect()
|
||||
db.execute(text("COMMIT"))
|
||||
db.execute(text(f"CREATE DATABASE {name}"))
|
||||
|
||||
async def create(self, name):
|
||||
return await asyncio.get_running_loop().run_in_executor(None, self.sync_create, name)
|
||||
|
||||
def sync_drop(self, name):
|
||||
engine = create_engine(self.url)
|
||||
db = engine.connect()
|
||||
db.execute(text("COMMIT"))
|
||||
db.execute(text(f"DROP DATABASE IF EXISTS {name}"))
|
||||
|
||||
async def drop(self, name):
|
||||
return await asyncio.get_running_loop().run_in_executor(None, self.sync_drop, name)
|
||||
|
||||
async def open(self):
|
||||
assert self.executor is None, "Database already open."
|
||||
self.progress_publisher.start()
|
||||
kwargs = {
|
||||
"initializer": initialize,
|
||||
"initargs": (
|
||||
self.ledger,
|
||||
self.message_queue, self.stop_event
|
||||
)
|
||||
}
|
||||
if self.workers > 1:
|
||||
self.executor = ProcessPoolExecutor(max_workers=self.workers, **kwargs)
|
||||
else:
|
||||
self.executor = ThreadPoolExecutor(max_workers=1, **kwargs)
|
||||
return await self.run(q.check_version_and_create_tables)
|
||||
|
||||
async def close(self):
|
||||
self.progress_publisher.stop()
|
||||
if self.executor is not None:
|
||||
if isinstance(self.executor, ThreadPoolExecutor):
|
||||
await self.run(uninitialize)
|
||||
self.executor.shutdown()
|
||||
self.executor = None
|
||||
# fixes "OSError: handle is closed"
|
||||
# seems to only happen when running in PyCharm
|
||||
# https://github.com/python/cpython/pull/6084#issuecomment-564585446
|
||||
# TODO: delete this in Python 3.8/3.9?
|
||||
from concurrent.futures.process import _threads_wakeups # pylint: disable=import-outside-toplevel
|
||||
_threads_wakeups.clear()
|
||||
|
||||
async def run(self, func, *args, **kwargs):
|
||||
if kwargs:
|
||||
clean_wallet_account_ids(kwargs)
|
||||
return await asyncio.get_running_loop().run_in_executor(
|
||||
self.executor, partial(func, *args, **kwargs)
|
||||
)
|
||||
|
||||
async def fetch_result(self, func, *args, **kwargs) -> Result:
|
||||
rows, total = await self.run(func, *args, **kwargs)
|
||||
return Result(rows, total)
|
||||
|
||||
async def execute(self, sql):
|
||||
return await self.run(q.execute, sql)
|
||||
|
||||
async def execute_sql_object(self, sql):
|
||||
return await self.run(q.execute_sql_object, sql)
|
||||
|
||||
async def execute_fetchall(self, sql):
|
||||
return await self.run(q.execute_fetchall, sql)
|
||||
|
||||
async def has_filters(self):
|
||||
return await self.run(q.has_filters)
|
||||
|
||||
async def has_claims(self):
|
||||
return await self.run(q.has_claims)
|
||||
|
||||
async def has_supports(self):
|
||||
return await self.run(q.has_supports)
|
||||
|
||||
async def has_wallet(self, wallet_id):
|
||||
return await self.run(q.has_wallet, wallet_id)
|
||||
|
||||
async def get_wallet(self, wallet_id: str):
|
||||
return await self.run(q.get_wallet, wallet_id)
|
||||
|
||||
async def add_wallet(self, wallet_id: str, data: str):
|
||||
return await self.run(q.add_wallet, wallet_id, data)
|
||||
|
||||
async def get_best_block_height(self) -> int:
|
||||
return await self.run(q.get_best_block_height)
|
||||
|
||||
async def process_all_things_after_sync(self):
|
||||
return await self.run(sync.process_all_things_after_sync)
|
||||
|
||||
async def get_block_headers(self, start_height: int, end_height: int = None):
|
||||
return await self.run(q.get_block_headers, start_height, end_height)
|
||||
|
||||
async def get_filters(self, start_height: int, end_height: int = None, granularity: int = 0):
|
||||
filters = []
|
||||
for row in await self.run(q.get_filters, start_height, end_height, granularity):
|
||||
record = {
|
||||
"height": row["height"],
|
||||
"filter": hexlify(row["address_filter"]).decode(),
|
||||
}
|
||||
if granularity == 0:
|
||||
record["txid"] = hexlify(row["tx_hash"][::-1]).decode()
|
||||
filters.append(record)
|
||||
return filters
|
||||
|
||||
async def get_missing_required_filters(self, height) -> Dict[int, Tuple[int, int]]:
|
||||
return await self.run(q.get_missing_required_filters, height)
|
||||
|
||||
async def get_missing_sub_filters_for_addresses(self, granularity, address_manager):
|
||||
return await self.run(q.get_missing_sub_filters_for_addresses, granularity, address_manager)
|
||||
|
||||
async def get_missing_tx_for_addresses(self, address_manager):
|
||||
return await self.run(q.get_missing_tx_for_addresses, address_manager)
|
||||
|
||||
async def insert_blocks(self, blocks):
|
||||
return await self.run(q.insert_blocks, blocks)
|
||||
|
||||
async def insert_block_filters(self, filters):
|
||||
return await self.run(q.insert_block_filters, filters)
|
||||
|
||||
async def insert_tx_filters(self, filters):
|
||||
return await self.run(q.insert_tx_filters, filters)
|
||||
|
||||
async def insert_transaction(self, block_hash, tx):
|
||||
return await self.run(q.insert_transaction, block_hash, tx)
|
||||
|
||||
async def update_address_used_times(self, addresses):
|
||||
return await self.run(q.update_address_used_times, addresses)
|
||||
|
||||
async def reserve_outputs(self, txos, is_reserved=True):
|
||||
txo_hashes = [txo.hash for txo in txos]
|
||||
if txo_hashes:
|
||||
return await self.run(
|
||||
q.reserve_outputs, txo_hashes, is_reserved
|
||||
)
|
||||
|
||||
async def release_outputs(self, txos):
|
||||
return await self.reserve_outputs(txos, is_reserved=False)
|
||||
|
||||
async def release_tx(self, tx):
|
||||
return await self.release_outputs([txi.txo_ref.txo for txi in tx.inputs])
|
||||
|
||||
async def release_all_outputs(self, account):
|
||||
return await self.run(q.release_all_outputs, account.id)
|
||||
|
||||
async def get_balance(self, **constraints):
|
||||
return await self.run(q.get_balance, **constraints)
|
||||
|
||||
async def get_report(self, accounts):
|
||||
return await self.run(q.get_report, accounts=accounts)
|
||||
|
||||
async def get_addresses(self, **constraints) -> Result[dict]:
|
||||
addresses = await self.fetch_result(q.get_addresses, **constraints)
|
||||
if addresses and 'pubkey' in addresses[0]:
|
||||
for address in addresses:
|
||||
address['pubkey'] = PubKey(
|
||||
self.ledger, bytes(address.pop('pubkey')), bytes(address.pop('chain_code')),
|
||||
address.pop('n'), address.pop('depth')
|
||||
)
|
||||
return addresses
|
||||
|
||||
async def get_all_addresses(self):
|
||||
return await self.run(q.get_all_addresses)
|
||||
|
||||
async def get_address(self, **constraints):
|
||||
for address in await self.get_addresses(limit=1, **constraints):
|
||||
return address
|
||||
|
||||
async def add_keys(self, account, chain, pubkeys):
|
||||
return await self.run(q.add_keys, [{
|
||||
'account': account.id,
|
||||
'address': k.address,
|
||||
'chain': chain,
|
||||
'pubkey': k.pubkey_bytes,
|
||||
'chain_code': k.chain_code,
|
||||
'n': k.n,
|
||||
'depth': k.depth
|
||||
} for k in pubkeys])
|
||||
|
||||
async def generate_addresses_using_filters(self, best_height, allowed_gap, address_manager):
|
||||
return await self.run(
|
||||
q.generate_addresses_using_filters, best_height, allowed_gap, address_manager
|
||||
)
|
||||
|
||||
async def get_raw_transactions(self, **constraints):
|
||||
return await self.run(q.get_raw_transactions, **constraints)
|
||||
|
||||
async def get_transactions(self, **constraints) -> Result[Transaction]:
|
||||
return await self.fetch_result(q.get_transactions, **constraints)
|
||||
|
||||
async def get_transaction(self, **constraints) -> Optional[Transaction]:
|
||||
txs = await self.get_transactions(limit=1, **constraints)
|
||||
if txs:
|
||||
return txs[0]
|
||||
|
||||
async def get_purchases(self, **constraints) -> Result[Output]:
|
||||
return await self.fetch_result(q.get_purchases, **constraints)
|
||||
|
||||
async def search_claims(self, **constraints) -> Result[Output]:
|
||||
#assert set(constraints).issubset(SEARCH_PARAMS), \
|
||||
# f"Search query contains invalid arguments: {set(constraints).difference(SEARCH_PARAMS)}"
|
||||
claims, total, censor = await self.run(q.search_claims, **constraints)
|
||||
return Result(claims, total, censor)
|
||||
|
||||
async def protobuf_search_claims(self, **constraints) -> str:
|
||||
return await self.run(q.protobuf_search_claims, **constraints)
|
||||
|
||||
async def search_supports(self, **constraints) -> Result[Output]:
|
||||
return await self.fetch_result(q.search_supports, **constraints)
|
||||
|
||||
async def sum_supports(self, claim_hash, include_channel_content=False, exclude_own_supports=False) \
|
||||
-> Tuple[List[Dict], int]:
|
||||
return await self.run(q.sum_supports, claim_hash, include_channel_content, exclude_own_supports)
|
||||
|
||||
async def resolve(self, urls, **kwargs) -> Dict[str, Output]:
|
||||
return await self.run(q.resolve, urls, **kwargs)
|
||||
|
||||
async def protobuf_resolve(self, urls, **kwargs) -> str:
|
||||
return await self.run(q.protobuf_resolve, urls, **kwargs)
|
||||
|
||||
async def get_txo_sum(self, **constraints) -> int:
|
||||
return await self.run(q.get_txo_sum, **constraints)
|
||||
|
||||
async def get_txo_plot(self, **constraints) -> List[dict]:
|
||||
return await self.run(q.get_txo_plot, **constraints)
|
||||
|
||||
async def get_txos(self, **constraints) -> Result[Output]:
|
||||
txos = await self.fetch_result(q.get_txos, **constraints)
|
||||
if 'wallet' in constraints:
|
||||
await add_channel_keys_to_txo_results(constraints['wallet'].accounts, txos)
|
||||
return txos
|
||||
|
||||
async def get_utxos(self, **constraints) -> Result[Output]:
|
||||
return await self.get_txos(spent_height=0, **constraints)
|
||||
|
||||
async def get_supports(self, **constraints) -> Result[Output]:
|
||||
return await self.get_utxos(txo_type=TXO_TYPES['support'], **constraints)
|
||||
|
||||
async def get_claims(self, **constraints) -> Result[Output]:
|
||||
if 'txo_type' not in constraints:
|
||||
constraints['txo_type__in'] = CLAIM_TYPE_CODES
|
||||
txos = await self.fetch_result(q.get_txos, **constraints)
|
||||
if 'wallet' in constraints:
|
||||
await add_channel_keys_to_txo_results(constraints['wallet'].accounts, txos)
|
||||
return txos
|
||||
|
||||
async def get_streams(self, **constraints) -> Result[Output]:
|
||||
return await self.get_claims(txo_type=TXO_TYPES['stream'], **constraints)
|
||||
|
||||
async def get_channels(self, **constraints) -> Result[Output]:
|
||||
return await self.get_claims(txo_type=TXO_TYPES['channel'], **constraints)
|
||||
|
||||
async def get_collections(self, **constraints) -> Result[Output]:
|
||||
return await self.get_claims(txo_type=TXO_TYPES['collection'], **constraints)
|
||||
@@ -1,7 +0,0 @@
|
||||
from .base import *
|
||||
from .txio import *
|
||||
from .search import *
|
||||
from .resolve import *
|
||||
from .address import *
|
||||
from .wallet import *
|
||||
from .filters import *
|
||||
@@ -1,180 +0,0 @@
|
||||
import logging
|
||||
from typing import Tuple, List, Set, Iterator, Optional
|
||||
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.future import select
|
||||
|
||||
from lbry.crypto.hash import hash160
|
||||
from lbry.crypto.bip32 import PubKey
|
||||
|
||||
from ..utils import query
|
||||
from ..query_context import context
|
||||
from ..tables import TXO, PubkeyAddress, AccountAddress
|
||||
from .filters import (
|
||||
get_filter_matchers, get_filter_matchers_at_granularity, has_filter_range,
|
||||
get_tx_matchers_for_missing_txs,
|
||||
)
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DatabaseAddressIterator:
|
||||
|
||||
def __init__(self, account_id, chain):
|
||||
self.account_id = account_id
|
||||
self.chain = chain
|
||||
self.n = -1
|
||||
|
||||
def __iter__(self) -> Iterator[Tuple[bytes, int, bool]]:
|
||||
with context().connect_streaming() as c:
|
||||
sql = (
|
||||
select(
|
||||
AccountAddress.c.pubkey,
|
||||
AccountAddress.c.n
|
||||
).where(
|
||||
(AccountAddress.c.account == self.account_id) &
|
||||
(AccountAddress.c.chain == self.chain)
|
||||
).order_by(AccountAddress.c.n)
|
||||
)
|
||||
for row in c.execute(sql):
|
||||
self.n = row['n']
|
||||
yield hash160(row['pubkey']), self.n, False
|
||||
|
||||
|
||||
class PersistingAddressIterator(DatabaseAddressIterator):
|
||||
|
||||
def __init__(self, account_id, chain, pubkey_bytes, chain_code, depth):
|
||||
super().__init__(account_id, chain)
|
||||
self.pubkey_bytes = pubkey_bytes
|
||||
self.chain_code = chain_code
|
||||
self.depth = depth
|
||||
self.pubkey_buffer = []
|
||||
|
||||
def flush(self):
|
||||
if self.pubkey_buffer:
|
||||
add_keys([{
|
||||
'account': self.account_id,
|
||||
'address': k.address,
|
||||
'chain': self.chain,
|
||||
'pubkey': k.pubkey_bytes,
|
||||
'chain_code': k.chain_code,
|
||||
'n': k.n,
|
||||
'depth': k.depth
|
||||
} for k in self.pubkey_buffer])
|
||||
self.pubkey_buffer.clear()
|
||||
|
||||
def __enter__(self) -> 'PersistingAddressIterator':
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.flush()
|
||||
|
||||
def __iter__(self) -> Iterator[Tuple[bytes, int, bool]]:
|
||||
yield from super().__iter__()
|
||||
pubkey = PubKey(context().ledger, self.pubkey_bytes, self.chain_code, 0, self.depth)
|
||||
while True:
|
||||
self.n += 1
|
||||
pubkey_child = pubkey.child(self.n)
|
||||
self.pubkey_buffer.append(pubkey_child)
|
||||
if len(self.pubkey_buffer) >= 900:
|
||||
self.flush()
|
||||
yield hash160(pubkey_child.pubkey_bytes), self.n, True
|
||||
|
||||
|
||||
def generate_addresses_using_filters(best_height, allowed_gap, address_manager) -> Set:
|
||||
need, have = set(), set()
|
||||
matchers = get_filter_matchers(best_height)
|
||||
with PersistingAddressIterator(*address_manager) as addresses:
|
||||
gap = 0
|
||||
for address_hash, n, is_new in addresses: # pylint: disable=unused-variable
|
||||
gap += 1
|
||||
address_bytes = bytearray(address_hash)
|
||||
for matcher, filter_range in matchers:
|
||||
if matcher.Match(address_bytes):
|
||||
gap = 0
|
||||
if filter_range not in need and filter_range not in have:
|
||||
if has_filter_range(*filter_range):
|
||||
have.add(filter_range)
|
||||
else:
|
||||
need.add(filter_range)
|
||||
if gap >= allowed_gap:
|
||||
break
|
||||
return need
|
||||
|
||||
|
||||
def get_missing_sub_filters_for_addresses(granularity, address_manager):
|
||||
need = set()
|
||||
for matcher, filter_range in get_filter_matchers_at_granularity(granularity):
|
||||
for address_hash, _, _ in DatabaseAddressIterator(*address_manager):
|
||||
address_bytes = bytearray(address_hash)
|
||||
if matcher.Match(address_bytes) and not has_filter_range(*filter_range):
|
||||
need.add(filter_range)
|
||||
break
|
||||
return need
|
||||
|
||||
|
||||
def get_missing_tx_for_addresses(address_manager):
|
||||
need = set()
|
||||
for tx_hash, matcher in get_tx_matchers_for_missing_txs():
|
||||
for address_hash, _, _ in DatabaseAddressIterator(*address_manager):
|
||||
address_bytes = bytearray(address_hash)
|
||||
if matcher.Match(address_bytes):
|
||||
need.add(tx_hash)
|
||||
break
|
||||
return need
|
||||
|
||||
|
||||
def update_address_used_times(addresses):
|
||||
context().execute(
|
||||
PubkeyAddress.update()
|
||||
.values(used_times=(
|
||||
select(func.count(TXO.c.address))
|
||||
.where((TXO.c.address == PubkeyAddress.c.address)),
|
||||
))
|
||||
.where(PubkeyAddress.c.address._in(addresses))
|
||||
)
|
||||
|
||||
|
||||
def select_addresses(cols, **constraints):
|
||||
return context().fetchall(query(
|
||||
[AccountAddress, PubkeyAddress],
|
||||
select(*cols).select_from(PubkeyAddress.join(AccountAddress)),
|
||||
**constraints
|
||||
))
|
||||
|
||||
|
||||
def get_addresses(cols=None, include_total=False, **constraints) -> Tuple[List[dict], Optional[int]]:
|
||||
if cols is None:
|
||||
cols = (
|
||||
PubkeyAddress.c.address,
|
||||
PubkeyAddress.c.used_times,
|
||||
AccountAddress.c.account,
|
||||
AccountAddress.c.chain,
|
||||
AccountAddress.c.pubkey,
|
||||
AccountAddress.c.chain_code,
|
||||
AccountAddress.c.n,
|
||||
AccountAddress.c.depth
|
||||
)
|
||||
return (
|
||||
select_addresses(cols, **constraints),
|
||||
get_address_count(**constraints) if include_total else None
|
||||
)
|
||||
|
||||
|
||||
def get_address_count(**constraints):
|
||||
count = select_addresses([func.count().label("total")], **constraints)
|
||||
return count[0]["total"] or 0
|
||||
|
||||
|
||||
def get_all_addresses():
|
||||
return [r["address"] for r in context().fetchall(select(PubkeyAddress.c.address))]
|
||||
|
||||
|
||||
def add_keys(pubkeys):
|
||||
c = context()
|
||||
current_limit = c.variable_limit // len(pubkeys[0]) # (overall limit) // (maximum on a query)
|
||||
for start in range(0, len(pubkeys), current_limit - 1):
|
||||
batch = pubkeys[start:(start + current_limit - 1)]
|
||||
c.execute(c.insert_or_ignore(PubkeyAddress).values([{'address': k['address']} for k in batch]))
|
||||
c.execute(c.insert_or_ignore(AccountAddress).values(batch))
|
||||
@@ -1,96 +0,0 @@
|
||||
from sqlalchemy import text, between
|
||||
from sqlalchemy.future import select
|
||||
|
||||
from ..query_context import context
|
||||
from ..tables import (
|
||||
SCHEMA_VERSION, metadata, Version,
|
||||
Claim, Support, Block, TX,
|
||||
pg_add_account_address_constraints_and_indexes
|
||||
)
|
||||
|
||||
|
||||
def execute(sql):
|
||||
return context().execute(text(sql))
|
||||
|
||||
|
||||
def execute_sql_object(sql):
|
||||
return context().execute(sql)
|
||||
|
||||
|
||||
def execute_fetchall(sql):
|
||||
return context().fetchall(text(sql))
|
||||
|
||||
|
||||
def has_claims():
|
||||
return context().has_records(Claim)
|
||||
|
||||
|
||||
def has_supports():
|
||||
return context().has_records(Support)
|
||||
|
||||
|
||||
def get_best_block_height():
|
||||
return context().fetchmax(Block.c.height, -1)
|
||||
|
||||
|
||||
def insert_blocks(blocks):
|
||||
loader = context().get_bulk_loader()
|
||||
for block in blocks:
|
||||
loader.add_block(block)
|
||||
loader.flush(return_row_count_for_table=None)
|
||||
|
||||
|
||||
def get_block_headers(first, last=None):
|
||||
q = (
|
||||
select(
|
||||
Block.c.height,
|
||||
Block.c.block_hash,
|
||||
Block.c.previous_hash,
|
||||
Block.c.timestamp,
|
||||
)
|
||||
.select_from(Block)
|
||||
)
|
||||
if last is not None:
|
||||
query = (
|
||||
q.where(between(Block.c.height, first, last))
|
||||
.order_by(Block.c.height)
|
||||
)
|
||||
else:
|
||||
query = q.where(Block.c.height == first)
|
||||
rows = context().fetchall(query)
|
||||
for row in rows:
|
||||
row['block_hash'] = bytes(row['block_hash'])
|
||||
row['previous_hash'] = bytes(row['previous_hash'])
|
||||
return rows
|
||||
|
||||
|
||||
def insert_transaction(block_hash, tx):
|
||||
context().get_bulk_loader().add_transaction(block_hash, tx).flush(TX)
|
||||
|
||||
|
||||
def check_version_and_create_tables():
|
||||
with context("db.connecting") as ctx:
|
||||
if ctx.has_table('version'):
|
||||
version = ctx.fetchone(select(Version.c.version).limit(1))
|
||||
if version and version['version'] == SCHEMA_VERSION:
|
||||
return
|
||||
metadata.drop_all(ctx.engine)
|
||||
metadata.create_all(ctx.engine)
|
||||
ctx.execute(Version.insert().values(version=SCHEMA_VERSION))
|
||||
for table in metadata.sorted_tables:
|
||||
disable_trigger_and_constraints(table.name)
|
||||
if ctx.is_postgres:
|
||||
for statement in pg_add_account_address_constraints_and_indexes:
|
||||
ctx.execute(text(statement))
|
||||
|
||||
|
||||
def disable_trigger_and_constraints(table_name):
|
||||
ctx = context()
|
||||
if ctx.is_postgres:
|
||||
ctx.execute(text(f"ALTER TABLE {table_name} DISABLE TRIGGER ALL;"))
|
||||
if table_name in ('tag', 'stake', 'block_filter', 'mempool_filter'):
|
||||
return
|
||||
if ctx.is_postgres:
|
||||
ctx.execute(text(
|
||||
f"ALTER TABLE {table_name} DROP CONSTRAINT {table_name}_pkey CASCADE;"
|
||||
))
|
||||
@@ -1,169 +0,0 @@
|
||||
from math import log10
|
||||
from typing import Dict, List, Tuple, Set, Optional
|
||||
|
||||
from sqlalchemy import between, func, or_
|
||||
from sqlalchemy.future import select
|
||||
|
||||
from lbry.blockchain.block import PyBIP158, get_address_filter
|
||||
|
||||
from ..query_context import context
|
||||
from ..tables import BlockFilter, TXFilter, TX
|
||||
|
||||
|
||||
def has_filters():
|
||||
return context().has_records(BlockFilter)
|
||||
|
||||
|
||||
def get_sub_filter_range(granularity: int, height: int):
|
||||
end = height
|
||||
if granularity >= 3:
|
||||
end = height + 10**(granularity-1) * 9
|
||||
elif granularity == 2:
|
||||
end = height + 99
|
||||
return granularity - 1, height, end
|
||||
|
||||
|
||||
def has_filter_range(factor: int, start: int, end: int):
|
||||
if factor >= 1:
|
||||
filters = context().fetchtotal(
|
||||
(BlockFilter.c.factor == factor) &
|
||||
between(BlockFilter.c.height, start, end)
|
||||
)
|
||||
expected = 10 if factor >= 2 else 100
|
||||
return filters == expected
|
||||
elif factor == 0:
|
||||
tx_filters_count = context().fetchtotal(TXFilter.c.height == start)
|
||||
return tx_filters_count > 0
|
||||
|
||||
|
||||
def get_filters(start_height, end_height=None, granularity=0):
|
||||
assert granularity >= 0, "filter granularity must be 0 or positive number"
|
||||
if granularity == 0:
|
||||
query = (
|
||||
select(TXFilter.c.height, TXFilter.c.address_filter, TXFilter.c.tx_hash)
|
||||
.select_from(TXFilter)
|
||||
.where(between(TXFilter.c.height, start_height, end_height))
|
||||
.order_by(TXFilter.c.height)
|
||||
)
|
||||
else:
|
||||
factor = granularity if granularity <= 4 else log10(granularity)
|
||||
if end_height is None:
|
||||
height_condition = (BlockFilter.c.height == start_height)
|
||||
elif end_height == -1:
|
||||
height_condition = (BlockFilter.c.height >= start_height)
|
||||
else:
|
||||
height_condition = between(BlockFilter.c.height, start_height, end_height)
|
||||
query = (
|
||||
select(BlockFilter.c.height, BlockFilter.c.address_filter)
|
||||
.select_from(BlockFilter)
|
||||
.where(height_condition & (BlockFilter.c.factor == factor))
|
||||
.order_by(BlockFilter.c.height)
|
||||
)
|
||||
return context().fetchall(query)
|
||||
|
||||
|
||||
def get_minimal_required_filter_ranges(height) -> Dict[int, Tuple[int, int]]:
|
||||
minimal = {}
|
||||
if height >= 10_000:
|
||||
minimal[4] = (0, ((height // 10_000)-1) * 10_000)
|
||||
if height >= 1_000:
|
||||
start = height - height % 10_000
|
||||
minimal[3] = (start, start+(((height-start) // 1_000)-1) * 1_000)
|
||||
if height >= 100:
|
||||
start = height - height % 1_000
|
||||
minimal[2] = (start, start+(((height-start) // 100)-1) * 100)
|
||||
start = height - height % 100
|
||||
if start < height:
|
||||
minimal[1] = (start, height)
|
||||
return minimal
|
||||
|
||||
|
||||
def get_maximum_known_filters() -> Dict[str, Optional[int]]:
|
||||
query = select(
|
||||
select(func.max(BlockFilter.c.height))
|
||||
.where(BlockFilter.c.factor == 1)
|
||||
.scalar_subquery().label('1'),
|
||||
select(func.max(BlockFilter.c.height))
|
||||
.where(BlockFilter.c.factor == 2)
|
||||
.scalar_subquery().label('2'),
|
||||
select(func.max(BlockFilter.c.height))
|
||||
.where(BlockFilter.c.factor == 3)
|
||||
.scalar_subquery().label('3'),
|
||||
select(func.max(BlockFilter.c.height))
|
||||
.where(BlockFilter.c.factor == 4)
|
||||
.scalar_subquery().label('4'),
|
||||
)
|
||||
return context().fetchone(query)
|
||||
|
||||
|
||||
def get_missing_required_filters(height) -> Set[Tuple[int, int, int]]:
|
||||
known_filters = get_maximum_known_filters()
|
||||
missing_filters = set()
|
||||
for granularity, (start, end) in get_minimal_required_filter_ranges(height).items():
|
||||
known_height = known_filters.get(str(granularity))
|
||||
if known_height is not None and known_height > start:
|
||||
if granularity == 1:
|
||||
adjusted_height = known_height + 1
|
||||
else:
|
||||
adjusted_height = known_height + 10**granularity
|
||||
if adjusted_height <= end:
|
||||
missing_filters.add((granularity, adjusted_height, end))
|
||||
else:
|
||||
missing_filters.add((granularity, start, end))
|
||||
return missing_filters
|
||||
|
||||
|
||||
def get_filter_matchers(height) -> List[Tuple[PyBIP158, Tuple[int, int, int]]]:
|
||||
conditions = []
|
||||
for granularity, (start, end) in get_minimal_required_filter_ranges(height).items():
|
||||
conditions.append(
|
||||
(BlockFilter.c.factor == granularity) &
|
||||
between(BlockFilter.c.height, start, end)
|
||||
)
|
||||
query = (
|
||||
select(BlockFilter.c.factor, BlockFilter.c.height, BlockFilter.c.address_filter)
|
||||
.select_from(BlockFilter)
|
||||
.where(or_(*conditions))
|
||||
.order_by(BlockFilter.c.height.desc())
|
||||
)
|
||||
return [(
|
||||
get_address_filter(bf["address_filter"]),
|
||||
get_sub_filter_range(bf["factor"], bf["height"])
|
||||
) for bf in context().fetchall(query)]
|
||||
|
||||
|
||||
def get_filter_matchers_at_granularity(granularity) -> List[Tuple[PyBIP158, Tuple]]:
|
||||
query = (
|
||||
select(BlockFilter.c.height, BlockFilter.c.address_filter)
|
||||
.where(BlockFilter.c.factor == granularity)
|
||||
.order_by(BlockFilter.c.height.desc())
|
||||
)
|
||||
return [(
|
||||
get_address_filter(bf["address_filter"]),
|
||||
get_sub_filter_range(granularity, bf["height"])
|
||||
) for bf in context().fetchall(query)]
|
||||
|
||||
|
||||
def get_tx_matchers_for_missing_txs() -> List[Tuple[int, PyBIP158]]:
|
||||
query = (
|
||||
select(TXFilter.c.tx_hash, TXFilter.c.address_filter)
|
||||
.where(TXFilter.c.tx_hash.notin_(select(TX.c.tx_hash)))
|
||||
)
|
||||
return [
|
||||
(bf["tx_hash"], get_address_filter(bf["address_filter"]))
|
||||
for bf in context().fetchall(query)
|
||||
]
|
||||
|
||||
|
||||
def insert_block_filters(filters):
|
||||
loader = context().get_bulk_loader()
|
||||
for height, factor, address_filter in filters:
|
||||
loader.add_block_filter(height, factor, address_filter)
|
||||
loader.flush(return_row_count_for_table=None)
|
||||
|
||||
|
||||
def insert_tx_filters(filters):
|
||||
loader = context().get_bulk_loader()
|
||||
for tx_hash, height, address_filter in filters:
|
||||
loader.add_transaction_filter(tx_hash, height, address_filter)
|
||||
loader.flush(return_row_count_for_table=None)
|
||||
@@ -1,101 +0,0 @@
|
||||
import logging
|
||||
import itertools
|
||||
from typing import List, Dict
|
||||
|
||||
from lbry.schema.url import URL
|
||||
from lbry.schema.result import Outputs as ResultOutput
|
||||
from lbry.error import ResolveCensoredError
|
||||
from lbry.blockchain.transaction import Output
|
||||
from . import rows_to_txos
|
||||
|
||||
from ..query_context import context
|
||||
from .search import select_claims
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def resolve_claims(**constraints):
|
||||
censor = context().get_resolve_censor()
|
||||
rows = context().fetchall(select_claims(**constraints))
|
||||
rows = censor.apply(rows)
|
||||
return rows_to_txos(rows), censor
|
||||
|
||||
|
||||
def _get_referenced_rows(txo_rows: List[Output], censor_channels: List[bytes]):
|
||||
repost_hashes = set(txo.reposted_claim.claim_hash for txo in txo_rows if txo.reposted_claim)
|
||||
channel_hashes = set(itertools.chain(
|
||||
(txo.channel.claim_hash for txo in txo_rows if txo.channel),
|
||||
censor_channels
|
||||
))
|
||||
|
||||
reposted_txos = []
|
||||
if repost_hashes:
|
||||
reposted_txos = resolve_claims(**{'claim.claim_hash__in': repost_hashes})
|
||||
if reposted_txos:
|
||||
reposted_txos = reposted_txos[0]
|
||||
channel_hashes |= set(txo.channel.claim_hash for txo in reposted_txos if txo.channel)
|
||||
|
||||
channel_txos = []
|
||||
if channel_hashes:
|
||||
channel_txos = resolve_claims(**{'claim.claim_hash__in': channel_hashes})
|
||||
channel_txos = channel_txos[0] if channel_txos else []
|
||||
|
||||
# channels must come first for client side inflation to work properly
|
||||
return channel_txos + reposted_txos
|
||||
|
||||
|
||||
def protobuf_resolve(urls, **kwargs) -> str:
|
||||
txo_rows = [resolve_url(raw_url) for raw_url in urls]
|
||||
extra_txo_rows = _get_referenced_rows(
|
||||
[txo_row for txo_row in txo_rows if isinstance(txo_row, Output)],
|
||||
[txo.censor_hash for txo in txo_rows if isinstance(txo, ResolveCensoredError)]
|
||||
)
|
||||
return ResultOutput.to_base64(txo_rows, extra_txo_rows)
|
||||
|
||||
|
||||
def resolve(urls, **kwargs) -> Dict[str, Output]:
|
||||
return {url: resolve_url(url) for url in urls}
|
||||
|
||||
|
||||
def resolve_url(raw_url):
|
||||
try:
|
||||
url = URL.parse(raw_url)
|
||||
except ValueError as e:
|
||||
return e
|
||||
|
||||
channel = None
|
||||
|
||||
if url.has_channel:
|
||||
q = url.channel.to_dict()
|
||||
if set(q) == {'name'}:
|
||||
q['is_controlling'] = True
|
||||
else:
|
||||
q['order_by'] = ['^creation_height']
|
||||
matches, censor = resolve_claims(**q, limit=1)
|
||||
if matches:
|
||||
channel = matches[0]
|
||||
elif censor.censored:
|
||||
return ResolveCensoredError(raw_url, next(iter(censor.censored)))
|
||||
elif not channel:
|
||||
return LookupError(f'Could not find channel in "{raw_url}".')
|
||||
|
||||
if url.has_stream:
|
||||
q = url.stream.to_dict()
|
||||
if channel is not None:
|
||||
q['order_by'] = ['^creation_height']
|
||||
q['channel_hash'] = channel.claim_hash
|
||||
q['is_signature_valid'] = True
|
||||
elif set(q) == {'name'}:
|
||||
q['is_controlling'] = True
|
||||
matches, censor = resolve_claims(**q, limit=1)
|
||||
if matches:
|
||||
stream = matches[0]
|
||||
stream.channel = channel
|
||||
return stream
|
||||
elif censor.censored:
|
||||
return ResolveCensoredError(raw_url, next(iter(censor.censored)))
|
||||
else:
|
||||
return LookupError(f'Could not find claim at "{raw_url}".')
|
||||
|
||||
return channel
|
||||
@@ -1,472 +0,0 @@
|
||||
import struct
|
||||
import logging
|
||||
from decimal import Decimal
|
||||
from binascii import unhexlify
|
||||
from typing import Tuple, List, Optional, Dict
|
||||
|
||||
from sqlalchemy import func, case, text
|
||||
from sqlalchemy.future import select, Select
|
||||
|
||||
from lbry.schema.tags import clean_tags
|
||||
from lbry.schema.result import Censor, Outputs as ResultOutput
|
||||
from lbry.schema.url import normalize_name
|
||||
from lbry.blockchain.transaction import Output
|
||||
|
||||
from ..utils import query
|
||||
from ..query_context import context
|
||||
from ..tables import TX, TXO, Claim, Support, Trend, CensoredClaim
|
||||
from ..constants import (
|
||||
TXO_TYPES, STREAM_TYPES, ATTRIBUTE_ARRAY_MAX_LENGTH,
|
||||
SEARCH_INTEGER_PARAMS, SEARCH_ORDER_FIELDS
|
||||
)
|
||||
|
||||
from .txio import BASE_SELECT_TXO_COLUMNS, rows_to_txos
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
BASE_SELECT_SUPPORT_COLUMNS = BASE_SELECT_TXO_COLUMNS + [
|
||||
Support.c.channel_hash,
|
||||
Support.c.is_signature_valid,
|
||||
]
|
||||
|
||||
|
||||
def compat_layer(**constraints):
|
||||
# for old sdk, to be removed later
|
||||
replacements = {
|
||||
"effective_amount": "staked_amount",
|
||||
"trending_mixed": "trend_mixed",
|
||||
"trending_group": "trend_group",
|
||||
"trending_local": "trend_local"
|
||||
}
|
||||
for old_key, new_key in replacements.items():
|
||||
if old_key in constraints:
|
||||
constraints[new_key] = constraints.pop(old_key)
|
||||
order_by = constraints.get("order_by", [])
|
||||
if old_key in order_by:
|
||||
constraints["order_by"] = [order_key if order_key != old_key else new_key for order_key in order_by]
|
||||
return constraints
|
||||
|
||||
|
||||
def select_supports(cols: List = None, **constraints) -> Select:
|
||||
if cols is None:
|
||||
cols = BASE_SELECT_SUPPORT_COLUMNS
|
||||
joins = Support.join(TXO, ).join(TX)
|
||||
return query([Support], select(*cols).select_from(joins), **constraints)
|
||||
|
||||
|
||||
def search_supports(**constraints) -> Tuple[List[Output], Optional[int]]:
|
||||
total = None
|
||||
if constraints.pop('include_total', False):
|
||||
total = search_support_count(**constraints)
|
||||
if 'claim_id' in constraints:
|
||||
constraints['claim_hash'] = unhexlify(constraints.pop('claim_id'))[::-1]
|
||||
rows = context().fetchall(select_supports(**constraints))
|
||||
txos = rows_to_txos(rows, include_tx=False)
|
||||
return txos, total
|
||||
|
||||
|
||||
def sum_supports(claim_hash, include_channel_content=False, exclude_own_supports=False) -> Tuple[List[Dict], int]:
|
||||
supporter = Claim.alias("supporter")
|
||||
content = Claim.alias("content")
|
||||
where_condition = (content.c.claim_hash == claim_hash)
|
||||
if include_channel_content:
|
||||
where_condition |= (content.c.channel_hash == claim_hash)
|
||||
support_join_condition = TXO.c.channel_hash == supporter.c.claim_hash
|
||||
if exclude_own_supports:
|
||||
support_join_condition &= TXO.c.channel_hash != claim_hash
|
||||
|
||||
q = select(
|
||||
supporter.c.short_url.label("supporter"),
|
||||
func.sum(TXO.c.amount).label("staked"),
|
||||
).select_from(
|
||||
TXO
|
||||
.join(content, TXO.c.claim_hash == content.c.claim_hash)
|
||||
.join(supporter, support_join_condition)
|
||||
).where(
|
||||
where_condition &
|
||||
(TXO.c.txo_type == TXO_TYPES["support"]) &
|
||||
((TXO.c.address == content.c.address) | ((TXO.c.address != content.c.address) & (TXO.c.spent_height == 0)))
|
||||
).group_by(
|
||||
supporter.c.short_url
|
||||
).order_by(
|
||||
text("staked DESC, supporter ASC")
|
||||
)
|
||||
|
||||
result = context().fetchall(q)
|
||||
total = sum([row['staked'] for row in result])
|
||||
return result, total
|
||||
|
||||
|
||||
def search_support_count(**constraints) -> int:
|
||||
constraints.pop('offset', None)
|
||||
constraints.pop('limit', None)
|
||||
constraints.pop('order_by', None)
|
||||
count = context().fetchall(select_supports([func.count().label('total')], **constraints))
|
||||
return count[0]['total'] or 0
|
||||
|
||||
|
||||
channel_claim = Claim.alias('channel')
|
||||
BASE_SELECT_CLAIM_COLUMNS = BASE_SELECT_TXO_COLUMNS + [
|
||||
Claim.c.activation_height,
|
||||
Claim.c.takeover_height,
|
||||
Claim.c.creation_height,
|
||||
Claim.c.expiration_height,
|
||||
Claim.c.is_controlling,
|
||||
Claim.c.channel_hash,
|
||||
Claim.c.reposted_count,
|
||||
Claim.c.reposted_claim_hash,
|
||||
Claim.c.short_url,
|
||||
Claim.c.signed_claim_count,
|
||||
Claim.c.signed_support_count,
|
||||
(Claim.c.amount + Claim.c.staked_support_amount).label('staked_amount'),
|
||||
Claim.c.staked_support_amount,
|
||||
Claim.c.staked_support_count,
|
||||
Claim.c.is_signature_valid,
|
||||
case([(
|
||||
channel_claim.c.short_url.isnot(None),
|
||||
channel_claim.c.short_url + '/' + Claim.c.short_url
|
||||
)]).label('canonical_url'),
|
||||
func.coalesce(Trend.c.trend_local, 0).label('trend_local'),
|
||||
func.coalesce(Trend.c.trend_mixed, 0).label('trend_mixed'),
|
||||
func.coalesce(Trend.c.trend_global, 0).label('trend_global'),
|
||||
func.coalesce(Trend.c.trend_group, 0).label('trend_group'),
|
||||
CensoredClaim.c.censor_type,
|
||||
CensoredClaim.c.censoring_channel_hash
|
||||
]
|
||||
|
||||
|
||||
def select_claims(cols: List = None, for_count=False, **constraints) -> Select:
|
||||
constraints = compat_layer(**constraints)
|
||||
if cols is None:
|
||||
cols = BASE_SELECT_CLAIM_COLUMNS
|
||||
if 'order_by' in constraints:
|
||||
order_by_parts = constraints['order_by']
|
||||
if isinstance(order_by_parts, str):
|
||||
order_by_parts = [order_by_parts]
|
||||
sql_order_by = []
|
||||
for order_by in order_by_parts:
|
||||
is_asc = order_by.startswith('^')
|
||||
column = order_by[1:] if is_asc else order_by
|
||||
if column not in SEARCH_ORDER_FIELDS:
|
||||
raise NameError(f'{column} is not a valid order_by field')
|
||||
if column == 'name':
|
||||
column = 'claim_name'
|
||||
table = "trend" if column.startswith('trend') else "claim"
|
||||
column = f"{table}.{column}"
|
||||
if column in ('trend.trend_group', 'trend.trend_mixed', 'claim.release_time'):
|
||||
column = f"COALESCE({column}, {-1 * (1<<32)})"
|
||||
sql_order_by.append(
|
||||
f"{column} {'ASC' if is_asc else 'DESC'}"
|
||||
)
|
||||
constraints['order_by'] = sql_order_by
|
||||
|
||||
ops = {'<=': '__lte', '>=': '__gte', '<': '__lt', '>': '__gt'}
|
||||
for constraint in SEARCH_INTEGER_PARAMS:
|
||||
if constraint in constraints:
|
||||
value = constraints.pop(constraint)
|
||||
postfix = ''
|
||||
if isinstance(value, str):
|
||||
if len(value) >= 2 and value[:2] in ops:
|
||||
postfix, value = ops[value[:2]], value[2:]
|
||||
elif len(value) >= 1 and value[0] in ops:
|
||||
postfix, value = ops[value[0]], value[1:]
|
||||
if constraint == 'fee_amount':
|
||||
value = Decimal(value)*1000
|
||||
constraints[f'{constraint}{postfix}'] = int(value)
|
||||
|
||||
if 'sequence' in constraints:
|
||||
constraints['order_by'] = 'activation_height ASC'
|
||||
constraints['offset'] = int(constraints.pop('sequence')) - 1
|
||||
constraints['limit'] = 1
|
||||
if 'amount_order' in constraints:
|
||||
constraints['order_by'] = 'staked_amount DESC'
|
||||
constraints['offset'] = int(constraints.pop('amount_order')) - 1
|
||||
constraints['limit'] = 1
|
||||
|
||||
if 'claim_id' in constraints:
|
||||
claim_id = constraints.pop('claim_id')
|
||||
if len(claim_id) == 40:
|
||||
constraints['claim_id'] = claim_id
|
||||
else:
|
||||
constraints['claim_id__like'] = f'{claim_id[:40]}%'
|
||||
elif 'claim_ids' in constraints:
|
||||
constraints['claim_id__in'] = set(constraints.pop('claim_ids'))
|
||||
|
||||
if 'reposted_claim_id' in constraints:
|
||||
constraints['reposted_claim_hash'] = unhexlify(constraints.pop('reposted_claim_id'))[::-1]
|
||||
|
||||
if 'name' in constraints:
|
||||
constraints['normalized'] = normalize_name(constraints.pop('name'))
|
||||
|
||||
if 'public_key_id' in constraints:
|
||||
constraints['public_key_hash'] = (
|
||||
context().ledger.address_to_hash160(constraints.pop('public_key_id')))
|
||||
if 'channel_id' in constraints:
|
||||
channel_id = constraints.pop('channel_id')
|
||||
if channel_id:
|
||||
if isinstance(channel_id, str):
|
||||
channel_id = [channel_id]
|
||||
constraints['channel_hash__in'] = {
|
||||
unhexlify(cid)[::-1] for cid in channel_id
|
||||
}
|
||||
if 'not_channel_id' in constraints:
|
||||
not_channel_ids = constraints.pop('not_channel_id')
|
||||
if not_channel_ids:
|
||||
not_channel_ids_binary = {
|
||||
unhexlify(ncid)[::-1] for ncid in not_channel_ids
|
||||
}
|
||||
constraints['claim_hash__not_in#not_channel_ids'] = not_channel_ids_binary
|
||||
if constraints.get('has_channel_signature', False):
|
||||
constraints['channel_hash__not_in'] = not_channel_ids_binary
|
||||
else:
|
||||
constraints['null_or_not_channel__or'] = {
|
||||
'signature_valid__is_null': True,
|
||||
'channel_hash__not_in': not_channel_ids_binary
|
||||
}
|
||||
if 'is_signature_valid' in constraints:
|
||||
has_channel_signature = constraints.pop('has_channel_signature', False)
|
||||
is_signature_valid = constraints.pop('is_signature_valid')
|
||||
if has_channel_signature:
|
||||
constraints['is_signature_valid'] = is_signature_valid
|
||||
else:
|
||||
constraints['null_or_signature__or'] = {
|
||||
'is_signature_valid__is_null': True,
|
||||
'is_signature_valid': is_signature_valid
|
||||
}
|
||||
elif constraints.pop('has_channel_signature', False):
|
||||
constraints['is_signature_valid__is_not_null'] = True
|
||||
|
||||
if 'txid' in constraints:
|
||||
tx_hash = unhexlify(constraints.pop('txid'))[::-1]
|
||||
nout = constraints.pop('nout', 0)
|
||||
constraints['txo_hash'] = tx_hash + struct.pack('<I', nout)
|
||||
|
||||
if 'claim_type' in constraints:
|
||||
claim_types = constraints.pop('claim_type')
|
||||
if isinstance(claim_types, str):
|
||||
claim_types = {claim_types}
|
||||
if claim_types:
|
||||
constraints['claim_type__in'] = {
|
||||
TXO_TYPES[claim_type] for claim_type in claim_types
|
||||
}
|
||||
if 'stream_type' in constraints:
|
||||
stream_types = constraints.pop('stream_type')
|
||||
if isinstance(stream_types, str):
|
||||
stream_types = {stream_types}
|
||||
if stream_types:
|
||||
constraints['stream_type__in'] = {
|
||||
STREAM_TYPES[stream_type] for stream_type in stream_types
|
||||
}
|
||||
if 'media_type' in constraints:
|
||||
media_types = constraints.pop('media_type')
|
||||
if isinstance(media_types, str):
|
||||
media_types = {media_types}
|
||||
if media_types:
|
||||
constraints['media_type__in'] = set(media_types)
|
||||
|
||||
if 'fee_currency' in constraints:
|
||||
constraints['fee_currency'] = constraints.pop('fee_currency').lower()
|
||||
|
||||
_apply_constraints_for_array_attributes(constraints, 'tag', clean_tags, for_count)
|
||||
_apply_constraints_for_array_attributes(constraints, 'language', lambda _: _, for_count)
|
||||
_apply_constraints_for_array_attributes(constraints, 'location', lambda _: _, for_count)
|
||||
|
||||
if 'text' in constraints:
|
||||
# TODO: fix
|
||||
constraints["search"] = constraints.pop("text")
|
||||
|
||||
return query(
|
||||
[Claim, TXO],
|
||||
select(*cols)
|
||||
.select_from(
|
||||
Claim.join(TXO).join(TX)
|
||||
.join(Trend, Trend.c.claim_hash == Claim.c.claim_hash, isouter=True)
|
||||
.join(channel_claim, Claim.c.channel_hash == channel_claim.c.claim_hash, isouter=True)
|
||||
.join(
|
||||
CensoredClaim,
|
||||
(CensoredClaim.c.claim_hash == Claim.c.claim_hash) |
|
||||
(CensoredClaim.c.claim_hash == Claim.c.reposted_claim_hash) |
|
||||
(CensoredClaim.c.claim_hash == Claim.c.channel_hash),
|
||||
isouter=True
|
||||
)
|
||||
), **constraints
|
||||
)
|
||||
|
||||
|
||||
def protobuf_search_claims(**constraints) -> str:
|
||||
txos, _, censor = search_claims(**constraints)
|
||||
return ResultOutput.to_base64(txos, [], blocked=censor)
|
||||
|
||||
|
||||
def search_claims(**constraints) -> Tuple[List[Output], Optional[int], Optional[Censor]]:
|
||||
ctx = context()
|
||||
search_censor = ctx.get_search_censor()
|
||||
|
||||
total = None
|
||||
if constraints.pop('include_total', False):
|
||||
total = search_claim_count(**constraints)
|
||||
|
||||
constraints['offset'] = abs(constraints.get('offset', 0))
|
||||
constraints['limit'] = min(abs(constraints.get('limit', 10)), 50)
|
||||
|
||||
channel_url = constraints.pop('channel', None)
|
||||
if channel_url:
|
||||
from .resolve import resolve_url # pylint: disable=import-outside-toplevel
|
||||
channel = resolve_url(channel_url)
|
||||
if isinstance(channel, Output):
|
||||
constraints['channel_hash'] = channel.claim_hash
|
||||
else:
|
||||
return [], total, search_censor
|
||||
|
||||
rows = ctx.fetchall(select_claims(**constraints))
|
||||
rows = search_censor.apply(rows)
|
||||
txos = rows_to_txos(rows, include_tx=False)
|
||||
annotate_with_channels(txos)
|
||||
return txos, total, search_censor
|
||||
|
||||
|
||||
def annotate_with_channels(txos):
|
||||
channel_hashes = set()
|
||||
for txo in txos:
|
||||
if txo.can_decode_claim and txo.claim.is_signed:
|
||||
channel_hashes.add(txo.claim.signing_channel_hash)
|
||||
if channel_hashes:
|
||||
rows = context().fetchall(select_claims(claim_hash__in=channel_hashes))
|
||||
channels = {
|
||||
txo.claim_hash: txo for txo in
|
||||
rows_to_txos(rows, include_tx=False)
|
||||
}
|
||||
for txo in txos:
|
||||
if txo.can_decode_claim and txo.claim.is_signed:
|
||||
txo.channel = channels.get(txo.claim.signing_channel_hash, None)
|
||||
|
||||
|
||||
def search_claim_count(**constraints) -> int:
|
||||
constraints.pop('offset', None)
|
||||
constraints.pop('limit', None)
|
||||
constraints.pop('order_by', None)
|
||||
count = context().fetchall(select_claims([func.count().label('total')], **constraints))
|
||||
return count[0]['total'] or 0
|
||||
|
||||
|
||||
CLAIM_HASH_OR_REPOST_HASH_SQL = f"""
|
||||
CASE WHEN claim.claim_type = {TXO_TYPES['repost']}
|
||||
THEN claim.reposted_claim_hash
|
||||
ELSE claim.claim_hash
|
||||
END
|
||||
"""
|
||||
|
||||
|
||||
def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_count=False):
|
||||
any_items = set(cleaner(constraints.pop(f'any_{attr}', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
|
||||
all_items = set(cleaner(constraints.pop(f'all_{attr}', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
|
||||
not_items = set(cleaner(constraints.pop(f'not_{attr}', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
|
||||
|
||||
all_items = {item for item in all_items if item not in not_items}
|
||||
any_items = {item for item in any_items if item not in not_items}
|
||||
|
||||
any_queries = {}
|
||||
|
||||
# if attr == 'tag':
|
||||
# common_tags = any_items & COMMON_TAGS.keys()
|
||||
# if common_tags:
|
||||
# any_items -= common_tags
|
||||
# if len(common_tags) < 5:
|
||||
# for item in common_tags:
|
||||
# index_name = COMMON_TAGS[item]
|
||||
# any_queries[f'#_common_tag_{index_name}'] = f"""
|
||||
# EXISTS(
|
||||
# SELECT 1 FROM tag INDEXED BY tag_{index_name}_idx
|
||||
# WHERE {CLAIM_HASH_OR_REPOST_HASH_SQL}=tag.claim_hash
|
||||
# AND tag = '{item}'
|
||||
# )
|
||||
# """
|
||||
# elif len(common_tags) >= 5:
|
||||
# constraints.update({
|
||||
# f'$any_common_tag{i}': item for i, item in enumerate(common_tags)
|
||||
# })
|
||||
# values = ', '.join(
|
||||
# f':$any_common_tag{i}' for i in range(len(common_tags))
|
||||
# )
|
||||
# any_queries[f'#_any_common_tags'] = f"""
|
||||
# EXISTS(
|
||||
# SELECT 1 FROM tag WHERE {CLAIM_HASH_OR_REPOST_HASH_SQL}=tag.claim_hash
|
||||
# AND tag IN ({values})
|
||||
# )
|
||||
# """
|
||||
|
||||
if any_items:
|
||||
|
||||
constraints.update({
|
||||
f'$any_{attr}{i}': item for i, item in enumerate(any_items)
|
||||
})
|
||||
values = ', '.join(
|
||||
f':$any_{attr}{i}' for i in range(len(any_items))
|
||||
)
|
||||
if for_count or attr == 'tag':
|
||||
any_queries[f'#_any_{attr}'] = f"""
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
else:
|
||||
any_queries[f'#_any_{attr}'] = f"""
|
||||
EXISTS(
|
||||
SELECT 1 FROM {attr} WHERE
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL}={attr}.claim_hash
|
||||
AND {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
|
||||
if len(any_queries) == 1:
|
||||
constraints.update(any_queries)
|
||||
elif len(any_queries) > 1:
|
||||
constraints[f'ORed_{attr}_queries__any'] = any_queries
|
||||
|
||||
if all_items:
|
||||
constraints[f'$all_{attr}_count'] = len(all_items)
|
||||
constraints.update({
|
||||
f'$all_{attr}{i}': item for i, item in enumerate(all_items)
|
||||
})
|
||||
values = ', '.join(
|
||||
f':$all_{attr}{i}' for i in range(len(all_items))
|
||||
)
|
||||
if for_count:
|
||||
constraints[f'#_all_{attr}'] = f"""
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
GROUP BY claim_hash HAVING COUNT({attr}) = :$all_{attr}_count
|
||||
)
|
||||
"""
|
||||
else:
|
||||
constraints[f'#_all_{attr}'] = f"""
|
||||
{len(all_items)}=(
|
||||
SELECT count(*) FROM {attr} WHERE
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL}={attr}.claim_hash
|
||||
AND {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
|
||||
if not_items:
|
||||
constraints.update({
|
||||
f'$not_{attr}{i}': item for i, item in enumerate(not_items)
|
||||
})
|
||||
values = ', '.join(
|
||||
f':$not_{attr}{i}' for i in range(len(not_items))
|
||||
)
|
||||
if for_count:
|
||||
constraints[f'#_not_{attr}'] = f"""
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL} NOT IN (
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
else:
|
||||
constraints[f'#_not_{attr}'] = f"""
|
||||
NOT EXISTS(
|
||||
SELECT 1 FROM {attr} WHERE
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL}={attr}.claim_hash
|
||||
AND {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
@@ -1,676 +0,0 @@
|
||||
import logging
|
||||
from datetime import date
|
||||
from typing import Tuple, List, Optional, Union
|
||||
|
||||
from sqlalchemy import union, func, text, between, distinct, case, false, not_, exists
|
||||
from sqlalchemy.future import select, Select
|
||||
|
||||
from lbry.constants import INVALIDATED_SIGNATURE_GRACE_PERIOD
|
||||
from ...blockchain.transaction import (
|
||||
Transaction, Output, OutputScript, TXRefImmutable
|
||||
)
|
||||
from ..tables import (
|
||||
TX, TXO, TXI, txi_join_account, txo_join_account,
|
||||
Claim, Support, AccountAddress
|
||||
)
|
||||
from ..utils import query, in_account_ids
|
||||
from ..query_context import context
|
||||
from ..constants import TXO_TYPES, CLAIM_TYPE_CODES, MAX_QUERY_VARIABLES
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
minimum_txo_columns = (
|
||||
TXO.c.amount, TXO.c.position.label('txo_position'),
|
||||
TX.c.tx_hash, TX.c.height, TX.c.timestamp,
|
||||
func.substr(TX.c.raw, TXO.c.script_offset + 1, TXO.c.script_length).label('src'),
|
||||
)
|
||||
|
||||
|
||||
def row_to_txo(row):
|
||||
return Output(
|
||||
amount=row.amount,
|
||||
script=OutputScript(row.src),
|
||||
tx_ref=TXRefImmutable.from_hash(row.tx_hash, row.height, row.timestamp),
|
||||
position=row.txo_position,
|
||||
)
|
||||
|
||||
|
||||
def where_txo_type_in(txo_type: Optional[Union[tuple, int]] = None):
|
||||
if txo_type is not None:
|
||||
if isinstance(txo_type, int):
|
||||
return TXO.c.txo_type == txo_type
|
||||
assert len(txo_type) > 0
|
||||
if len(txo_type) == 1:
|
||||
return TXO.c.txo_type == txo_type[0]
|
||||
else:
|
||||
return TXO.c.txo_type.in_(txo_type)
|
||||
return TXO.c.txo_type.in_(CLAIM_TYPE_CODES)
|
||||
|
||||
|
||||
def where_unspent_txos(
|
||||
txo_types: Tuple[int, ...],
|
||||
blocks: Tuple[int, int] = None,
|
||||
missing_in_supports_table: bool = False,
|
||||
missing_in_claims_table: bool = False,
|
||||
missing_or_stale_in_claims_table: bool = False,
|
||||
):
|
||||
condition = where_txo_type_in(txo_types) & (TXO.c.spent_height == 0)
|
||||
if blocks is not None:
|
||||
condition &= between(TXO.c.height, *blocks)
|
||||
if missing_in_supports_table:
|
||||
condition &= not_(
|
||||
exists(select(1).where(Support.c.txo_hash == TXO.c.txo_hash))
|
||||
)
|
||||
elif missing_or_stale_in_claims_table:
|
||||
condition &= not_(
|
||||
exists(select(1).where(Claim.c.txo_hash == TXO.c.txo_hash))
|
||||
)
|
||||
elif missing_in_claims_table:
|
||||
condition &= not_(
|
||||
exists(select(1).where(Claim.c.claim_hash == TXO.c.claim_hash))
|
||||
)
|
||||
return condition
|
||||
|
||||
|
||||
def where_abandoned_claims():
|
||||
return Claim.c.claim_hash.notin_(
|
||||
select(TXO.c.claim_hash).where(where_unspent_txos(CLAIM_TYPE_CODES))
|
||||
)
|
||||
|
||||
|
||||
def count_abandoned_claims():
|
||||
return context().fetchtotal(where_abandoned_claims())
|
||||
|
||||
|
||||
def where_abandoned_supports():
|
||||
return Support.c.txo_hash.notin_(
|
||||
select(TXO.c.txo_hash).where(where_unspent_txos(TXO_TYPES['support']))
|
||||
)
|
||||
|
||||
|
||||
def count_abandoned_supports():
|
||||
return context().fetchtotal(where_abandoned_supports())
|
||||
|
||||
|
||||
def count_unspent_txos(
|
||||
txo_types: Tuple[int, ...],
|
||||
blocks: Tuple[int, int] = None,
|
||||
missing_in_supports_table: bool = False,
|
||||
missing_in_claims_table: bool = False,
|
||||
missing_or_stale_in_claims_table: bool = False,
|
||||
):
|
||||
return context().fetchtotal(
|
||||
where_unspent_txos(
|
||||
txo_types, blocks,
|
||||
missing_in_supports_table,
|
||||
missing_in_claims_table,
|
||||
missing_or_stale_in_claims_table,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def distribute_unspent_txos(
|
||||
txo_types: Tuple[int, ...],
|
||||
blocks: Tuple[int, int] = None,
|
||||
missing_in_supports_table: bool = False,
|
||||
missing_in_claims_table: bool = False,
|
||||
missing_or_stale_in_claims_table: bool = False,
|
||||
number_of_buckets: int = 10
|
||||
) -> Tuple[int, List[Tuple[int, int]]]:
|
||||
chunks = (
|
||||
select(func.ntile(number_of_buckets).over(order_by=TXO.c.height).label('chunk'), TXO.c.height)
|
||||
.where(
|
||||
where_unspent_txos(
|
||||
txo_types, blocks,
|
||||
missing_in_supports_table,
|
||||
missing_in_claims_table,
|
||||
missing_or_stale_in_claims_table,
|
||||
)
|
||||
).cte('chunks')
|
||||
)
|
||||
sql = (
|
||||
select(
|
||||
func.count('*').label('items'),
|
||||
func.min(chunks.c.height).label('start_height'),
|
||||
func.max(chunks.c.height).label('end_height'),
|
||||
).group_by(chunks.c.chunk).order_by(chunks.c.chunk)
|
||||
)
|
||||
total = 0
|
||||
buckets = []
|
||||
for bucket in context().fetchall(sql):
|
||||
total += bucket['items']
|
||||
if len(buckets) > 0:
|
||||
if buckets[-1][-1] == bucket['start_height']:
|
||||
if bucket['start_height'] == bucket['end_height']:
|
||||
continue
|
||||
bucket['start_height'] += 1
|
||||
buckets.append((bucket['start_height'], bucket['end_height']))
|
||||
return total, buckets
|
||||
|
||||
|
||||
def where_changed_support_txos(blocks: Optional[Tuple[int, int]]):
|
||||
return (
|
||||
(TXO.c.txo_type == TXO_TYPES['support']) & (
|
||||
between(TXO.c.height, blocks[0], blocks[-1]) |
|
||||
between(TXO.c.spent_height, blocks[0], blocks[-1])
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def where_claims_with_changed_supports(blocks: Optional[Tuple[int, int]]):
|
||||
return Claim.c.claim_hash.in_(
|
||||
select(TXO.c.claim_hash).where(
|
||||
where_changed_support_txos(blocks)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def count_claims_with_changed_supports(blocks: Optional[Tuple[int, int]]) -> int:
|
||||
sql = (
|
||||
select(func.count(distinct(TXO.c.claim_hash)).label('total'))
|
||||
.where(where_changed_support_txos(blocks))
|
||||
)
|
||||
return context().fetchone(sql)['total']
|
||||
|
||||
|
||||
def where_channels_changed(blocks: Optional[Tuple[int, int]]):
|
||||
channel = TXO.alias('channel')
|
||||
return TXO.c.channel_hash.in_(
|
||||
select(channel.c.claim_hash).where(
|
||||
(channel.c.txo_type == TXO_TYPES['channel']) & (
|
||||
between(channel.c.height, blocks[0], blocks[-1]) |
|
||||
between(channel.c.spent_height, blocks[0], blocks[-1])
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def where_changed_content_txos(blocks: Optional[Tuple[int, int]]):
|
||||
return (
|
||||
(TXO.c.channel_hash.isnot(None)) & (
|
||||
between(TXO.c.height, blocks[0], blocks[-1]) |
|
||||
between(TXO.c.spent_height, blocks[0], blocks[-1])
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def where_channels_with_changed_content(blocks: Optional[Tuple[int, int]]):
|
||||
content = Claim.alias("content")
|
||||
return Claim.c.claim_hash.in_(
|
||||
union(
|
||||
select(TXO.c.channel_hash).where(where_changed_content_txos(blocks)),
|
||||
select(content.c.channel_hash).where(
|
||||
content.c.channel_hash.isnot(None) &
|
||||
# content.c.public_key_height is updated when
|
||||
# channel signature is revalidated
|
||||
between(content.c.public_key_height, blocks[0], blocks[-1])
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def count_channels_with_changed_content(blocks: Optional[Tuple[int, int]]):
|
||||
return context().fetchtotal(where_channels_with_changed_content(blocks))
|
||||
|
||||
|
||||
def where_changed_repost_txos(blocks: Optional[Tuple[int, int]]):
|
||||
return (
|
||||
(TXO.c.txo_type == TXO_TYPES['repost']) & (
|
||||
between(TXO.c.height, blocks[0], blocks[-1]) |
|
||||
between(TXO.c.spent_height, blocks[0], blocks[-1])
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def where_claims_with_changed_reposts(blocks: Optional[Tuple[int, int]]):
|
||||
return Claim.c.claim_hash.in_(
|
||||
select(TXO.c.reposted_claim_hash).where(
|
||||
where_changed_repost_txos(blocks)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def count_claims_with_changed_reposts(blocks: Optional[Tuple[int, int]]):
|
||||
sql = (
|
||||
select(func.count(distinct(TXO.c.reposted_claim_hash)).label('total'))
|
||||
.where(where_changed_repost_txos(blocks))
|
||||
)
|
||||
return context().fetchone(sql)['total']
|
||||
|
||||
|
||||
def where_claims_with_stale_signatures(s, blocks: Optional[Tuple[int, int]], stream=None):
|
||||
stream = Claim.alias('stream') if stream is None else stream
|
||||
channel = Claim.alias('channel')
|
||||
return (
|
||||
s.select_from(stream.join(channel, stream.c.channel_hash == channel.c.claim_hash))
|
||||
.where(
|
||||
(stream.c.public_key_height < channel.c.public_key_height) &
|
||||
(stream.c.public_key_height <= blocks[1]-INVALIDATED_SIGNATURE_GRACE_PERIOD)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def count_claims_with_stale_signatures(blocks: Optional[Tuple[int, int]]):
|
||||
return context().fetchone(
|
||||
where_claims_with_stale_signatures(select(func.count('*').label('total')), blocks)
|
||||
)['total']
|
||||
|
||||
|
||||
def select_transactions(cols, account_ids=None, **constraints):
|
||||
s: Select = select(*cols).select_from(TX)
|
||||
if not {'tx_hash', 'tx_hash__in'}.intersection(constraints):
|
||||
assert account_ids, (
|
||||
"'accounts' argument required when "
|
||||
"no 'tx_hash' constraint is present"
|
||||
)
|
||||
where = in_account_ids(account_ids)
|
||||
tx_hashes = union(
|
||||
select(TXO.c.tx_hash).select_from(txo_join_account).where(where),
|
||||
select(TXI.c.tx_hash).select_from(txi_join_account).where(where)
|
||||
)
|
||||
s = s.where(TX.c.tx_hash.in_(tx_hashes))
|
||||
return context().fetchall(query([TX], s, **constraints))
|
||||
|
||||
|
||||
TXO_NOT_MINE = Output(None, None, is_my_output=False)
|
||||
|
||||
|
||||
def get_raw_transactions(tx_hash__in):
|
||||
return context().fetchall(
|
||||
select(TX.c.tx_hash, TX.c.raw).where(TX.c.tx_hash.in_(tx_hash__in))
|
||||
)
|
||||
|
||||
|
||||
def get_transactions(include_total=False, **constraints) -> Tuple[List[Transaction], Optional[int]]:
|
||||
account_ids = constraints.pop('account_ids', None)
|
||||
include_is_my_input = constraints.pop('include_is_my_input', False)
|
||||
include_is_my_output = constraints.pop('include_is_my_output', False)
|
||||
|
||||
tx_rows = select_transactions(
|
||||
[TX.c.tx_hash, TX.c.raw, TX.c.height, TX.c.position, TX.c.timestamp, TX.c.is_verified],
|
||||
order_by=constraints.pop('order_by', ["height=0 DESC", "height DESC", "position DESC"]),
|
||||
account_ids=account_ids,
|
||||
**constraints
|
||||
)
|
||||
|
||||
txids, txs, txi_txoids = [], [], []
|
||||
for row in tx_rows:
|
||||
txids.append(row['tx_hash'])
|
||||
txs.append(Transaction(
|
||||
raw=row['raw'], height=row['height'], position=row['position'],
|
||||
timestamp=row['timestamp'], is_verified=bool(row['is_verified'])
|
||||
))
|
||||
for txi in txs[-1].inputs:
|
||||
txi_txoids.append(txi.txo_ref.hash)
|
||||
|
||||
annotated_txos = {}
|
||||
for offset in range(0, len(txids), MAX_QUERY_VARIABLES):
|
||||
annotated_txos.update({
|
||||
txo.id: txo for txo in
|
||||
get_txos(
|
||||
wallet_account_ids=account_ids,
|
||||
tx_hash__in=txids[offset:offset + MAX_QUERY_VARIABLES], order_by='txo.tx_hash',
|
||||
include_is_my_input=include_is_my_input,
|
||||
include_is_my_output=include_is_my_output,
|
||||
)[0]
|
||||
})
|
||||
|
||||
referenced_txos = {}
|
||||
for offset in range(0, len(txi_txoids), MAX_QUERY_VARIABLES):
|
||||
referenced_txos.update({
|
||||
txo.id: txo for txo in
|
||||
get_txos(
|
||||
wallet_account_ids=account_ids,
|
||||
txo_hash__in=txi_txoids[offset:offset + MAX_QUERY_VARIABLES], order_by='txo.txo_hash',
|
||||
include_is_my_output=include_is_my_output,
|
||||
)[0]
|
||||
})
|
||||
|
||||
for tx in txs:
|
||||
for txi in tx.inputs:
|
||||
txo = referenced_txos.get(txi.txo_ref.id)
|
||||
if txo:
|
||||
txi.txo_ref = txo.ref
|
||||
for txo in tx.outputs:
|
||||
_txo = annotated_txos.get(txo.id)
|
||||
if _txo:
|
||||
txo.update_annotations(_txo)
|
||||
else:
|
||||
txo.update_annotations(TXO_NOT_MINE)
|
||||
|
||||
for tx in txs:
|
||||
txos = tx.outputs
|
||||
if len(txos) >= 2 and txos[1].can_decode_purchase_data:
|
||||
txos[0].purchase = txos[1]
|
||||
|
||||
return txs, get_transaction_count(**constraints) if include_total else None
|
||||
|
||||
|
||||
def get_transaction_count(**constraints):
|
||||
constraints.pop('wallet', None)
|
||||
constraints.pop('offset', None)
|
||||
constraints.pop('limit', None)
|
||||
constraints.pop('order_by', None)
|
||||
count = select_transactions([func.count().label('total')], **constraints)
|
||||
return count[0]['total'] or 0
|
||||
|
||||
|
||||
BASE_SELECT_TXO_COLUMNS = [
|
||||
TX.c.tx_hash, TX.c.raw, TX.c.height, TX.c.position.label('tx_position'),
|
||||
TX.c.is_verified, TX.c.timestamp,
|
||||
TXO.c.txo_type, TXO.c.position.label('txo_position'), TXO.c.amount, TXO.c.spent_height,
|
||||
TXO.c.script_offset, TXO.c.script_length,
|
||||
]
|
||||
|
||||
|
||||
def select_txos(
|
||||
cols=None, account_ids=None, is_my_input=None,
|
||||
is_my_output=True, is_my_input_or_output=None, exclude_internal_transfers=False,
|
||||
include_is_my_input=False, claim_id_not_in_claim_table=None,
|
||||
txo_id_not_in_claim_table=None, txo_id_not_in_support_table=None,
|
||||
**constraints
|
||||
) -> Select:
|
||||
if cols is None:
|
||||
cols = BASE_SELECT_TXO_COLUMNS
|
||||
s: Select = select(*cols)
|
||||
if account_ids:
|
||||
my_addresses = select(AccountAddress.c.address).where(in_account_ids(account_ids))
|
||||
if is_my_input_or_output:
|
||||
include_is_my_input = True
|
||||
s = s.where(
|
||||
TXO.c.address.in_(my_addresses) | (
|
||||
(TXI.c.address.isnot(None)) &
|
||||
(TXI.c.address.in_(my_addresses))
|
||||
)
|
||||
)
|
||||
else:
|
||||
if is_my_output:
|
||||
s = s.where(TXO.c.address.in_(my_addresses))
|
||||
elif is_my_output is False:
|
||||
s = s.where(TXO.c.address.notin_(my_addresses))
|
||||
if is_my_input:
|
||||
include_is_my_input = True
|
||||
s = s.where(
|
||||
(TXI.c.address.isnot(None)) &
|
||||
(TXI.c.address.in_(my_addresses))
|
||||
)
|
||||
elif is_my_input is False:
|
||||
include_is_my_input = True
|
||||
s = s.where(
|
||||
(TXI.c.address.is_(None)) |
|
||||
(TXI.c.address.notin_(my_addresses))
|
||||
)
|
||||
if exclude_internal_transfers:
|
||||
include_is_my_input = True
|
||||
s = s.where(
|
||||
(TXO.c.txo_type != TXO_TYPES['other']) |
|
||||
(TXO.c.address.notin_(my_addresses))
|
||||
(TXI.c.address.is_(None)) |
|
||||
(TXI.c.address.notin_(my_addresses))
|
||||
)
|
||||
joins = TXO.join(TX)
|
||||
if constraints.pop('is_spent', None) is False:
|
||||
s = s.where((TXO.c.spent_height == 0) & (TXO.c.is_reserved == false()))
|
||||
if include_is_my_input:
|
||||
joins = joins.join(TXI, (TXI.c.position == 0) & (TXI.c.tx_hash == TXO.c.tx_hash), isouter=True)
|
||||
if claim_id_not_in_claim_table:
|
||||
s = s.where(TXO.c.claim_hash.notin_(select(Claim.c.claim_hash)))
|
||||
elif txo_id_not_in_claim_table:
|
||||
s = s.where(TXO.c.txo_hash.notin_(select(Claim.c.txo_hash)))
|
||||
elif txo_id_not_in_support_table:
|
||||
s = s.where(TXO.c.txo_hash.notin_(select(Support.c.txo_hash)))
|
||||
return query([TXO, TX], s.select_from(joins), **constraints)
|
||||
|
||||
|
||||
META_ATTRS = (
|
||||
'activation_height', 'takeover_height', 'creation_height', 'staked_amount',
|
||||
'short_url', 'canonical_url', 'staked_support_amount', 'staked_support_count',
|
||||
'signed_claim_count', 'signed_support_count', 'is_signature_valid',
|
||||
'trend_group', 'trend_mixed', 'trend_local', 'trend_global',
|
||||
'reposted_count', 'expiration_height',
|
||||
)
|
||||
|
||||
|
||||
def rows_to_txos(rows: List[dict], include_tx=True) -> List[Output]:
|
||||
txos = []
|
||||
tx_cache = {}
|
||||
for row in rows:
|
||||
if include_tx:
|
||||
if row['tx_hash'] not in tx_cache:
|
||||
tx_cache[row['tx_hash']] = Transaction(
|
||||
row['raw'], height=row['height'], position=row['tx_position'],
|
||||
timestamp=row['timestamp'],
|
||||
is_verified=bool(row['is_verified']),
|
||||
)
|
||||
txo = tx_cache[row['tx_hash']].outputs[row['txo_position']]
|
||||
else:
|
||||
source = row['raw'][row['script_offset']:row['script_offset']+row['script_length']]
|
||||
txo = Output(
|
||||
amount=row['amount'],
|
||||
script=OutputScript(source),
|
||||
tx_ref=TXRefImmutable.from_hash(row['tx_hash'], row['height'], row['timestamp']),
|
||||
position=row['txo_position'],
|
||||
)
|
||||
txo.spent_height = row['spent_height']
|
||||
if 'is_my_input' in row:
|
||||
txo.is_my_input = bool(row['is_my_input'])
|
||||
if 'is_my_output' in row:
|
||||
txo.is_my_output = bool(row['is_my_output'])
|
||||
if 'is_my_input' in row and 'is_my_output' in row:
|
||||
if txo.is_my_input and txo.is_my_output and row['txo_type'] == TXO_TYPES['other']:
|
||||
txo.is_internal_transfer = True
|
||||
else:
|
||||
txo.is_internal_transfer = False
|
||||
if 'received_tips' in row:
|
||||
txo.received_tips = row['received_tips']
|
||||
for attr in META_ATTRS:
|
||||
if attr in row:
|
||||
txo.meta[attr] = row[attr]
|
||||
txos.append(txo)
|
||||
return txos
|
||||
|
||||
|
||||
def get_txos(no_tx=False, include_total=False, **constraints) -> Tuple[List[Output], Optional[int]]:
|
||||
wallet_account_ids = constraints.pop('wallet_account_ids', [])
|
||||
include_is_my_input = constraints.get('include_is_my_input', False)
|
||||
include_is_my_output = constraints.pop('include_is_my_output', False)
|
||||
include_received_tips = constraints.pop('include_received_tips', False)
|
||||
|
||||
select_columns = BASE_SELECT_TXO_COLUMNS + [
|
||||
TXO.c.claim_name
|
||||
]
|
||||
|
||||
my_accounts = None
|
||||
if wallet_account_ids:
|
||||
my_accounts = select(AccountAddress.c.address).where(in_account_ids(wallet_account_ids))
|
||||
|
||||
if include_is_my_output and my_accounts is not None:
|
||||
if constraints.get('is_my_output', None) in (True, False):
|
||||
select_columns.append(text(f"{1 if constraints['is_my_output'] else 0} AS is_my_output"))
|
||||
else:
|
||||
select_columns.append(TXO.c.address.in_(my_accounts).label('is_my_output'))
|
||||
|
||||
if include_is_my_input and my_accounts is not None:
|
||||
if constraints.get('is_my_input', None) in (True, False):
|
||||
select_columns.append(text(f"{1 if constraints['is_my_input'] else 0} AS is_my_input"))
|
||||
else:
|
||||
select_columns.append((
|
||||
(TXI.c.address.isnot(None)) &
|
||||
(TXI.c.address.in_(my_accounts))
|
||||
).label('is_my_input'))
|
||||
|
||||
if include_received_tips:
|
||||
support = TXO.alias('support')
|
||||
select_columns.append(
|
||||
select(func.coalesce(func.sum(support.c.amount), 0))
|
||||
.select_from(support).where(
|
||||
(support.c.claim_hash == TXO.c.claim_hash) &
|
||||
(support.c.txo_type == TXO_TYPES['support']) &
|
||||
(support.c.address.in_(my_accounts)) &
|
||||
(support.c.txo_hash.notin_(select(TXI.c.txo_hash)))
|
||||
).label('received_tips')
|
||||
)
|
||||
|
||||
if 'order_by' not in constraints or constraints['order_by'] == 'height':
|
||||
constraints['order_by'] = [
|
||||
"tx.height=0 DESC", "tx.height DESC", "tx.position DESC", "txo.position"
|
||||
]
|
||||
elif constraints.get('order_by', None) == 'none':
|
||||
del constraints['order_by']
|
||||
|
||||
rows = context().fetchall(select_txos(select_columns, **constraints))
|
||||
txos = rows_to_txos(rows, not no_tx)
|
||||
|
||||
channel_hashes = set()
|
||||
for txo in txos:
|
||||
if txo.is_claim and txo.can_decode_claim:
|
||||
if txo.claim.is_signed:
|
||||
channel_hashes.add(txo.claim.signing_channel_hash)
|
||||
|
||||
if channel_hashes:
|
||||
channels = {
|
||||
txo.claim_hash: txo for txo in
|
||||
get_txos(
|
||||
txo_type=TXO_TYPES['channel'], spent_height=0,
|
||||
wallet_account_ids=wallet_account_ids, claim_hash__in=channel_hashes
|
||||
)[0]
|
||||
}
|
||||
for txo in txos:
|
||||
if txo.is_claim and txo.can_decode_claim:
|
||||
txo.channel = channels.get(txo.claim.signing_channel_hash, None)
|
||||
|
||||
return txos, get_txo_count(**constraints) if include_total else None
|
||||
|
||||
|
||||
def _clean_txo_constraints_for_aggregation(constraints):
|
||||
constraints.pop('include_is_my_input', None)
|
||||
constraints.pop('include_is_my_output', None)
|
||||
constraints.pop('include_received_tips', None)
|
||||
constraints.pop('wallet_account_ids', None)
|
||||
constraints.pop('offset', None)
|
||||
constraints.pop('limit', None)
|
||||
constraints.pop('order_by', None)
|
||||
|
||||
|
||||
def get_txo_count(**constraints):
|
||||
_clean_txo_constraints_for_aggregation(constraints)
|
||||
count = context().fetchall(select_txos([func.count().label('total')], **constraints))
|
||||
return count[0]['total'] or 0
|
||||
|
||||
|
||||
def get_txo_sum(**constraints):
|
||||
_clean_txo_constraints_for_aggregation(constraints)
|
||||
result = context().fetchall(select_txos([func.sum(TXO.c.amount).label('total')], **constraints))
|
||||
return result[0]['total'] or 0
|
||||
|
||||
|
||||
def get_balance(account_ids):
|
||||
ctx = context()
|
||||
my_addresses = select(AccountAddress.c.address).where(in_account_ids(account_ids))
|
||||
if ctx.is_postgres:
|
||||
txo_address_check = TXO.c.address == func.any(func.array(my_addresses))
|
||||
txi_address_check = TXI.c.address == func.any(func.array(my_addresses))
|
||||
else:
|
||||
txo_address_check = TXO.c.address.in_(my_addresses)
|
||||
txi_address_check = TXI.c.address.in_(my_addresses)
|
||||
s: Select = (
|
||||
select(
|
||||
func.coalesce(func.sum(TXO.c.amount), 0).label("total"),
|
||||
func.coalesce(func.sum(case(
|
||||
[(TXO.c.txo_type != TXO_TYPES["other"], TXO.c.amount)],
|
||||
)), 0).label("reserved"),
|
||||
func.coalesce(func.sum(case(
|
||||
[(where_txo_type_in(CLAIM_TYPE_CODES), TXO.c.amount)],
|
||||
)), 0).label("claims"),
|
||||
func.coalesce(func.sum(case(
|
||||
[(where_txo_type_in(TXO_TYPES["support"]), TXO.c.amount)],
|
||||
)), 0).label("supports"),
|
||||
func.coalesce(func.sum(case(
|
||||
[(where_txo_type_in(TXO_TYPES["support"]) & (
|
||||
(TXI.c.address.isnot(None)) & txi_address_check
|
||||
), TXO.c.amount)],
|
||||
)), 0).label("my_supports"),
|
||||
)
|
||||
.where((TXO.c.spent_height == 0) & txo_address_check)
|
||||
.select_from(
|
||||
TXO.join(TXI, (TXI.c.position == 0) & (TXI.c.tx_hash == TXO.c.tx_hash), isouter=True)
|
||||
)
|
||||
)
|
||||
result = ctx.fetchone(s)
|
||||
return {
|
||||
"total": result["total"],
|
||||
"available": result["total"] - result["reserved"],
|
||||
"reserved": result["reserved"],
|
||||
"reserved_subtotals": {
|
||||
"claims": result["claims"],
|
||||
"supports": result["my_supports"],
|
||||
"tips": result["supports"] - result["my_supports"]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def get_report(account_ids):
|
||||
return
|
||||
|
||||
|
||||
def get_txo_plot(start_day=None, days_back=0, end_day=None, days_after=None, **constraints):
|
||||
_clean_txo_constraints_for_aggregation(constraints)
|
||||
if start_day is None:
|
||||
# TODO: Fix
|
||||
current_ordinal = 0 # self.ledger.headers.estimated_date(self.ledger.headers.height).toordinal()
|
||||
constraints['day__gte'] = current_ordinal - days_back
|
||||
else:
|
||||
constraints['day__gte'] = date.fromisoformat(start_day).toordinal()
|
||||
if end_day is not None:
|
||||
constraints['day__lte'] = date.fromisoformat(end_day).toordinal()
|
||||
elif days_after is not None:
|
||||
constraints['day__lte'] = constraints['day__gte'] + days_after
|
||||
plot = context().fetchall(select_txos(
|
||||
[TX.c.day, func.sum(TXO.c.amount).label('total')],
|
||||
group_by='day', order_by='day', **constraints
|
||||
))
|
||||
for row in plot:
|
||||
row['day'] = date.fromordinal(row['day'])
|
||||
return plot
|
||||
|
||||
|
||||
def get_purchases(**constraints) -> Tuple[List[Output], Optional[int]]:
|
||||
accounts = constraints.pop('accounts', None)
|
||||
assert accounts, "'accounts' argument required to find purchases"
|
||||
if not {'purchased_claim_hash', 'purchased_claim_hash__in'}.intersection(constraints):
|
||||
constraints['purchased_claim_hash__is_not_null'] = True
|
||||
constraints['tx_hash__in'] = (
|
||||
select(TXI.c.tx_hash).select_from(txi_join_account).where(in_account_ids(accounts))
|
||||
)
|
||||
txs, count = get_transactions(**constraints)
|
||||
return [tx.outputs[0] for tx in txs], count
|
||||
|
||||
|
||||
def get_supports_summary(self, **constraints):
|
||||
return get_txos(
|
||||
txo_type=TXO_TYPES['support'],
|
||||
spent_height=0, is_my_output=True,
|
||||
include_is_my_input=True,
|
||||
no_tx=True,
|
||||
**constraints
|
||||
)
|
||||
|
||||
|
||||
def reserve_outputs(txo_hashes, is_reserved=True):
|
||||
context().execute(
|
||||
TXO.update()
|
||||
.values(is_reserved=is_reserved)
|
||||
.where(TXO.c.txo_hash.in_(txo_hashes))
|
||||
)
|
||||
|
||||
|
||||
def release_all_outputs(account_id):
|
||||
context().execute(
|
||||
TXO.update().values(is_reserved=False).where(
|
||||
TXO.c.is_reserved & TXO.c.address.in_(
|
||||
select(AccountAddress.c.address).where(in_account_ids(account_id))
|
||||
)
|
||||
)
|
||||
)
|
||||
@@ -1,24 +0,0 @@
|
||||
from sqlalchemy import exists
|
||||
from sqlalchemy.future import select
|
||||
|
||||
from ..query_context import context
|
||||
from ..tables import Wallet
|
||||
|
||||
|
||||
def has_wallet(wallet_id: str) -> bool:
|
||||
sql = select(exists(select(Wallet.c.wallet_id).where(Wallet.c.wallet_id == wallet_id)))
|
||||
return context().execute(sql).fetchone()[0]
|
||||
|
||||
|
||||
def get_wallet(wallet_id: str):
|
||||
return context().fetchone(
|
||||
select(Wallet.c.data).where(Wallet.c.wallet_id == wallet_id)
|
||||
)
|
||||
|
||||
|
||||
def add_wallet(wallet_id: str, data: str):
|
||||
c = context()
|
||||
c.execute(
|
||||
c.insert_or_replace(Wallet, ["data"])
|
||||
.values(wallet_id=wallet_id, data=data)
|
||||
)
|
||||
@@ -1,766 +0,0 @@
|
||||
import os
|
||||
import time
|
||||
import traceback
|
||||
import functools
|
||||
from io import BytesIO
|
||||
import multiprocessing as mp
|
||||
from decimal import Decimal
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from dataclasses import dataclass, field
|
||||
from contextvars import ContextVar
|
||||
|
||||
from sqlalchemy import create_engine, inspect, bindparam, func, case, exists, event as sqlalchemy_event
|
||||
from sqlalchemy.future import select
|
||||
from sqlalchemy.engine import Engine
|
||||
from sqlalchemy.sql import Insert, text
|
||||
try:
|
||||
from pgcopy import CopyManager
|
||||
except ImportError:
|
||||
CopyManager = None
|
||||
|
||||
from lbry.event import EventQueuePublisher
|
||||
from lbry.blockchain.ledger import Ledger
|
||||
from lbry.blockchain.transaction import Transaction, Output, Input
|
||||
from lbry.schema.tags import clean_tags
|
||||
from lbry.schema.result import Censor
|
||||
from lbry.schema.mime_types import guess_stream_type
|
||||
|
||||
from .utils import pg_insert
|
||||
from .tables import (
|
||||
Block, BlockFilter,
|
||||
TX, TXFilter, TXO, TXI, Claim, Tag, Support
|
||||
)
|
||||
from .constants import TXO_TYPES, STREAM_TYPES
|
||||
|
||||
|
||||
_context: ContextVar['QueryContext'] = ContextVar('_context')
|
||||
|
||||
|
||||
@dataclass
|
||||
class QueryContext:
|
||||
engine: Engine
|
||||
ledger: Ledger
|
||||
message_queue: mp.Queue
|
||||
stop_event: mp.Event
|
||||
stack: List[List]
|
||||
metrics: Dict
|
||||
is_tracking_metrics: bool
|
||||
blocked_streams: Dict
|
||||
blocked_channels: Dict
|
||||
filtered_streams: Dict
|
||||
filtered_channels: Dict
|
||||
pid: int
|
||||
|
||||
# QueryContext __enter__/__exit__ state
|
||||
current_timer_name: Optional[str] = None
|
||||
current_timer_time: float = 0
|
||||
current_progress: Optional['ProgressContext'] = None
|
||||
|
||||
copy_managers: Dict[str, CopyManager] = field(default_factory=dict)
|
||||
_variable_limit: Optional[int] = None
|
||||
|
||||
@property
|
||||
def is_postgres(self):
|
||||
return self.engine.dialect.name == 'postgresql'
|
||||
|
||||
@property
|
||||
def is_sqlite(self):
|
||||
return self.engine.dialect.name == 'sqlite'
|
||||
|
||||
@property
|
||||
def variable_limit(self):
|
||||
if self._variable_limit is not None:
|
||||
return self._variable_limit
|
||||
if self.is_sqlite:
|
||||
for result in self.fetchall(text('PRAGMA COMPILE_OPTIONS;')):
|
||||
for _, value in result.items():
|
||||
if value.startswith('MAX_VARIABLE_NUMBER'):
|
||||
self._variable_limit = int(value.split('=')[1])
|
||||
return self._variable_limit
|
||||
self._variable_limit = 999 # todo: default for 3.32.0 is 32766, but we are still hitting 999 somehow
|
||||
else:
|
||||
self._variable_limit = 32766
|
||||
return self._variable_limit
|
||||
|
||||
def raise_unsupported_dialect(self):
|
||||
raise RuntimeError(f'Unsupported database dialect: {self.engine.dialect.name}.')
|
||||
|
||||
@classmethod
|
||||
def get_resolve_censor(cls) -> Censor:
|
||||
return Censor(Censor.RESOLVE)
|
||||
|
||||
@classmethod
|
||||
def get_search_censor(cls) -> Censor:
|
||||
return Censor(Censor.SEARCH)
|
||||
|
||||
def pg_copy(self, table, rows):
|
||||
with self.engine.begin() as c:
|
||||
copy_manager = self.copy_managers.get(table.name)
|
||||
if copy_manager is None:
|
||||
self.copy_managers[table.name] = copy_manager = CopyManager(
|
||||
c.connection, table.name, rows[0].keys()
|
||||
)
|
||||
copy_manager.conn = c.connection
|
||||
copy_manager.copy(map(dict.values, rows), BytesIO)
|
||||
copy_manager.conn = None
|
||||
|
||||
def pg_has_pk_constraint(self, table_name):
|
||||
claim_constraints = self.fetchall(text(
|
||||
f"select * from information_schema.table_constraints as tc "
|
||||
f"where tc.table_name='{table_name}' and constraint_type='PRIMARY KEY'"
|
||||
))
|
||||
return len(claim_constraints) > 0
|
||||
|
||||
def connect_without_transaction(self):
|
||||
return self.engine.connect().execution_options(isolation_level="AUTOCOMMIT")
|
||||
|
||||
def connect_streaming(self):
|
||||
return self.engine.connect().execution_options(stream_results=True)
|
||||
|
||||
def execute_notx(self, sql, *args):
|
||||
with self.connect_without_transaction() as c:
|
||||
return c.execute(sql, *args)
|
||||
|
||||
def execute(self, sql, *args):
|
||||
with self.engine.begin() as c:
|
||||
return c.execute(sql, *args)
|
||||
|
||||
def fetchone(self, sql, *args):
|
||||
with self.engine.begin() as c:
|
||||
row = c.execute(sql, *args).fetchone()
|
||||
return dict(row._mapping) if row else row
|
||||
|
||||
def fetchall(self, sql, *args):
|
||||
with self.engine.begin() as c:
|
||||
rows = c.execute(sql, *args).fetchall()
|
||||
return [dict(row._mapping) for row in rows]
|
||||
|
||||
def fetchtotal(self, condition) -> int:
|
||||
sql = select(func.count('*').label('total')).where(condition)
|
||||
return self.fetchone(sql)['total']
|
||||
|
||||
def fetchmax(self, column, default: int) -> int:
|
||||
sql = select(func.coalesce(func.max(column), default).label('max_result'))
|
||||
return self.fetchone(sql)['max_result']
|
||||
|
||||
def has_records(self, table) -> bool:
|
||||
sql = select(exists([1], from_obj=table).label('result'))
|
||||
return bool(self.fetchone(sql)['result'])
|
||||
|
||||
def insert_or_ignore(self, table):
|
||||
if self.is_sqlite:
|
||||
return table.insert().prefix_with("OR IGNORE")
|
||||
elif self.is_postgres:
|
||||
return pg_insert(table).on_conflict_do_nothing()
|
||||
else:
|
||||
self.raise_unsupported_dialect()
|
||||
|
||||
def insert_or_replace(self, table, replace):
|
||||
if self.is_sqlite:
|
||||
return table.insert().prefix_with("OR REPLACE")
|
||||
elif self.is_postgres:
|
||||
insert = pg_insert(table)
|
||||
return insert.on_conflict_do_update(
|
||||
table.primary_key, set_={col: getattr(insert.excluded, col) for col in replace}
|
||||
)
|
||||
else:
|
||||
self.raise_unsupported_dialect()
|
||||
|
||||
def has_table(self, table):
|
||||
return inspect(self.engine).has_table(table)
|
||||
|
||||
def get_bulk_loader(self) -> 'BulkLoader':
|
||||
return BulkLoader(self)
|
||||
|
||||
def reset_metrics(self):
|
||||
self.stack = []
|
||||
self.metrics = {}
|
||||
|
||||
def with_timer(self, timer_name: str) -> 'QueryContext':
|
||||
self.current_timer_name = timer_name
|
||||
return self
|
||||
|
||||
@property
|
||||
def elapsed(self):
|
||||
return time.perf_counter() - self.current_timer_time
|
||||
|
||||
def __enter__(self) -> 'QueryContext':
|
||||
self.current_timer_time = time.perf_counter()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.current_timer_name = None
|
||||
self.current_timer_time = 0
|
||||
self.current_progress = None
|
||||
|
||||
|
||||
def context(with_timer: str = None) -> 'QueryContext':
|
||||
if isinstance(with_timer, str):
|
||||
return _context.get().with_timer(with_timer)
|
||||
return _context.get()
|
||||
|
||||
|
||||
def set_postgres_settings(connection, _):
|
||||
cursor = connection.cursor()
|
||||
cursor.execute('SET work_mem="500MB";')
|
||||
cursor.execute('COMMIT;')
|
||||
cursor.close()
|
||||
|
||||
|
||||
def set_sqlite_settings(connection, _):
|
||||
connection.isolation_level = None
|
||||
cursor = connection.cursor()
|
||||
cursor.execute('PRAGMA journal_mode=WAL;')
|
||||
cursor.close()
|
||||
|
||||
|
||||
def do_sqlite_begin(connection):
|
||||
# see: https://bit.ly/3j4vvXm
|
||||
connection.exec_driver_sql("BEGIN")
|
||||
|
||||
|
||||
def initialize(
|
||||
ledger: Ledger, message_queue: mp.Queue, stop_event: mp.Event,
|
||||
track_metrics=False, block_and_filter=None):
|
||||
url = ledger.conf.db_url_or_default
|
||||
engine = create_engine(url)
|
||||
if engine.name == "postgresql":
|
||||
sqlalchemy_event.listen(engine, "connect", set_postgres_settings)
|
||||
elif engine.name == "sqlite":
|
||||
sqlalchemy_event.listen(engine, "connect", set_sqlite_settings)
|
||||
sqlalchemy_event.listen(engine, "begin", do_sqlite_begin)
|
||||
if block_and_filter is not None:
|
||||
blocked_streams, blocked_channels, filtered_streams, filtered_channels = block_and_filter
|
||||
else:
|
||||
blocked_streams = blocked_channels = filtered_streams = filtered_channels = {}
|
||||
_context.set(
|
||||
QueryContext(
|
||||
pid=os.getpid(), engine=engine,
|
||||
ledger=ledger, message_queue=message_queue, stop_event=stop_event,
|
||||
stack=[], metrics={}, is_tracking_metrics=track_metrics,
|
||||
blocked_streams=blocked_streams, blocked_channels=blocked_channels,
|
||||
filtered_streams=filtered_streams, filtered_channels=filtered_channels,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def uninitialize():
|
||||
ctx = _context.get(None)
|
||||
if ctx is not None:
|
||||
ctx.engine.dispose()
|
||||
_context.set(None)
|
||||
|
||||
|
||||
class Event:
|
||||
_events: List['Event'] = []
|
||||
__slots__ = 'id', 'name', 'units'
|
||||
|
||||
def __init__(self, name: str, units: Tuple[str]):
|
||||
self.id = None
|
||||
self.name = name
|
||||
self.units = units
|
||||
|
||||
@classmethod
|
||||
def get_by_id(cls, event_id) -> 'Event':
|
||||
return cls._events[event_id]
|
||||
|
||||
@classmethod
|
||||
def get_by_name(cls, name) -> 'Event':
|
||||
for event in cls._events:
|
||||
if event.name == name:
|
||||
return event
|
||||
|
||||
@classmethod
|
||||
def add(cls, name: str, *units: str) -> 'Event':
|
||||
assert cls.get_by_name(name) is None, f"Event {name} already exists."
|
||||
assert name.count('.') == 3, f"Event {name} does not follow pattern of: [module].sync.[phase].[task]"
|
||||
event = cls(name, units)
|
||||
cls._events.append(event)
|
||||
event.id = cls._events.index(event)
|
||||
return event
|
||||
|
||||
|
||||
def event_emitter(name: str, *units: str, throttle=1):
|
||||
event = Event.add(name, *units)
|
||||
|
||||
def wrapper(f):
|
||||
@functools.wraps(f)
|
||||
def with_progress(*args, **kwargs):
|
||||
with progress(event, throttle=throttle) as p:
|
||||
try:
|
||||
return f(*args, **kwargs, p=p)
|
||||
except BreakProgress:
|
||||
raise
|
||||
except:
|
||||
traceback.print_exc()
|
||||
raise
|
||||
return with_progress
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
class ProgressPublisher(EventQueuePublisher):
|
||||
|
||||
def message_to_event(self, message):
|
||||
total, extra = None, None
|
||||
if len(message) == 3:
|
||||
event_id, progress_id, done = message
|
||||
elif len(message) == 5:
|
||||
event_id, progress_id, done, total, extra = message
|
||||
else:
|
||||
raise TypeError("progress message must be tuple of 3 or 5 values.")
|
||||
event = Event.get_by_id(event_id)
|
||||
d = {
|
||||
"event": event.name,
|
||||
"data": {"id": progress_id, "done": done}
|
||||
}
|
||||
if total is not None:
|
||||
d['data']['total'] = total
|
||||
d['data']['units'] = event.units
|
||||
if isinstance(extra, dict):
|
||||
d['data'].update(extra)
|
||||
return d
|
||||
|
||||
|
||||
class BreakProgress(Exception):
|
||||
"""Break out of progress when total is 0."""
|
||||
|
||||
|
||||
class Progress:
|
||||
|
||||
def __init__(self, message_queue: mp.Queue, event: Event, throttle=1):
|
||||
self.message_queue = message_queue
|
||||
self.event = event
|
||||
self.progress_id = 0
|
||||
self.throttle = throttle
|
||||
self.last_done = (0,)*len(event.units)
|
||||
self.last_done_queued = (0,)*len(event.units)
|
||||
self.totals = (0,)*len(event.units)
|
||||
|
||||
def __enter__(self) -> 'Progress':
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
if self.last_done != self.last_done_queued:
|
||||
self.message_queue.put((self.event.id, self.progress_id, self.last_done))
|
||||
self.last_done_queued = self.last_done
|
||||
if exc_type == BreakProgress:
|
||||
return True
|
||||
if self.last_done != self.totals: # or exc_type is not None:
|
||||
# TODO: add exception info into closing message if there is any
|
||||
self.message_queue.put((
|
||||
self.event.id, self.progress_id, (-1,)*len(self.event.units)
|
||||
))
|
||||
|
||||
def start(self, *totals: int, progress_id=0, label=None, extra=None):
|
||||
assert len(totals) == len(self.event.units), \
|
||||
f"Totals {totals} do not match up with units {self.event.units}."
|
||||
if not any(totals):
|
||||
raise BreakProgress
|
||||
self.totals = totals
|
||||
self.progress_id = progress_id
|
||||
extra = {} if extra is None else extra.copy()
|
||||
if label is not None:
|
||||
extra['label'] = label
|
||||
self.step(*((0,)*len(totals)), force=True, extra=extra)
|
||||
|
||||
def step(self, *done: int, force=False, extra=None):
|
||||
if done == ():
|
||||
assert len(self.totals) == 1, "Incrementing step() only works with one unit progress."
|
||||
done = (self.last_done[0]+1,)
|
||||
assert len(done) == len(self.totals), \
|
||||
f"Done elements {done} don't match total elements {self.totals}."
|
||||
self.last_done = done
|
||||
send_condition = force or extra is not None or (
|
||||
# throttle rate of events being generated (only throttles first unit value)
|
||||
(self.throttle == 1 or done[0] % self.throttle == 0) and
|
||||
# deduplicate finish event by not sending a step where done == total
|
||||
any(i < j for i, j in zip(done, self.totals)) and
|
||||
# deduplicate same event
|
||||
done != self.last_done_queued
|
||||
)
|
||||
if send_condition:
|
||||
if extra is not None:
|
||||
self.message_queue.put_nowait(
|
||||
(self.event.id, self.progress_id, done, self.totals, extra)
|
||||
)
|
||||
else:
|
||||
self.message_queue.put_nowait(
|
||||
(self.event.id, self.progress_id, done)
|
||||
)
|
||||
self.last_done_queued = done
|
||||
|
||||
def add(self, *done: int, force=False, extra=None):
|
||||
assert len(done) == len(self.last_done), \
|
||||
f"Done elements {done} don't match total elements {self.last_done}."
|
||||
self.step(
|
||||
*(i+j for i, j in zip(self.last_done, done)),
|
||||
force=force, extra=extra
|
||||
)
|
||||
|
||||
def iter(self, items: List):
|
||||
self.start(len(items))
|
||||
for item in items:
|
||||
yield item
|
||||
self.step()
|
||||
|
||||
|
||||
class ProgressContext(Progress):
|
||||
|
||||
def __init__(self, ctx: QueryContext, event: Event, throttle=1):
|
||||
super().__init__(ctx.message_queue, event, throttle)
|
||||
self.ctx = ctx
|
||||
|
||||
def __enter__(self) -> 'ProgressContext':
|
||||
self.ctx.__enter__()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
return any((
|
||||
self.ctx.__exit__(exc_type, exc_val, exc_tb),
|
||||
super().__exit__(exc_type, exc_val, exc_tb)
|
||||
))
|
||||
|
||||
|
||||
def progress(e: Event, throttle=1) -> ProgressContext:
|
||||
ctx = context(e.name)
|
||||
ctx.current_progress = ProgressContext(ctx, e, throttle=throttle)
|
||||
return ctx.current_progress
|
||||
|
||||
|
||||
class BulkLoader:
|
||||
|
||||
def __init__(self, ctx: QueryContext):
|
||||
self.ctx = ctx
|
||||
self.ledger = ctx.ledger
|
||||
self.blocks = []
|
||||
self.txs = []
|
||||
self.txos = []
|
||||
self.txis = []
|
||||
self.supports = []
|
||||
self.claims = []
|
||||
self.tags = []
|
||||
self.update_claims = []
|
||||
self.delete_tags = []
|
||||
self.tx_filters = []
|
||||
self.block_filters = []
|
||||
|
||||
@staticmethod
|
||||
def block_to_row(block: Block) -> dict:
|
||||
return {
|
||||
'block_hash': block.block_hash,
|
||||
'previous_hash': block.prev_block_hash,
|
||||
'file_number': block.file_number,
|
||||
'height': 0 if block.is_first_block else block.height,
|
||||
'timestamp': block.timestamp,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def tx_to_row(block_hash: bytes, tx: Transaction) -> dict:
|
||||
row = {
|
||||
'tx_hash': tx.hash,
|
||||
#'block_hash': block_hash,
|
||||
'raw': tx.raw,
|
||||
'height': tx.height,
|
||||
'position': tx.position,
|
||||
'is_verified': tx.is_verified,
|
||||
'timestamp': tx.timestamp,
|
||||
'day': tx.day,
|
||||
'purchased_claim_hash': None,
|
||||
}
|
||||
txos = tx.outputs
|
||||
if len(txos) >= 2 and txos[1].can_decode_purchase_data:
|
||||
txos[0].purchase = txos[1]
|
||||
row['purchased_claim_hash'] = txos[1].purchase_data.claim_hash
|
||||
return row
|
||||
|
||||
@staticmethod
|
||||
def txi_to_row(tx: Transaction, txi: Input) -> dict:
|
||||
return {
|
||||
'tx_hash': tx.hash,
|
||||
'txo_hash': txi.txo_ref.hash,
|
||||
'position': txi.position,
|
||||
'height': tx.height,
|
||||
}
|
||||
|
||||
def txo_to_row(self, tx: Transaction, txo: Output) -> dict:
|
||||
row = {
|
||||
'tx_hash': tx.hash,
|
||||
'txo_hash': txo.hash,
|
||||
'address': txo.get_address(self.ledger) if txo.has_address else None,
|
||||
'position': txo.position,
|
||||
'amount': txo.amount,
|
||||
'height': tx.height,
|
||||
'script_offset': txo.script.offset,
|
||||
'script_length': txo.script.length,
|
||||
'txo_type': 0,
|
||||
'claim_id': None,
|
||||
'claim_hash': None,
|
||||
'claim_name': None,
|
||||
'channel_hash': None,
|
||||
'signature': None,
|
||||
'signature_digest': None,
|
||||
'reposted_claim_hash': None,
|
||||
'public_key': None,
|
||||
'public_key_hash': None
|
||||
}
|
||||
if txo.is_claim:
|
||||
if txo.can_decode_claim:
|
||||
claim = txo.claim
|
||||
row['txo_type'] = TXO_TYPES.get(claim.claim_type, TXO_TYPES['stream'])
|
||||
if claim.is_channel:
|
||||
row['public_key'] = claim.channel.public_key_bytes
|
||||
row['public_key_hash'] = self.ledger.address_to_hash160(
|
||||
self.ledger.public_key_to_address(claim.channel.public_key_bytes)
|
||||
)
|
||||
elif claim.is_repost:
|
||||
row['reposted_claim_hash'] = claim.repost.reference.claim_hash
|
||||
else:
|
||||
row['txo_type'] = TXO_TYPES['stream']
|
||||
elif txo.is_support:
|
||||
row['txo_type'] = TXO_TYPES['support']
|
||||
elif txo.purchase is not None:
|
||||
row['txo_type'] = TXO_TYPES['purchase']
|
||||
row['claim_id'] = txo.purchased_claim_id
|
||||
row['claim_hash'] = txo.purchased_claim_hash
|
||||
if txo.script.is_claim_involved:
|
||||
signable = txo.can_decode_signable
|
||||
if signable and signable.is_signed:
|
||||
row['channel_hash'] = signable.signing_channel_hash
|
||||
row['signature'] = txo.get_encoded_signature()
|
||||
row['signature_digest'] = txo.get_signature_digest(self.ledger)
|
||||
row['claim_id'] = txo.claim_id
|
||||
row['claim_hash'] = txo.claim_hash
|
||||
try:
|
||||
row['claim_name'] = txo.claim_name.replace('\x00', '')
|
||||
except UnicodeDecodeError:
|
||||
pass
|
||||
return row
|
||||
|
||||
def claim_to_rows(
|
||||
self, txo: Output, staked_support_amount: int, staked_support_count: int, reposted_count: int,
|
||||
signature: bytes = None, signature_digest: bytes = None, channel_public_key: bytes = None,
|
||||
) -> Tuple[dict, List]:
|
||||
|
||||
tx = txo.tx_ref
|
||||
d = {
|
||||
'claim_type': None,
|
||||
'address': txo.get_address(self.ledger),
|
||||
'txo_hash': txo.hash,
|
||||
'amount': txo.amount,
|
||||
'height': tx.height,
|
||||
'timestamp': tx.timestamp,
|
||||
# support
|
||||
'staked_amount': txo.amount + staked_support_amount,
|
||||
'staked_support_amount': staked_support_amount,
|
||||
'staked_support_count': staked_support_count,
|
||||
# basic metadata
|
||||
'title': None,
|
||||
'description': None,
|
||||
'author': None,
|
||||
# streams
|
||||
'stream_type': None,
|
||||
'media_type': None,
|
||||
'duration': None,
|
||||
'release_time': None,
|
||||
'fee_amount': 0,
|
||||
'fee_currency': None,
|
||||
# reposts
|
||||
'reposted_claim_hash': None,
|
||||
'reposted_count': reposted_count,
|
||||
# signed claims
|
||||
'channel_hash': None,
|
||||
'is_signature_valid': None,
|
||||
# channels (on last change) and streams (on last re-validation)
|
||||
'public_key_hash': None,
|
||||
'public_key_height': None,
|
||||
}
|
||||
|
||||
claim = txo.can_decode_claim
|
||||
if not claim:
|
||||
return d, []
|
||||
|
||||
if claim.is_stream:
|
||||
d['claim_type'] = TXO_TYPES['stream']
|
||||
d['media_type'] = claim.stream.source.media_type
|
||||
d['stream_type'] = STREAM_TYPES[guess_stream_type(d['media_type'])]
|
||||
d['title'] = claim.stream.title.replace('\x00', '')
|
||||
d['description'] = claim.stream.description.replace('\x00', '')
|
||||
d['author'] = claim.stream.author.replace('\x00', '')
|
||||
if claim.stream.video and claim.stream.video.duration:
|
||||
d['duration'] = claim.stream.video.duration
|
||||
if claim.stream.audio and claim.stream.audio.duration:
|
||||
d['duration'] = claim.stream.audio.duration
|
||||
if claim.stream.release_time:
|
||||
d['release_time'] = claim.stream.release_time
|
||||
if claim.stream.has_fee:
|
||||
fee = claim.stream.fee
|
||||
if isinstance(fee.amount, Decimal):
|
||||
d['fee_amount'] = int(fee.amount*1000)
|
||||
if isinstance(fee.currency, str):
|
||||
d['fee_currency'] = fee.currency.lower()
|
||||
elif claim.is_repost:
|
||||
d['claim_type'] = TXO_TYPES['repost']
|
||||
d['reposted_claim_hash'] = claim.repost.reference.claim_hash
|
||||
elif claim.is_channel:
|
||||
d['claim_type'] = TXO_TYPES['channel']
|
||||
d['public_key_hash'] = self.ledger.address_to_hash160(
|
||||
self.ledger.public_key_to_address(claim.channel.public_key_bytes)
|
||||
)
|
||||
if claim.is_signed:
|
||||
d['channel_hash'] = claim.signing_channel_hash
|
||||
d['is_signature_valid'] = (
|
||||
all((signature, signature_digest, channel_public_key)) and
|
||||
Output.is_signature_valid(
|
||||
signature, signature_digest, channel_public_key
|
||||
)
|
||||
)
|
||||
if channel_public_key:
|
||||
d['public_key_hash'] = self.ledger.address_to_hash160(
|
||||
self.ledger.public_key_to_address(channel_public_key)
|
||||
)
|
||||
|
||||
tags = []
|
||||
if claim.message.tags:
|
||||
claim_hash = txo.claim_hash
|
||||
tags = [
|
||||
{'claim_hash': claim_hash, 'tag': tag}
|
||||
for tag in clean_tags(claim.message.tags)
|
||||
]
|
||||
|
||||
return d, tags
|
||||
|
||||
def support_to_row(
|
||||
self, txo: Output, channel_public_key: bytes = None,
|
||||
signature: bytes = None, signature_digest: bytes = None
|
||||
):
|
||||
tx = txo.tx_ref
|
||||
d = {
|
||||
'txo_hash': txo.ref.hash,
|
||||
'claim_hash': txo.claim_hash,
|
||||
'address': txo.get_address(self.ledger),
|
||||
'amount': txo.amount,
|
||||
'height': tx.height,
|
||||
'timestamp': tx.timestamp,
|
||||
'emoji': None,
|
||||
'channel_hash': None,
|
||||
'is_signature_valid': None,
|
||||
}
|
||||
support = txo.can_decode_support
|
||||
if support:
|
||||
d['emoji'] = support.emoji
|
||||
if support.is_signed:
|
||||
d['channel_hash'] = support.signing_channel_hash
|
||||
d['is_signature_valid'] = (
|
||||
all((signature, signature_digest, channel_public_key)) and
|
||||
Output.is_signature_valid(
|
||||
signature, signature_digest, channel_public_key
|
||||
)
|
||||
)
|
||||
return d
|
||||
|
||||
def add_block(self, block: Block):
|
||||
self.blocks.append(self.block_to_row(block))
|
||||
for tx in block.txs:
|
||||
self.add_transaction(block.block_hash, tx)
|
||||
return self
|
||||
|
||||
def add_block_filter(self, height: int, factor: int, address_filter: bytes):
|
||||
self.block_filters.append({
|
||||
'height': height,
|
||||
'factor': factor,
|
||||
'address_filter': address_filter
|
||||
})
|
||||
|
||||
def add_transaction(self, block_hash: bytes, tx: Transaction):
|
||||
self.txs.append(self.tx_to_row(block_hash, tx))
|
||||
for txi in tx.inputs:
|
||||
if txi.coinbase is None:
|
||||
self.txis.append(self.txi_to_row(tx, txi))
|
||||
for txo in tx.outputs:
|
||||
self.txos.append(self.txo_to_row(tx, txo))
|
||||
return self
|
||||
|
||||
def add_transaction_filter(self, tx_hash: bytes, height: int, address_filter: bytes):
|
||||
self.tx_filters.append({
|
||||
'tx_hash': tx_hash,
|
||||
'height': height,
|
||||
'address_filter': address_filter
|
||||
})
|
||||
|
||||
def add_support(self, txo: Output, **extra):
|
||||
self.supports.append(self.support_to_row(txo, **extra))
|
||||
|
||||
def add_claim(
|
||||
self, txo: Output, short_url: str,
|
||||
creation_height: int, activation_height: int, expiration_height: int,
|
||||
takeover_height: int = None, **extra
|
||||
):
|
||||
try:
|
||||
claim_name = txo.claim_name.replace('\x00', '')
|
||||
normalized_name = txo.normalized_name
|
||||
except UnicodeDecodeError:
|
||||
claim_name = normalized_name = ''
|
||||
d, tags = self.claim_to_rows(txo, **extra)
|
||||
d['claim_hash'] = txo.claim_hash
|
||||
d['claim_id'] = txo.claim_id
|
||||
d['claim_name'] = claim_name
|
||||
d['normalized'] = normalized_name
|
||||
d['short_url'] = short_url
|
||||
d['creation_height'] = creation_height
|
||||
d['activation_height'] = activation_height
|
||||
d['expiration_height'] = expiration_height
|
||||
d['takeover_height'] = takeover_height
|
||||
d['is_controlling'] = takeover_height is not None
|
||||
if d['public_key_hash'] is not None:
|
||||
d['public_key_height'] = d['height']
|
||||
self.claims.append(d)
|
||||
self.tags.extend(tags)
|
||||
return self
|
||||
|
||||
def update_claim(self, txo: Output, public_key_height=None, **extra):
|
||||
d, tags = self.claim_to_rows(txo, **extra)
|
||||
d['pk'] = txo.claim_hash
|
||||
d['_public_key_height'] = public_key_height or d['height']
|
||||
d['_public_key_hash'] = d['public_key_hash']
|
||||
self.update_claims.append(d)
|
||||
self.delete_tags.append({'pk': txo.claim_hash})
|
||||
self.tags.extend(tags)
|
||||
return self
|
||||
|
||||
def get_queries(self):
|
||||
return (
|
||||
(Block.insert(), self.blocks),
|
||||
(BlockFilter.insert(), self.block_filters),
|
||||
(TX.insert(), self.txs),
|
||||
(TXFilter.insert(), self.tx_filters),
|
||||
(TXO.insert(), self.txos),
|
||||
(TXI.insert(), self.txis),
|
||||
(Claim.insert(), self.claims),
|
||||
(Tag.delete().where(Tag.c.claim_hash == bindparam('pk')), self.delete_tags),
|
||||
(Claim.update()
|
||||
.values(public_key_height=case([
|
||||
(bindparam('_public_key_hash').is_(None), None),
|
||||
(Claim.c.public_key_hash.is_(None) |
|
||||
(Claim.c.public_key_hash != bindparam('_public_key_hash')),
|
||||
bindparam('_public_key_height')),
|
||||
], else_=Claim.c.public_key_height))
|
||||
.where(Claim.c.claim_hash == bindparam('pk')),
|
||||
self.update_claims),
|
||||
(Tag.insert(), self.tags),
|
||||
(Support.insert(), self.supports),
|
||||
)
|
||||
|
||||
def flush(self, return_row_count_for_table) -> int:
|
||||
done = 0
|
||||
for sql, rows in self.get_queries():
|
||||
if not rows:
|
||||
continue
|
||||
if self.ctx.is_postgres and isinstance(sql, Insert):
|
||||
self.ctx.pg_copy(sql.table, rows)
|
||||
else:
|
||||
self.ctx.execute(sql, rows)
|
||||
if sql.table == return_row_count_for_table:
|
||||
done += len(rows)
|
||||
rows.clear()
|
||||
return done
|
||||
-103
@@ -1,103 +0,0 @@
|
||||
from sqlalchemy.future import select
|
||||
|
||||
from lbry.db.query_context import progress, Event
|
||||
from lbry.db.tables import TX, TXI, TXO, Claim, Support
|
||||
from .constants import TXO_TYPES, CLAIM_TYPE_CODES
|
||||
from .queries import (
|
||||
BASE_SELECT_TXO_COLUMNS,
|
||||
rows_to_txos, where_unspent_txos,
|
||||
where_abandoned_supports,
|
||||
where_abandoned_claims
|
||||
)
|
||||
|
||||
|
||||
SPENDS_UPDATE_EVENT = Event.add("client.sync.spends.update", "steps")
|
||||
CLAIMS_INSERT_EVENT = Event.add("client.sync.claims.insert", "claims")
|
||||
CLAIMS_UPDATE_EVENT = Event.add("client.sync.claims.update", "claims")
|
||||
CLAIMS_DELETE_EVENT = Event.add("client.sync.claims.delete", "claims")
|
||||
SUPPORT_INSERT_EVENT = Event.add("client.sync.supports.insert", "supports")
|
||||
SUPPORT_UPDATE_EVENT = Event.add("client.sync.supports.update", "supports")
|
||||
SUPPORT_DELETE_EVENT = Event.add("client.sync.supports.delete", "supports")
|
||||
|
||||
|
||||
def process_all_things_after_sync():
|
||||
with progress(SPENDS_UPDATE_EVENT) as p:
|
||||
p.start(2)
|
||||
update_spent_outputs(p.ctx)
|
||||
p.step(1)
|
||||
set_input_addresses(p.ctx)
|
||||
p.step(2)
|
||||
with progress(SUPPORT_DELETE_EVENT) as p:
|
||||
p.start(1)
|
||||
sql = Support.delete().where(where_abandoned_supports())
|
||||
p.ctx.execute(sql)
|
||||
with progress(SUPPORT_INSERT_EVENT) as p:
|
||||
loader = p.ctx.get_bulk_loader()
|
||||
sql = (
|
||||
select(*BASE_SELECT_TXO_COLUMNS)
|
||||
.where(where_unspent_txos(TXO_TYPES['support'], missing_in_supports_table=True))
|
||||
.select_from(TXO.join(TX))
|
||||
)
|
||||
for support in rows_to_txos(p.ctx.fetchall(sql)):
|
||||
loader.add_support(support)
|
||||
loader.flush(Support)
|
||||
with progress(CLAIMS_DELETE_EVENT) as p:
|
||||
p.start(1)
|
||||
sql = Claim.delete().where(where_abandoned_claims())
|
||||
p.ctx.execute(sql)
|
||||
with progress(CLAIMS_INSERT_EVENT) as p:
|
||||
loader = p.ctx.get_bulk_loader()
|
||||
sql = (
|
||||
select(*BASE_SELECT_TXO_COLUMNS)
|
||||
.where(where_unspent_txos(CLAIM_TYPE_CODES, missing_in_claims_table=True))
|
||||
.select_from(TXO.join(TX))
|
||||
)
|
||||
for claim in rows_to_txos(p.ctx.fetchall(sql)):
|
||||
loader.add_claim(claim, '', 0, 0, 0, 0, staked_support_amount=0, staked_support_count=0)
|
||||
loader.flush(Claim)
|
||||
with progress(CLAIMS_UPDATE_EVENT) as p:
|
||||
loader = p.ctx.get_bulk_loader()
|
||||
sql = (
|
||||
select(*BASE_SELECT_TXO_COLUMNS)
|
||||
.where(where_unspent_txos(CLAIM_TYPE_CODES, missing_or_stale_in_claims_table=True))
|
||||
.select_from(TXO.join(TX))
|
||||
)
|
||||
for claim in rows_to_txos(p.ctx.fetchall(sql)):
|
||||
loader.update_claim(claim)
|
||||
loader.flush(Claim)
|
||||
|
||||
|
||||
def set_input_addresses(ctx):
|
||||
# Update TXIs to have the address of TXO they are spending.
|
||||
if ctx.is_sqlite:
|
||||
address_query = select(TXO.c.address).where(TXI.c.txo_hash == TXO.c.txo_hash)
|
||||
set_addresses = (
|
||||
TXI.update()
|
||||
.values(address=address_query.scalar_subquery())
|
||||
.where(TXI.c.address.is_(None))
|
||||
)
|
||||
else:
|
||||
set_addresses = (
|
||||
TXI.update()
|
||||
.values({TXI.c.address: TXO.c.address})
|
||||
.where((TXI.c.address.is_(None)) & (TXI.c.txo_hash == TXO.c.txo_hash))
|
||||
)
|
||||
ctx.execute(set_addresses)
|
||||
|
||||
|
||||
def update_spent_outputs(ctx):
|
||||
# Update spent TXOs setting spent_height
|
||||
set_spent_height = (
|
||||
TXO.update()
|
||||
.values({
|
||||
TXO.c.spent_height: (
|
||||
select(TXI.c.height)
|
||||
.where(TXI.c.txo_hash == TXO.c.txo_hash)
|
||||
.scalar_subquery()
|
||||
)
|
||||
}).where(
|
||||
(TXO.c.spent_height == 0) &
|
||||
(TXO.c.txo_hash.in_(select(TXI.c.txo_hash).where(TXI.c.address.is_(None))))
|
||||
)
|
||||
)
|
||||
ctx.execute(set_spent_height)
|
||||
@@ -1,331 +0,0 @@
|
||||
# pylint: skip-file
|
||||
|
||||
from sqlalchemy import (
|
||||
MetaData, Table, Column, ForeignKey,
|
||||
LargeBinary, Text, SmallInteger, Integer, BigInteger, Boolean,
|
||||
)
|
||||
from .constants import TXO_TYPES, CLAIM_TYPE_CODES
|
||||
|
||||
|
||||
SCHEMA_VERSION = '1.4'
|
||||
|
||||
|
||||
metadata = MetaData()
|
||||
|
||||
|
||||
Version = Table(
|
||||
'version', metadata,
|
||||
Column('version', Text, primary_key=True),
|
||||
)
|
||||
|
||||
|
||||
Wallet = Table(
|
||||
'wallet', metadata,
|
||||
Column('wallet_id', Text, primary_key=True),
|
||||
Column('data', Text),
|
||||
)
|
||||
|
||||
|
||||
PubkeyAddress = Table(
|
||||
'pubkey_address', metadata,
|
||||
Column('address', Text, primary_key=True),
|
||||
Column('used_times', Integer, server_default='0'),
|
||||
)
|
||||
|
||||
|
||||
AccountAddress = Table(
|
||||
'account_address', metadata,
|
||||
Column('account', Text, primary_key=True),
|
||||
Column('address', Text, ForeignKey(PubkeyAddress.columns.address), primary_key=True),
|
||||
Column('chain', SmallInteger),
|
||||
Column('pubkey', LargeBinary),
|
||||
Column('chain_code', LargeBinary),
|
||||
Column('n', Integer),
|
||||
Column('depth', SmallInteger),
|
||||
)
|
||||
|
||||
|
||||
pg_add_account_address_constraints_and_indexes = [
|
||||
"CREATE UNIQUE INDEX account_address_idx ON account_address (account, address);"
|
||||
]
|
||||
|
||||
|
||||
Block = Table(
|
||||
'block', metadata,
|
||||
Column('height', Integer, primary_key=True),
|
||||
Column('block_hash', LargeBinary),
|
||||
Column('previous_hash', LargeBinary),
|
||||
Column('file_number', SmallInteger),
|
||||
Column('timestamp', Integer),
|
||||
)
|
||||
|
||||
pg_add_block_constraints_and_indexes = [
|
||||
"ALTER TABLE block ADD PRIMARY KEY (height);",
|
||||
]
|
||||
|
||||
|
||||
BlockFilter = Table(
|
||||
'block_filter', metadata,
|
||||
Column('height', Integer),
|
||||
Column('factor', SmallInteger),
|
||||
Column('address_filter', LargeBinary),
|
||||
)
|
||||
|
||||
pg_add_block_filter_constraints_and_indexes = [
|
||||
"ALTER TABLE block_filter ADD PRIMARY KEY (height, factor);",
|
||||
]
|
||||
|
||||
|
||||
TX = Table(
|
||||
'tx', metadata,
|
||||
Column('tx_hash', LargeBinary, primary_key=True),
|
||||
Column('raw', LargeBinary),
|
||||
Column('height', Integer),
|
||||
Column('position', SmallInteger),
|
||||
Column('timestamp', Integer, nullable=True),
|
||||
Column('day', Integer, nullable=True),
|
||||
Column('is_verified', Boolean, server_default='FALSE'),
|
||||
Column('purchased_claim_hash', LargeBinary, nullable=True),
|
||||
)
|
||||
|
||||
pg_add_tx_constraints_and_indexes = [
|
||||
"ALTER TABLE tx ADD PRIMARY KEY (tx_hash);",
|
||||
]
|
||||
|
||||
|
||||
TXFilter = Table(
|
||||
'tx_filter', metadata,
|
||||
Column('tx_hash', LargeBinary, primary_key=True),
|
||||
Column('height', Integer),
|
||||
Column('address_filter', LargeBinary),
|
||||
)
|
||||
|
||||
pg_add_tx_filter_constraints_and_indexes = [
|
||||
"ALTER TABLE tx_filter ADD PRIMARY KEY (tx_hash);",
|
||||
"ALTER TABLE tx_filter ADD CONSTRAINT fk_tx_filter"
|
||||
" FOREIGN KEY (tx_hash) REFERENCES tx (tx_hash) ON DELETE CASCADE;"
|
||||
]
|
||||
|
||||
|
||||
TXO = Table(
|
||||
'txo', metadata,
|
||||
Column('tx_hash', LargeBinary, ForeignKey(TX.columns.tx_hash)),
|
||||
Column('txo_hash', LargeBinary, primary_key=True),
|
||||
Column('address', Text),
|
||||
Column('position', SmallInteger),
|
||||
Column('amount', BigInteger),
|
||||
Column('height', Integer),
|
||||
Column('spent_height', Integer, server_default='0'),
|
||||
Column('script_offset', Integer),
|
||||
Column('script_length', Integer),
|
||||
Column('is_reserved', Boolean, server_default='0'),
|
||||
|
||||
# claims
|
||||
Column('txo_type', SmallInteger, server_default='0'),
|
||||
Column('claim_id', Text, nullable=True),
|
||||
Column('claim_hash', LargeBinary, nullable=True),
|
||||
Column('claim_name', Text, nullable=True),
|
||||
Column('channel_hash', LargeBinary, nullable=True), # claims in channel
|
||||
Column('signature', LargeBinary, nullable=True),
|
||||
Column('signature_digest', LargeBinary, nullable=True),
|
||||
|
||||
# reposts
|
||||
Column('reposted_claim_hash', LargeBinary, nullable=True),
|
||||
|
||||
# channels
|
||||
Column('public_key', LargeBinary, nullable=True),
|
||||
Column('public_key_hash', LargeBinary, nullable=True),
|
||||
)
|
||||
|
||||
txo_join_account = TXO.join(AccountAddress, TXO.columns.address == AccountAddress.columns.address)
|
||||
|
||||
pg_add_txo_constraints_and_indexes = [
|
||||
"ALTER TABLE txo ADD PRIMARY KEY (txo_hash);",
|
||||
# find appropriate channel public key for signing a content claim
|
||||
f"CREATE INDEX txo_channel_hash_by_height_desc_w_pub_key "
|
||||
f"ON txo (claim_hash, height desc) INCLUDE (public_key) "
|
||||
f"WHERE txo_type={TXO_TYPES['channel']};",
|
||||
# for calculating supports on a claim
|
||||
f"CREATE INDEX txo_unspent_supports ON txo (claim_hash) INCLUDE (amount) "
|
||||
f"WHERE spent_height = 0 AND txo_type={TXO_TYPES['support']};",
|
||||
# for calculating balance
|
||||
f"CREATE INDEX txo_unspent_by_address ON txo (address) INCLUDE (amount, txo_type, tx_hash) "
|
||||
f"WHERE spent_height = 0;",
|
||||
# for finding modified claims in a block range
|
||||
f"CREATE INDEX txo_claim_changes "
|
||||
f"ON txo (height DESC) INCLUDE (claim_hash, txo_hash) "
|
||||
f"WHERE spent_height = 0 AND txo_type IN {tuple(CLAIM_TYPE_CODES)};",
|
||||
# for finding claims which need support totals re-calculated in a block range
|
||||
f"CREATE INDEX txo_added_supports_by_height ON txo (height DESC) "
|
||||
f"INCLUDE (claim_hash) WHERE txo_type={TXO_TYPES['support']};",
|
||||
f"CREATE INDEX txo_spent_supports_by_height ON txo (spent_height DESC) "
|
||||
f"INCLUDE (claim_hash) WHERE txo_type={TXO_TYPES['support']};",
|
||||
# for finding claims which need repost totals re-calculated in a block range
|
||||
f"CREATE INDEX txo_added_reposts_by_height ON txo (height DESC) "
|
||||
f"INCLUDE (reposted_claim_hash) WHERE txo_type={TXO_TYPES['repost']};",
|
||||
f"CREATE INDEX txo_spent_reposts_by_height ON txo (spent_height DESC) "
|
||||
f"INCLUDE (reposted_claim_hash) WHERE txo_type={TXO_TYPES['repost']};",
|
||||
"CREATE INDEX txo_reposted_claim_hash ON txo (reposted_claim_hash)"
|
||||
"WHERE reposted_claim_hash IS NOT NULL AND spent_height = 0;",
|
||||
"CREATE INDEX txo_height ON txo (height);",
|
||||
# used by sum_supports query (at least)
|
||||
"CREATE INDEX txo_claim_hash ON txo (claim_hash)",
|
||||
]
|
||||
|
||||
|
||||
TXI = Table(
|
||||
'txi', metadata,
|
||||
Column('tx_hash', LargeBinary, ForeignKey(TX.columns.tx_hash)),
|
||||
Column('txo_hash', LargeBinary, ForeignKey(TXO.columns.txo_hash), primary_key=True),
|
||||
Column('address', Text, nullable=True),
|
||||
Column('position', SmallInteger),
|
||||
Column('height', Integer),
|
||||
)
|
||||
|
||||
txi_join_account = TXI.join(AccountAddress, TXI.columns.address == AccountAddress.columns.address)
|
||||
|
||||
pg_add_txi_constraints_and_indexes = [
|
||||
"ALTER TABLE txi ADD PRIMARY KEY (txo_hash);",
|
||||
"CREATE INDEX txi_height ON txi (height);",
|
||||
"CREATE INDEX txi_first_address ON txi (tx_hash) INCLUDE (address) WHERE position = 0;",
|
||||
]
|
||||
|
||||
|
||||
Claim = Table(
|
||||
'claim', metadata,
|
||||
Column('claim_hash', LargeBinary, primary_key=True),
|
||||
Column('claim_id', Text),
|
||||
Column('claim_name', Text),
|
||||
Column('normalized', Text),
|
||||
Column('address', Text),
|
||||
Column('txo_hash', LargeBinary, ForeignKey(TXO.columns.txo_hash)),
|
||||
Column('amount', BigInteger),
|
||||
Column('staked_amount', BigInteger),
|
||||
Column('timestamp', Integer), # last updated timestamp
|
||||
Column('creation_timestamp', Integer),
|
||||
Column('release_time', BigInteger, nullable=True),
|
||||
Column('height', Integer), # last updated height
|
||||
Column('creation_height', Integer),
|
||||
Column('activation_height', Integer),
|
||||
Column('expiration_height', Integer),
|
||||
Column('takeover_height', Integer, nullable=True),
|
||||
Column('is_controlling', Boolean),
|
||||
|
||||
# short_url: normalized#shortest-unique-claim_id
|
||||
Column('short_url', Text),
|
||||
# canonical_url: channel's-short_url/normalized#shortest-unique-claim_id-within-channel
|
||||
# canonical_url is computed dynamically
|
||||
|
||||
Column('title', Text, nullable=True),
|
||||
Column('author', Text, nullable=True),
|
||||
Column('description', Text, nullable=True),
|
||||
|
||||
Column('claim_type', SmallInteger),
|
||||
Column('staked_support_count', Integer, server_default='0'),
|
||||
Column('staked_support_amount', BigInteger, server_default='0'),
|
||||
|
||||
# streams
|
||||
Column('stream_type', SmallInteger, nullable=True),
|
||||
Column('media_type', Text, nullable=True),
|
||||
Column('fee_amount', BigInteger, server_default='0'),
|
||||
Column('fee_currency', Text, nullable=True),
|
||||
Column('duration', Integer, nullable=True),
|
||||
|
||||
# reposts
|
||||
Column('reposted_claim_hash', LargeBinary, nullable=True), # on claim doing the repost
|
||||
Column('reposted_count', Integer, server_default='0'), # on claim being reposted
|
||||
|
||||
# claims which are channels
|
||||
Column('signed_claim_count', Integer, server_default='0'),
|
||||
Column('signed_support_count', Integer, server_default='0'),
|
||||
Column('public_key_hash', LargeBinary, nullable=True), # included for claims in channel as well
|
||||
Column('public_key_height', Integer, nullable=True), # last updated height
|
||||
|
||||
# claims which are inside channels
|
||||
Column('channel_hash', LargeBinary, nullable=True),
|
||||
Column('is_signature_valid', Boolean, nullable=True),
|
||||
)
|
||||
|
||||
Tag = Table(
|
||||
'tag', metadata,
|
||||
Column('claim_hash', LargeBinary),
|
||||
Column('tag', Text),
|
||||
)
|
||||
|
||||
pg_add_claim_and_tag_constraints_and_indexes = [
|
||||
"ALTER TABLE claim ADD PRIMARY KEY (claim_hash);",
|
||||
# for checking if claim is up-to-date
|
||||
"CREATE UNIQUE INDEX claim_txo_hash ON claim (txo_hash);",
|
||||
# used by takeover process to reset winning claims
|
||||
"CREATE INDEX claim_normalized ON claim (normalized);",
|
||||
# ordering and search by release_time
|
||||
"CREATE INDEX claim_release_time ON claim (release_time DESC NULLs LAST);",
|
||||
# used to count()/sum() claims signed by channel
|
||||
"CREATE INDEX signed_content ON claim (channel_hash) "
|
||||
"INCLUDE (amount) WHERE is_signature_valid;",
|
||||
# used to count()/sum() reposted claims
|
||||
"CREATE INDEX reposted_content ON claim (reposted_claim_hash);",
|
||||
# basic tag indexes
|
||||
"ALTER TABLE tag ADD PRIMARY KEY (claim_hash, tag);",
|
||||
"CREATE INDEX tags ON tag (tag) INCLUDE (claim_hash);",
|
||||
# used by sum_supports query (at least)
|
||||
"CREATE INDEX claim_channel_hash ON claim (channel_hash)",
|
||||
]
|
||||
|
||||
|
||||
Support = Table(
|
||||
'support', metadata,
|
||||
|
||||
Column('txo_hash', LargeBinary, ForeignKey(TXO.columns.txo_hash), primary_key=True),
|
||||
Column('claim_hash', LargeBinary),
|
||||
Column('address', Text),
|
||||
Column('amount', BigInteger),
|
||||
Column('height', Integer),
|
||||
Column('timestamp', Integer),
|
||||
|
||||
# support metadata
|
||||
Column('emoji', Text),
|
||||
|
||||
# signed supports
|
||||
Column('channel_hash', LargeBinary, nullable=True),
|
||||
Column('signature', LargeBinary, nullable=True),
|
||||
Column('signature_digest', LargeBinary, nullable=True),
|
||||
Column('is_signature_valid', Boolean, nullable=True),
|
||||
)
|
||||
|
||||
pg_add_support_constraints_and_indexes = [
|
||||
"ALTER TABLE support ADD PRIMARY KEY (txo_hash);",
|
||||
# used to count()/sum() supports signed by channel
|
||||
"CREATE INDEX signed_support ON support (channel_hash) "
|
||||
"INCLUDE (amount) WHERE is_signature_valid;",
|
||||
]
|
||||
|
||||
|
||||
Stake = Table(
|
||||
'stake', metadata,
|
||||
Column('claim_hash', LargeBinary),
|
||||
Column('height', Integer),
|
||||
Column('stake_min', BigInteger),
|
||||
Column('stake_max', BigInteger),
|
||||
Column('stake_sum', BigInteger),
|
||||
Column('stake_count', Integer),
|
||||
Column('stake_unique', Integer),
|
||||
)
|
||||
|
||||
|
||||
Trend = Table(
|
||||
'trend', metadata,
|
||||
Column('claim_hash', LargeBinary, primary_key=True),
|
||||
Column('trend_group', BigInteger, server_default='0'),
|
||||
Column('trend_mixed', BigInteger, server_default='0'),
|
||||
Column('trend_local', BigInteger, server_default='0'),
|
||||
Column('trend_global', BigInteger, server_default='0'),
|
||||
)
|
||||
|
||||
|
||||
CensoredClaim = Table(
|
||||
'censored_claim', metadata,
|
||||
Column('claim_hash', LargeBinary, primary_key=True),
|
||||
Column('censor_type', SmallInteger),
|
||||
Column('censoring_channel_hash', LargeBinary),
|
||||
)
|
||||
@@ -1,25 +0,0 @@
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.sql import func
|
||||
|
||||
from lbry.db.query_context import event_emitter, ProgressContext
|
||||
from lbry.db.tables import Trend, Support, Claim
|
||||
WINDOW = 576 # a day
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.trending.update", "steps")
|
||||
def calculate_trending(height, p: ProgressContext):
|
||||
with p.ctx.engine.begin() as ctx:
|
||||
ctx.execute(Trend.delete())
|
||||
start = height - WINDOW
|
||||
trending = func.sum(Support.c.amount * (WINDOW - (height - Support.c.height)))
|
||||
sql = (
|
||||
select([Claim.c.claim_hash, trending, trending, trending, 4])
|
||||
.where(
|
||||
(Support.c.claim_hash == Claim.c.claim_hash) &
|
||||
(Support.c.height <= height) &
|
||||
(Support.c.height >= start)
|
||||
).group_by(Claim.c.claim_hash)
|
||||
)
|
||||
ctx.execute(Trend.insert().from_select(
|
||||
['claim_hash', 'trend_global', 'trend_local', 'trend_mixed', 'trend_group'], sql
|
||||
))
|
||||
@@ -1,174 +0,0 @@
|
||||
from itertools import islice
|
||||
from typing import List, Union
|
||||
|
||||
from sqlalchemy import text, and_, or_
|
||||
from sqlalchemy.sql.expression import Select, FunctionElement
|
||||
from sqlalchemy.types import Numeric
|
||||
from sqlalchemy.ext.compiler import compiles
|
||||
try:
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert # pylint: disable=unused-import
|
||||
except ImportError:
|
||||
pg_insert = None
|
||||
|
||||
from .tables import AccountAddress
|
||||
|
||||
|
||||
class greatest(FunctionElement): # pylint: disable=invalid-name
|
||||
type = Numeric()
|
||||
name = 'greatest'
|
||||
|
||||
|
||||
@compiles(greatest)
|
||||
def default_greatest(element, compiler, **kw):
|
||||
return "greatest(%s)" % compiler.process(element.clauses, **kw)
|
||||
|
||||
|
||||
@compiles(greatest, 'sqlite')
|
||||
def sqlite_greatest(element, compiler, **kw):
|
||||
return "max(%s)" % compiler.process(element.clauses, **kw)
|
||||
|
||||
|
||||
class least(FunctionElement): # pylint: disable=invalid-name
|
||||
type = Numeric()
|
||||
name = 'least'
|
||||
|
||||
|
||||
@compiles(least)
|
||||
def default_least(element, compiler, **kw):
|
||||
return "least(%s)" % compiler.process(element.clauses, **kw)
|
||||
|
||||
|
||||
@compiles(least, 'sqlite')
|
||||
def sqlite_least(element, compiler, **kw):
|
||||
return "min(%s)" % compiler.process(element.clauses, **kw)
|
||||
|
||||
|
||||
def chunk(rows, step):
|
||||
it, total = iter(rows), len(rows)
|
||||
for _ in range(0, total, step):
|
||||
yield list(islice(it, step))
|
||||
total -= step
|
||||
|
||||
|
||||
def constrain_single_or_list(constraints, column, value, convert=lambda x: x):
|
||||
if value is not None:
|
||||
if isinstance(value, list):
|
||||
value = [convert(v) for v in value]
|
||||
if len(value) == 1:
|
||||
constraints[column] = value[0]
|
||||
elif len(value) > 1:
|
||||
constraints[f"{column}__in"] = value
|
||||
else:
|
||||
constraints[column] = convert(value)
|
||||
return constraints
|
||||
|
||||
|
||||
def in_account_ids(account_ids: Union[List[str], str]):
|
||||
if isinstance(account_ids, list):
|
||||
if len(account_ids) > 1:
|
||||
return AccountAddress.c.account.in_(account_ids)
|
||||
account_ids = account_ids[0]
|
||||
return AccountAddress.c.account == account_ids
|
||||
|
||||
|
||||
def query(table, s: Select, **constraints) -> Select:
|
||||
limit = constraints.pop('limit', None)
|
||||
if limit is not None:
|
||||
s = s.limit(limit)
|
||||
|
||||
offset = constraints.pop('offset', None)
|
||||
if offset is not None:
|
||||
s = s.offset(offset)
|
||||
|
||||
order_by = constraints.pop('order_by', None)
|
||||
if order_by:
|
||||
if isinstance(order_by, str):
|
||||
s = s.order_by(text(order_by))
|
||||
elif isinstance(order_by, list):
|
||||
s = s.order_by(text(', '.join(order_by)))
|
||||
else:
|
||||
raise ValueError("order_by must be string or list")
|
||||
|
||||
group_by = constraints.pop('group_by', None)
|
||||
if group_by is not None:
|
||||
s = s.group_by(text(group_by))
|
||||
|
||||
account_ids = constraints.pop('account_ids', [])
|
||||
if account_ids:
|
||||
s = s.where(in_account_ids(account_ids))
|
||||
|
||||
if constraints:
|
||||
s = s.where(and_(*constraints_to_clause(table, constraints)))
|
||||
|
||||
return s
|
||||
|
||||
|
||||
def constraints_to_clause(tables, constraints):
|
||||
clause = []
|
||||
for key, constraint in constraints.items():
|
||||
if key.endswith('__not'):
|
||||
col, op = key[:-len('__not')], '__ne__'
|
||||
elif key.endswith('__is_null'):
|
||||
col = key[:-len('__is_null')]
|
||||
op = '__eq__'
|
||||
constraint = None
|
||||
elif key.endswith('__is_not_null'):
|
||||
col = key[:-len('__is_not_null')]
|
||||
op = '__ne__'
|
||||
constraint = None
|
||||
elif key.endswith('__lt'):
|
||||
col, op = key[:-len('__lt')], '__lt__'
|
||||
elif key.endswith('__lte'):
|
||||
col, op = key[:-len('__lte')], '__le__'
|
||||
elif key.endswith('__gt'):
|
||||
col, op = key[:-len('__gt')], '__gt__'
|
||||
elif key.endswith('__gte'):
|
||||
col, op = key[:-len('__gte')], '__ge__'
|
||||
elif key.endswith('__like'):
|
||||
col, op = key[:-len('__like')], 'like'
|
||||
elif key.endswith('__not_like'):
|
||||
col, op = key[:-len('__not_like')], 'notlike'
|
||||
elif key.endswith('__in') or key.endswith('__not_in'):
|
||||
if key.endswith('__in'):
|
||||
col, op, one_val_op = key[:-len('__in')], 'in_', '__eq__'
|
||||
else:
|
||||
col, op, one_val_op = key[:-len('__not_in')], 'notin_', '__ne__'
|
||||
if isinstance(constraint, Select):
|
||||
pass
|
||||
elif constraint:
|
||||
if isinstance(constraint, (list, set, tuple)):
|
||||
if len(constraint) == 1:
|
||||
op = one_val_op
|
||||
constraint = next(iter(constraint))
|
||||
elif isinstance(constraint, str):
|
||||
constraint = text(constraint)
|
||||
else:
|
||||
raise ValueError(f"{col} requires a list, set or string as constraint value.")
|
||||
else:
|
||||
continue
|
||||
elif key.endswith('__or'):
|
||||
clause.append(or_(*constraints_to_clause(tables, constraint)))
|
||||
continue
|
||||
else:
|
||||
col, op = key, '__eq__'
|
||||
attr = None
|
||||
if '.' in col:
|
||||
table_name, col = col.split('.')
|
||||
_table = None
|
||||
for table in tables:
|
||||
if table.name == table_name.lower():
|
||||
_table = table
|
||||
break
|
||||
if _table is not None:
|
||||
attr = getattr(_table.c, col)
|
||||
else:
|
||||
raise ValueError(f"Table '{table_name}' not available: {', '.join([t.name for t in tables])}.")
|
||||
else:
|
||||
for table in tables:
|
||||
attr = getattr(table.c, col, None)
|
||||
if attr is not None:
|
||||
break
|
||||
if attr is None:
|
||||
raise ValueError(f"Attribute '{col}' not found on tables: {', '.join([t.name for t in tables])}.")
|
||||
clause.append(getattr(attr, op)(constraint))
|
||||
return clause
|
||||
@@ -3,15 +3,11 @@ import typing
|
||||
import logging
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.dht.node import Node
|
||||
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SQLiteStorage:
|
||||
pass
|
||||
|
||||
|
||||
class BlobAnnouncer:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, node: 'Node', storage: 'SQLiteStorage'):
|
||||
self.loop = loop
|
||||
|
||||
+11
-28
@@ -1,14 +1,14 @@
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
import ipaddress
|
||||
from binascii import hexlify
|
||||
from dataclasses import dataclass, field
|
||||
from functools import lru_cache
|
||||
|
||||
from lbry.utils import is_valid_public_ipv4 as _is_valid_public_ipv4, LRUCache
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
|
||||
|
||||
ALLOW_LOCALHOST = False
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -20,26 +20,9 @@ def make_kademlia_peer(node_id: typing.Optional[bytes], address: typing.Optional
|
||||
return KademliaPeer(address, node_id, udp_port, tcp_port=tcp_port, allow_localhost=allow_localhost)
|
||||
|
||||
|
||||
# the ipaddress module does not show these subnets as reserved
|
||||
CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
|
||||
IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
|
||||
|
||||
ALLOW_LOCALHOST = False
|
||||
|
||||
|
||||
def is_valid_public_ipv4(address, allow_localhost: bool = False):
|
||||
allow_localhost = bool(allow_localhost or ALLOW_LOCALHOST)
|
||||
try:
|
||||
parsed_ip = ipaddress.ip_address(address)
|
||||
if parsed_ip.is_loopback and allow_localhost:
|
||||
return True
|
||||
return not any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local,
|
||||
parsed_ip.is_loopback, parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private,
|
||||
parsed_ip.is_reserved,
|
||||
CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
|
||||
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
|
||||
except ipaddress.AddressValueError:
|
||||
return False
|
||||
return _is_valid_public_ipv4(address, allow_localhost)
|
||||
|
||||
|
||||
class PeerManager:
|
||||
@@ -48,12 +31,12 @@ class PeerManager:
|
||||
self._rpc_failures: typing.Dict[
|
||||
typing.Tuple[str, int], typing.Tuple[typing.Optional[float], typing.Optional[float]]
|
||||
] = {}
|
||||
self._last_replied: typing.Dict[typing.Tuple[str, int], float] = {}
|
||||
self._last_sent: typing.Dict[typing.Tuple[str, int], float] = {}
|
||||
self._last_requested: typing.Dict[typing.Tuple[str, int], float] = {}
|
||||
self._node_id_mapping: typing.Dict[typing.Tuple[str, int], bytes] = {}
|
||||
self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = {}
|
||||
self._node_tokens: typing.Dict[bytes, (float, bytes)] = {}
|
||||
self._last_replied: typing.Dict[typing.Tuple[str, int], float] = LRUCache(2048)
|
||||
self._last_sent: typing.Dict[typing.Tuple[str, int], float] = LRUCache(2048)
|
||||
self._last_requested: typing.Dict[typing.Tuple[str, int], float] = LRUCache(2048)
|
||||
self._node_id_mapping: typing.Dict[typing.Tuple[str, int], bytes] = LRUCache(2048)
|
||||
self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = LRUCache(2048)
|
||||
self._node_tokens: typing.Dict[bytes, (float, bytes)] = LRUCache(2048)
|
||||
|
||||
def reset(self):
|
||||
for statistic in (self._rpc_failures, self._last_replied, self._last_sent, self._last_requested):
|
||||
@@ -172,9 +155,9 @@ class KademliaPeer:
|
||||
if self._node_id is not None:
|
||||
if not len(self._node_id) == constants.HASH_LENGTH:
|
||||
raise ValueError("invalid node_id: {}".format(hexlify(self._node_id).decode()))
|
||||
if self.udp_port is not None and not 1 <= self.udp_port <= 65535:
|
||||
if self.udp_port is not None and not 1024 <= self.udp_port <= 65535:
|
||||
raise ValueError("invalid udp port")
|
||||
if self.tcp_port is not None and not 1 <= self.tcp_port <= 65535:
|
||||
if self.tcp_port is not None and not 1024 <= self.tcp_port <= 65535:
|
||||
raise ValueError("invalid tcp port")
|
||||
if not is_valid_public_ipv4(self.address, self.allow_localhost):
|
||||
raise ValueError(f"invalid ip address: '{self.address}'")
|
||||
|
||||
@@ -10,6 +10,7 @@ from asyncio.protocols import DatagramProtocol
|
||||
from asyncio.transports import DatagramTransport
|
||||
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.serialization.bencoding import DecodeError
|
||||
from lbry.dht.serialization.datagram import decode_datagram, ErrorDatagram, ResponseDatagram, RequestDatagram
|
||||
from lbry.dht.serialization.datagram import RESPONSE_TYPE, ERROR_TYPE, PAGE_KEY
|
||||
from lbry.dht.error import RemoteException, TransportNotConnected
|
||||
@@ -554,7 +555,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
def datagram_received(self, datagram: bytes, address: typing.Tuple[str, int]) -> None: # pylint: disable=arguments-differ
|
||||
try:
|
||||
message = decode_datagram(datagram)
|
||||
except (ValueError, TypeError):
|
||||
except (ValueError, TypeError, DecodeError):
|
||||
self.peer_manager.report_failure(address[0], address[1])
|
||||
log.warning("Couldn't decode dht datagram from %s: %s", address, binascii.hexlify(datagram).decode())
|
||||
return
|
||||
|
||||
@@ -144,7 +144,7 @@ class ErrorDatagram(KademliaDatagramBase):
|
||||
self.response = response.decode()
|
||||
|
||||
|
||||
def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]:
|
||||
def _decode_datagram(datagram: bytes):
|
||||
msg_types = {
|
||||
REQUEST_TYPE: RequestDatagram,
|
||||
RESPONSE_TYPE: ResponseDatagram,
|
||||
@@ -152,26 +152,36 @@ def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDa
|
||||
}
|
||||
|
||||
primitive: typing.Dict = bdecode(datagram)
|
||||
if primitive[0] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
|
||||
datagram_type = primitive[0] # pylint: disable=unsubscriptable-object
|
||||
|
||||
converted = {
|
||||
str(k).encode() if not isinstance(k, bytes) else k: v for k, v in primitive.items()
|
||||
}
|
||||
|
||||
if converted[b'0'] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
|
||||
datagram_type = converted[b'0'] # pylint: disable=unsubscriptable-object
|
||||
else:
|
||||
raise ValueError("invalid datagram type")
|
||||
datagram_class = msg_types[datagram_type]
|
||||
decoded = {
|
||||
k: primitive[i] # pylint: disable=unsubscriptable-object
|
||||
k: converted[str(i).encode()] # pylint: disable=unsubscriptable-object
|
||||
for i, k in enumerate(datagram_class.required_fields)
|
||||
if i in primitive # pylint: disable=unsupported-membership-test
|
||||
if str(i).encode() in converted # pylint: disable=unsupported-membership-test
|
||||
}
|
||||
for i, _ in enumerate(OPTIONAL_FIELDS):
|
||||
if i + OPTIONAL_ARG_OFFSET in primitive:
|
||||
decoded[i + OPTIONAL_ARG_OFFSET] = primitive[i + OPTIONAL_ARG_OFFSET]
|
||||
if str(i + OPTIONAL_ARG_OFFSET).encode() in converted:
|
||||
decoded[i + OPTIONAL_ARG_OFFSET] = converted[str(i + OPTIONAL_ARG_OFFSET).encode()]
|
||||
return decoded, datagram_class
|
||||
|
||||
|
||||
def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]:
|
||||
decoded, datagram_class = _decode_datagram(datagram)
|
||||
return datagram_class(**decoded)
|
||||
|
||||
|
||||
def make_compact_ip(address: str) -> bytearray:
|
||||
compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), address.split('.'), bytearray())
|
||||
if len(compact_ip) != 4:
|
||||
raise ValueError("invalid IPv4 length")
|
||||
raise ValueError(f"invalid IPv4 length")
|
||||
return compact_ip
|
||||
|
||||
|
||||
@@ -180,7 +190,7 @@ def make_compact_address(node_id: bytes, address: str, port: int) -> bytearray:
|
||||
if not 0 < port < 65536:
|
||||
raise ValueError(f'Invalid port: {port}')
|
||||
if len(node_id) != constants.HASH_BITS // 8:
|
||||
raise ValueError("invalid node node_id length")
|
||||
raise ValueError(f"invalid node node_id length")
|
||||
return compact_ip + port.to_bytes(2, 'big') + node_id
|
||||
|
||||
|
||||
@@ -191,5 +201,5 @@ def decode_compact_address(compact_address: bytes) -> typing.Tuple[bytes, str, i
|
||||
if not 0 < port < 65536:
|
||||
raise ValueError(f'Invalid port: {port}')
|
||||
if len(node_id) != constants.HASH_BITS // 8:
|
||||
raise ValueError("invalid node node_id length")
|
||||
raise ValueError(f"invalid node node_id length")
|
||||
return node_id, address, port
|
||||
@@ -81,6 +81,3 @@ Code | Name | Message
|
||||
701 | InvalidExchangeRateResponse | Failed to get exchange rate from {source}: {reason}
|
||||
702 | CurrencyConversion | {message}
|
||||
703 | InvalidCurrency | Invalid currency: {currency} is not a supported currency.
|
||||
**8xx** | Lbrycrd | **Lbrycrd**
|
||||
801 | LbrycrdUnauthorized | Failed to authenticate with lbrycrd. Perhaps wrong username or password?
|
||||
811 | LbrycrdEventSubscription | Lbrycrd is not publishing '{event}' events.
|
||||
+3
-22
@@ -215,10 +215,10 @@ class ResolveTimeoutError(WalletError):
|
||||
|
||||
class ResolveCensoredError(WalletError):
|
||||
|
||||
def __init__(self, url, censor_hash):
|
||||
def __init__(self, url, censor_id):
|
||||
self.url = url
|
||||
self.censor_hash = censor_hash
|
||||
super().__init__(f"Resolve of '{url}' was censored by channel with claim id '{claim_id(censor_hash)}'.")
|
||||
self.censor_id = censor_id
|
||||
super().__init__(f"Resolve of '{url}' was censored by channel with claim id '{censor_id}'.")
|
||||
|
||||
|
||||
class KeyFeeAboveMaxAllowedError(WalletError):
|
||||
@@ -398,22 +398,3 @@ class InvalidCurrencyError(CurrencyExchangeError):
|
||||
def __init__(self, currency):
|
||||
self.currency = currency
|
||||
super().__init__(f"Invalid currency: {currency} is not a supported currency.")
|
||||
|
||||
|
||||
class LbrycrdError(BaseError):
|
||||
"""
|
||||
**Lbrycrd**
|
||||
"""
|
||||
|
||||
|
||||
class LbrycrdUnauthorizedError(LbrycrdError):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("Failed to authenticate with lbrycrd. Perhaps wrong username or password?")
|
||||
|
||||
|
||||
class LbrycrdEventSubscriptionError(LbrycrdError):
|
||||
|
||||
def __init__(self, event):
|
||||
self.event = event
|
||||
super().__init__(f"Lbrycrd is not publishing '{event}' events.")
|
||||
@@ -63,7 +63,7 @@ class ErrorClass:
|
||||
@staticmethod
|
||||
def get_fields(args):
|
||||
if len(args) > 1:
|
||||
return ''.join(f'\n{INDENT*2}self.{field} = {field}' for field in args[1:])
|
||||
return f''.join(f'\n{INDENT*2}self.{field} = {field}' for field in args[1:])
|
||||
return ''
|
||||
|
||||
@staticmethod
|
||||
|
||||
-263
@@ -1,263 +0,0 @@
|
||||
import time
|
||||
import asyncio
|
||||
import threading
|
||||
import logging
|
||||
from queue import Empty
|
||||
from multiprocessing import Queue
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BroadcastSubscription:
|
||||
|
||||
def __init__(self, controller: 'EventController', on_data, on_error, on_done):
|
||||
self._controller = controller
|
||||
self._previous = self._next = None
|
||||
self._on_data = on_data
|
||||
self._on_error = on_error
|
||||
self._on_done = on_done
|
||||
self.is_paused = False
|
||||
self.is_canceled = False
|
||||
self.is_closed = False
|
||||
|
||||
def pause(self):
|
||||
self.is_paused = True
|
||||
|
||||
def resume(self):
|
||||
self.is_paused = False
|
||||
|
||||
def cancel(self):
|
||||
self._controller._cancel(self)
|
||||
self.is_canceled = True
|
||||
|
||||
@property
|
||||
def can_fire(self):
|
||||
return not any((self.is_paused, self.is_canceled, self.is_closed))
|
||||
|
||||
def _add(self, data):
|
||||
if self.can_fire and self._on_data is not None:
|
||||
return self._on_data(data)
|
||||
|
||||
def _add_error(self, exception):
|
||||
if self.can_fire and self._on_error is not None:
|
||||
return self._on_error(exception)
|
||||
|
||||
def _close(self):
|
||||
try:
|
||||
if self.can_fire and self._on_done is not None:
|
||||
return self._on_done()
|
||||
finally:
|
||||
self.is_closed = True
|
||||
|
||||
|
||||
class EventController:
|
||||
|
||||
def __init__(self, merge_repeated_events=False):
|
||||
self.stream = EventStream(self)
|
||||
self._first_subscription = None
|
||||
self._last_subscription = None
|
||||
self._last_event = None
|
||||
self._merge_repeated = merge_repeated_events
|
||||
|
||||
@property
|
||||
def has_listener(self):
|
||||
return self._first_subscription is not None
|
||||
|
||||
@property
|
||||
def _iterate_subscriptions(self):
|
||||
next_sub = self._first_subscription
|
||||
while next_sub is not None:
|
||||
subscription = next_sub
|
||||
yield subscription
|
||||
next_sub = next_sub._next
|
||||
|
||||
async def _notify(self, notify, *args):
|
||||
try:
|
||||
maybe_coroutine = notify(*args)
|
||||
if maybe_coroutine is not None and asyncio.iscoroutine(maybe_coroutine):
|
||||
await maybe_coroutine
|
||||
except Exception as e:
|
||||
log.exception(e)
|
||||
raise
|
||||
|
||||
async def add(self, event):
|
||||
if self._merge_repeated and event == self._last_event:
|
||||
return
|
||||
self._last_event = event
|
||||
for subscription in self._iterate_subscriptions:
|
||||
await self._notify(subscription._add, event)
|
||||
|
||||
async def add_all(self, events):
|
||||
for event in events:
|
||||
await self.add(event)
|
||||
|
||||
async def add_error(self, exception):
|
||||
for subscription in self._iterate_subscriptions:
|
||||
await self._notify(subscription._add_error, exception)
|
||||
|
||||
async def close(self):
|
||||
for subscription in self._iterate_subscriptions:
|
||||
await self._notify(subscription._close)
|
||||
|
||||
def _cancel(self, subscription):
|
||||
previous = subscription._previous
|
||||
next_sub = subscription._next
|
||||
if previous is None:
|
||||
self._first_subscription = next_sub
|
||||
else:
|
||||
previous._next = next_sub
|
||||
if next_sub is None:
|
||||
self._last_subscription = previous
|
||||
else:
|
||||
next_sub._previous = previous
|
||||
|
||||
def _listen(self, on_data, on_error, on_done):
|
||||
subscription = BroadcastSubscription(self, on_data, on_error, on_done)
|
||||
old_last = self._last_subscription
|
||||
self._last_subscription = subscription
|
||||
subscription._previous = old_last
|
||||
subscription._next = None
|
||||
if old_last is None:
|
||||
self._first_subscription = subscription
|
||||
else:
|
||||
old_last._next = subscription
|
||||
return subscription
|
||||
|
||||
|
||||
class EventStream:
|
||||
|
||||
def __init__(self, controller: EventController):
|
||||
self._controller = controller
|
||||
|
||||
def listen(self, on_data, on_error=None, on_done=None) -> BroadcastSubscription:
|
||||
return self._controller._listen(on_data, on_error, on_done)
|
||||
|
||||
def where(self, condition) -> asyncio.Future:
|
||||
future = asyncio.get_running_loop().create_future()
|
||||
|
||||
def where_test(value):
|
||||
if condition(value):
|
||||
self._cancel_and_callback(subscription, future, value)
|
||||
|
||||
subscription = self.listen(
|
||||
where_test,
|
||||
lambda exception: self._cancel_and_error(subscription, future, exception)
|
||||
)
|
||||
|
||||
return future
|
||||
|
||||
@property
|
||||
def first(self) -> asyncio.Future:
|
||||
future = asyncio.get_running_loop().create_future()
|
||||
subscription = self.listen(
|
||||
lambda value: not future.done() and self._cancel_and_callback(subscription, future, value),
|
||||
lambda exception: not future.done() and self._cancel_and_error(subscription, future, exception)
|
||||
)
|
||||
return future
|
||||
|
||||
@property
|
||||
def last(self) -> asyncio.Future:
|
||||
future = asyncio.get_running_loop().create_future()
|
||||
value = None
|
||||
|
||||
def update_value(_value):
|
||||
nonlocal value
|
||||
value = _value
|
||||
|
||||
subscription = self.listen(
|
||||
update_value,
|
||||
lambda exception: not future.done() and self._cancel_and_error(subscription, future, exception),
|
||||
lambda: not future.done() and self._cancel_and_callback(subscription, future, value),
|
||||
)
|
||||
|
||||
return future
|
||||
|
||||
@staticmethod
|
||||
def _cancel_and_callback(subscription: BroadcastSubscription, future: asyncio.Future, value):
|
||||
subscription.cancel()
|
||||
future.set_result(value)
|
||||
|
||||
@staticmethod
|
||||
def _cancel_and_error(subscription: BroadcastSubscription, future: asyncio.Future, exception):
|
||||
subscription.cancel()
|
||||
future.set_exception(exception)
|
||||
|
||||
|
||||
class EventRegistry:
|
||||
|
||||
def __init__(self):
|
||||
self.events = {}
|
||||
|
||||
def register(self, name, stream: EventStream):
|
||||
assert name not in self.events
|
||||
self.events[name] = stream
|
||||
|
||||
def get(self, event_name):
|
||||
return self.events.get(event_name)
|
||||
|
||||
def clear(self):
|
||||
self.events.clear()
|
||||
|
||||
|
||||
class EventQueuePublisher(threading.Thread):
|
||||
|
||||
STOP = 'STOP'
|
||||
|
||||
def __init__(self, queue: Queue, event_controller: EventController):
|
||||
super().__init__()
|
||||
self.queue = queue
|
||||
self.event_controller = event_controller
|
||||
self.loop = None
|
||||
|
||||
@staticmethod
|
||||
def message_to_event(message):
|
||||
return message
|
||||
|
||||
def start(self):
|
||||
self.loop = asyncio.get_running_loop()
|
||||
super().start()
|
||||
|
||||
def run(self):
|
||||
queue_get_timeout = 0.2
|
||||
buffer_drain_size = 100
|
||||
buffer_drain_timeout = 0.1
|
||||
|
||||
buffer = []
|
||||
last_drained_ms_ago = time.perf_counter()
|
||||
while True:
|
||||
|
||||
try:
|
||||
msg = self.queue.get(timeout=queue_get_timeout)
|
||||
if msg != self.STOP:
|
||||
buffer.append(msg)
|
||||
except Empty:
|
||||
msg = None
|
||||
|
||||
drain = any((
|
||||
len(buffer) >= buffer_drain_size,
|
||||
(time.perf_counter() - last_drained_ms_ago) >= buffer_drain_timeout,
|
||||
msg == self.STOP
|
||||
))
|
||||
if drain and buffer:
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self.event_controller.add_all([
|
||||
self.message_to_event(msg) for msg in buffer
|
||||
]), self.loop
|
||||
)
|
||||
buffer.clear()
|
||||
last_drained_ms_ago = time.perf_counter()
|
||||
|
||||
if msg == self.STOP:
|
||||
return
|
||||
|
||||
def stop(self):
|
||||
self.queue.put(self.STOP)
|
||||
if self.is_alive():
|
||||
self.join()
|
||||
|
||||
def __enter__(self):
|
||||
self.start()
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.stop()
|
||||
File renamed without changes.
@@ -1,17 +1,78 @@
|
||||
import os
|
||||
import sys
|
||||
import asyncio
|
||||
import shutil
|
||||
import signal
|
||||
import pathlib
|
||||
import json
|
||||
import asyncio
|
||||
import argparse
|
||||
import textwrap
|
||||
import subprocess
|
||||
import logging
|
||||
import logging.handlers
|
||||
|
||||
import aiohttp
|
||||
from aiohttp.web import GracefulExit
|
||||
from docopt import docopt
|
||||
|
||||
from lbry import __version__
|
||||
from lbry import __version__ as lbrynet_version
|
||||
from lbry.extras.daemon.daemon import Daemon
|
||||
from lbry.conf import Config, CLIConfig
|
||||
from lbry.service import Daemon, Client, FullNode, LightClient
|
||||
from lbry.service.metadata import interface
|
||||
|
||||
log = logging.getLogger('lbry')
|
||||
|
||||
|
||||
def display(data):
|
||||
print(json.dumps(data, indent=2))
|
||||
|
||||
|
||||
async def execute_command(conf, method, params, callback=display):
|
||||
async with aiohttp.ClientSession() as session:
|
||||
try:
|
||||
message = {'method': method, 'params': params}
|
||||
async with session.get(conf.api_connection_url, json=message) as resp:
|
||||
try:
|
||||
data = await resp.json()
|
||||
if 'result' in data:
|
||||
return callback(data['result'])
|
||||
elif 'error' in data:
|
||||
return callback(data['error'])
|
||||
except Exception as e:
|
||||
log.exception('Could not process response from server:', exc_info=e)
|
||||
except aiohttp.ClientConnectionError:
|
||||
print("Could not connect to daemon. Are you sure it's running?")
|
||||
|
||||
|
||||
def normalize_value(x, key=None):
|
||||
if not isinstance(x, str):
|
||||
return x
|
||||
if key in ('uri', 'channel_name', 'name', 'file_name', 'claim_name', 'download_directory'):
|
||||
return x
|
||||
if x.lower() == 'true':
|
||||
return True
|
||||
if x.lower() == 'false':
|
||||
return False
|
||||
if x.isdigit():
|
||||
return int(x)
|
||||
return x
|
||||
|
||||
|
||||
def remove_brackets(key):
|
||||
if key.startswith("<") and key.endswith(">"):
|
||||
return str(key[1:-1])
|
||||
return key
|
||||
|
||||
|
||||
def set_kwargs(parsed_args):
|
||||
kwargs = {}
|
||||
for key, arg in parsed_args.items():
|
||||
if arg is None:
|
||||
continue
|
||||
k = None
|
||||
if key.startswith("--") and remove_brackets(key[2:]) not in kwargs:
|
||||
k = remove_brackets(key[2:])
|
||||
elif remove_brackets(key) not in kwargs:
|
||||
k = remove_brackets(key)
|
||||
kwargs[k] = normalize_value(arg, k)
|
||||
return kwargs
|
||||
|
||||
|
||||
def split_subparser_argument(parent, original, name, condition):
|
||||
@@ -40,7 +101,7 @@ class ArgumentParser(argparse.ArgumentParser):
|
||||
self._optionals.title = 'Options'
|
||||
if group_name is None:
|
||||
self.epilog = (
|
||||
"Run 'lbrynet COMMAND --help' for more information on a command or group."
|
||||
f"Run 'lbrynet COMMAND --help' for more information on a command or group."
|
||||
)
|
||||
else:
|
||||
self.epilog = (
|
||||
@@ -92,10 +153,17 @@ class HelpFormatter(argparse.HelpFormatter):
|
||||
)
|
||||
|
||||
|
||||
def add_command_parser(parent, method_name, command):
|
||||
short = command['desc']['text'][0] if command['desc'] else ''
|
||||
subcommand = parent.add_parser(command['name'], help=short)
|
||||
subcommand.set_defaults(api_method_name=method_name, command=command['name'], doc=command['help'])
|
||||
def add_command_parser(parent, command):
|
||||
subcommand = parent.add_parser(
|
||||
command['name'],
|
||||
help=command['doc'].strip().splitlines()[0]
|
||||
)
|
||||
subcommand.set_defaults(
|
||||
api_method_name=command['api_method_name'],
|
||||
command=command['name'],
|
||||
doc=command['doc'],
|
||||
replaced_by=command.get('replaced_by', None)
|
||||
)
|
||||
|
||||
|
||||
def get_argument_parser():
|
||||
@@ -114,9 +182,6 @@ def get_argument_parser():
|
||||
usage='lbrynet start [--config FILE] [--data-dir DIR] [--wallet-dir DIR] [--download-dir DIR] ...',
|
||||
help='Start LBRY Network interface.'
|
||||
)
|
||||
start.add_argument(
|
||||
"service", choices=[LightClient.name, FullNode.name], default=LightClient.name, nargs="?"
|
||||
)
|
||||
start.add_argument(
|
||||
'--quiet', dest='quiet', action="store_true",
|
||||
help='Disable all console output.'
|
||||
@@ -134,32 +199,26 @@ def get_argument_parser():
|
||||
'--initial-headers', dest='initial_headers',
|
||||
help='Specify path to initial blockchain headers, faster than downloading them on first run.'
|
||||
)
|
||||
install = sub.add_parser("install", help="Install lbrynet with various system services.")
|
||||
install.add_argument("system", choices=["systemd"])
|
||||
install.add_argument(
|
||||
"--global", dest="install_global", action="store_true",
|
||||
help="Install system wide (requires running as root), default is for current user only."
|
||||
)
|
||||
Config.contribute_to_argparse(start)
|
||||
start.set_defaults(command='start', start_parser=start, doc=start.format_help())
|
||||
install.set_defaults(command='install', install_parser=install, doc=install.format_help())
|
||||
|
||||
api = Daemon.get_api_definitions()
|
||||
groups = {}
|
||||
for group_name in sorted(interface['groups']):
|
||||
group_parser = sub.add_parser(group_name, group_name=group_name, help=interface['groups'][group_name])
|
||||
for group_name in sorted(api['groups']):
|
||||
group_parser = sub.add_parser(group_name, group_name=group_name, help=api['groups'][group_name])
|
||||
groups[group_name] = group_parser.add_subparsers(metavar='COMMAND')
|
||||
|
||||
nicer_order = ['stop', 'get', 'publish', 'resolve']
|
||||
for command_name in sorted(interface['commands']):
|
||||
for command_name in sorted(api['commands']):
|
||||
if command_name not in nicer_order:
|
||||
nicer_order.append(command_name)
|
||||
|
||||
for command_name in nicer_order:
|
||||
command = interface['commands'][command_name]
|
||||
if command.get('group') is None:
|
||||
add_command_parser(sub, command_name, command)
|
||||
command = api['commands'][command_name]
|
||||
if command['group'] is None:
|
||||
add_command_parser(sub, command)
|
||||
else:
|
||||
add_command_parser(groups[command['group']], command_name, command)
|
||||
add_command_parser(groups[command['group']], command)
|
||||
|
||||
return root
|
||||
|
||||
@@ -169,64 +228,61 @@ def ensure_directory_exists(path: str):
|
||||
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
async def execute_command(conf, method, params):
|
||||
client = Client(f"http://{conf.api}/ws")
|
||||
await client.connect()
|
||||
responses = await client.send(method, **params)
|
||||
result = await responses.first
|
||||
await client.disconnect()
|
||||
print(result)
|
||||
return result
|
||||
LOG_MODULES = 'lbry', 'aioupnp'
|
||||
|
||||
|
||||
def normalize_value(x, key=None):
|
||||
if not isinstance(x, str):
|
||||
return x
|
||||
if key in ('uri', 'channel_name', 'name', 'file_name', 'claim_name', 'download_directory'):
|
||||
return x
|
||||
if x.lower() == 'true':
|
||||
return True
|
||||
if x.lower() == 'false':
|
||||
return False
|
||||
if x.isdigit():
|
||||
return int(x)
|
||||
return x
|
||||
def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config):
|
||||
default_formatter = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s")
|
||||
file_handler = logging.handlers.RotatingFileHandler(conf.log_file_path, maxBytes=2097152, backupCount=5)
|
||||
file_handler.setFormatter(default_formatter)
|
||||
for module_name in LOG_MODULES:
|
||||
logger.getChild(module_name).addHandler(file_handler)
|
||||
if not args.quiet:
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(default_formatter)
|
||||
for module_name in LOG_MODULES:
|
||||
logger.getChild(module_name).addHandler(handler)
|
||||
|
||||
logger.getChild('lbry').setLevel(logging.INFO)
|
||||
logger.getChild('aioupnp').setLevel(logging.WARNING)
|
||||
logger.getChild('aiohttp').setLevel(logging.CRITICAL)
|
||||
|
||||
if args.verbose is not None:
|
||||
if len(args.verbose) > 0:
|
||||
for module in args.verbose:
|
||||
logger.getChild(module).setLevel(logging.DEBUG)
|
||||
else:
|
||||
logger.getChild('lbry').setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
def remove_brackets(key):
|
||||
if key.startswith("<") and key.endswith(">"):
|
||||
return str(key[1:-1])
|
||||
return key
|
||||
def run_daemon(args: argparse.Namespace, conf: Config):
|
||||
loop = asyncio.get_event_loop()
|
||||
if args.verbose is not None:
|
||||
loop.set_debug(True)
|
||||
if not args.no_logging:
|
||||
setup_logging(logging.getLogger(), args, conf)
|
||||
daemon = Daemon(conf)
|
||||
|
||||
def __exit():
|
||||
raise GracefulExit()
|
||||
|
||||
def set_kwargs(parsed_args):
|
||||
kwargs = {}
|
||||
for key, arg in parsed_args.items():
|
||||
if arg is None:
|
||||
continue
|
||||
k = None
|
||||
if key.startswith("--") and remove_brackets(key[2:]) not in kwargs:
|
||||
k = remove_brackets(key[2:])
|
||||
elif remove_brackets(key) not in kwargs:
|
||||
k = remove_brackets(key)
|
||||
kwargs[k] = normalize_value(arg, k)
|
||||
return kwargs
|
||||
try:
|
||||
loop.add_signal_handler(signal.SIGINT, __exit)
|
||||
loop.add_signal_handler(signal.SIGTERM, __exit)
|
||||
except NotImplementedError:
|
||||
pass # Not implemented on Windows
|
||||
|
||||
try:
|
||||
loop.run_until_complete(daemon.start())
|
||||
loop.run_forever()
|
||||
except (GracefulExit, KeyboardInterrupt, asyncio.CancelledError):
|
||||
pass
|
||||
finally:
|
||||
loop.run_until_complete(daemon.stop())
|
||||
logging.shutdown()
|
||||
|
||||
def install_systemd_service():
|
||||
systemd_service = textwrap.dedent(f"""\
|
||||
[Unit]
|
||||
Description=LBRYnet
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart={sys.argv[0]} start --full-node
|
||||
""")
|
||||
subprocess.run(
|
||||
["systemctl", "edit", "--user", "--full", "--force", "lbrynet.service"],
|
||||
input=systemd_service, text=True, check=True,
|
||||
env=dict(os.environ, SYSTEMD_EDITOR="cp /dev/stdin"),
|
||||
)
|
||||
if hasattr(loop, 'shutdown_asyncgens'):
|
||||
loop.run_until_complete(loop.shutdown_asyncgens())
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
@@ -234,41 +290,38 @@ def main(argv=None):
|
||||
parser = get_argument_parser()
|
||||
args, command_args = parser.parse_known_args(argv)
|
||||
|
||||
conf = Config()
|
||||
conf.set_arguments(args)
|
||||
conf.set_environment()
|
||||
conf.set_default_paths()
|
||||
conf.set_persisted()
|
||||
conf = Config.create_from_arguments(args)
|
||||
for directory in (conf.data_dir, conf.download_dir, conf.wallet_dir):
|
||||
ensure_directory_exists(directory)
|
||||
|
||||
if args.cli_version:
|
||||
print(f"lbrynet {__version__}")
|
||||
print(f"lbrynet {lbrynet_version}")
|
||||
elif args.command == 'start':
|
||||
if args.help:
|
||||
args.start_parser.print_help()
|
||||
elif args.service == FullNode.name:
|
||||
return Daemon.from_config(FullNode, conf).run()
|
||||
elif args.service == LightClient.name:
|
||||
return Daemon.from_config(LightClient, conf).run()
|
||||
else:
|
||||
print(f'Only `start {FullNode.name}` or `start {LightClient.name}` is currently supported.')
|
||||
elif args.command == 'install':
|
||||
if args.help:
|
||||
args.install_parser.print_help()
|
||||
elif args.system == 'systemd':
|
||||
install_systemd_service()
|
||||
if args.initial_headers:
|
||||
ledger_path = os.path.join(conf.wallet_dir, 'lbc_mainnet')
|
||||
ensure_directory_exists(ledger_path)
|
||||
current_size = 0
|
||||
headers_path = os.path.join(ledger_path, 'headers')
|
||||
if os.path.exists(headers_path):
|
||||
current_size = os.stat(headers_path).st_size
|
||||
if os.stat(args.initial_headers).st_size > current_size:
|
||||
log.info('Copying header from %s to %s', args.initial_headers, headers_path)
|
||||
shutil.copy(args.initial_headers, headers_path)
|
||||
run_daemon(args, conf)
|
||||
elif args.command is not None:
|
||||
doc = args.doc
|
||||
api_method_name = args.api_method_name
|
||||
if args.replaced_by:
|
||||
print(f"{args.api_method_name} is deprecated, using {args.replaced_by['api_method_name']}.")
|
||||
doc = args.replaced_by['doc']
|
||||
api_method_name = args.replaced_by['api_method_name']
|
||||
if args.help:
|
||||
print(doc)
|
||||
else:
|
||||
parsed = docopt(
|
||||
# TODO: ugly hack because docopt doesn't support commands with spaces in them
|
||||
doc.replace(api_method_name.replace('_', ' '), api_method_name, 1),
|
||||
command_args
|
||||
)
|
||||
parsed = docopt(doc, command_args)
|
||||
params = set_kwargs(parsed)
|
||||
asyncio.get_event_loop().run_until_complete(execute_command(conf, api_method_name, params))
|
||||
elif args.group is not None:
|
||||
File renamed without changes.
@@ -0,0 +1,233 @@
|
||||
import asyncio
|
||||
import collections
|
||||
import logging
|
||||
import typing
|
||||
import aiohttp
|
||||
from lbry import utils
|
||||
from lbry.conf import Config
|
||||
from lbry.extras import system_info
|
||||
|
||||
ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
|
||||
ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
|
||||
|
||||
# Things We Track
|
||||
SERVER_STARTUP = 'Server Startup'
|
||||
SERVER_STARTUP_SUCCESS = 'Server Startup Success'
|
||||
SERVER_STARTUP_ERROR = 'Server Startup Error'
|
||||
DOWNLOAD_STARTED = 'Download Started'
|
||||
DOWNLOAD_ERRORED = 'Download Errored'
|
||||
DOWNLOAD_FINISHED = 'Download Finished'
|
||||
HEARTBEAT = 'Heartbeat'
|
||||
CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon
|
||||
NEW_CHANNEL = 'New Channel'
|
||||
CREDITS_SENT = 'Credits Sent'
|
||||
UPNP_SETUP = "UPnP Setup"
|
||||
|
||||
BLOB_BYTES_UPLOADED = 'Blob Bytes Uploaded'
|
||||
|
||||
|
||||
TIME_TO_FIRST_BYTES = "Time To First Bytes"
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _event_properties(installation_id: str, session_id: str,
|
||||
event_properties: typing.Optional[typing.Dict]) -> typing.Dict:
|
||||
properties = {
|
||||
'lbry_id': installation_id,
|
||||
'session_id': session_id,
|
||||
}
|
||||
properties.update(event_properties or {})
|
||||
return properties
|
||||
|
||||
|
||||
def _download_properties(conf: Config, external_ip: str, resolve_duration: float,
|
||||
total_duration: typing.Optional[float], download_id: str, name: str,
|
||||
outpoint: str, active_peer_count: typing.Optional[int],
|
||||
tried_peers_count: typing.Optional[int], connection_failures_count: typing.Optional[int],
|
||||
added_fixed_peers: bool, fixed_peer_delay: float, sd_hash: str,
|
||||
sd_download_duration: typing.Optional[float] = None,
|
||||
head_blob_hash: typing.Optional[str] = None,
|
||||
head_blob_length: typing.Optional[int] = None,
|
||||
head_blob_download_duration: typing.Optional[float] = None,
|
||||
error: typing.Optional[str] = None, error_msg: typing.Optional[str] = None,
|
||||
wallet_server: typing.Optional[str] = None) -> typing.Dict:
|
||||
return {
|
||||
"external_ip": external_ip,
|
||||
"download_id": download_id,
|
||||
"total_duration": round(total_duration, 4),
|
||||
"resolve_duration": None if not resolve_duration else round(resolve_duration, 4),
|
||||
"error": error,
|
||||
"error_message": error_msg,
|
||||
'name': name,
|
||||
"outpoint": outpoint,
|
||||
|
||||
"node_rpc_timeout": conf.node_rpc_timeout,
|
||||
"peer_connect_timeout": conf.peer_connect_timeout,
|
||||
"blob_download_timeout": conf.blob_download_timeout,
|
||||
"use_fixed_peers": len(conf.fixed_peers) > 0,
|
||||
"fixed_peer_delay": fixed_peer_delay,
|
||||
"added_fixed_peers": added_fixed_peers,
|
||||
"active_peer_count": active_peer_count,
|
||||
"tried_peers_count": tried_peers_count,
|
||||
|
||||
"sd_blob_hash": sd_hash,
|
||||
"sd_blob_duration": None if not sd_download_duration else round(sd_download_duration, 4),
|
||||
|
||||
"head_blob_hash": head_blob_hash,
|
||||
"head_blob_length": head_blob_length,
|
||||
"head_blob_duration": None if not head_blob_download_duration else round(head_blob_download_duration, 4),
|
||||
|
||||
"connection_failures_count": connection_failures_count,
|
||||
"wallet_server": wallet_server
|
||||
}
|
||||
|
||||
|
||||
def _make_context(platform):
|
||||
# see https://segment.com/docs/spec/common/#context
|
||||
# they say they'll ignore fields outside the spec, but evidently they don't
|
||||
context = {
|
||||
'app': {
|
||||
'version': platform['lbrynet_version'],
|
||||
'build': platform['build'],
|
||||
},
|
||||
# TODO: expand os info to give linux/osx specific info
|
||||
'os': {
|
||||
'name': platform['os_system'],
|
||||
'version': platform['os_release']
|
||||
},
|
||||
}
|
||||
if 'desktop' in platform and 'distro' in platform:
|
||||
context['os']['desktop'] = platform['desktop']
|
||||
context['os']['distro'] = platform['distro']
|
||||
return context
|
||||
|
||||
|
||||
class AnalyticsManager:
|
||||
def __init__(self, conf: Config, installation_id: str, session_id: str):
|
||||
self.conf = conf
|
||||
self.cookies = {}
|
||||
self.url = ANALYTICS_ENDPOINT
|
||||
self._write_key = utils.deobfuscate(ANALYTICS_TOKEN)
|
||||
self._tracked_data = collections.defaultdict(list)
|
||||
self.context = _make_context(system_info.get_platform())
|
||||
self.installation_id = installation_id
|
||||
self.session_id = session_id
|
||||
self.task: typing.Optional[asyncio.Task] = None
|
||||
self.external_ip: typing.Optional[str] = None
|
||||
|
||||
@property
|
||||
def enabled(self):
|
||||
return self.conf.share_usage_data
|
||||
|
||||
@property
|
||||
def is_started(self):
|
||||
return self.task is not None
|
||||
|
||||
async def start(self):
|
||||
if self.task is None:
|
||||
self.task = asyncio.create_task(self.run())
|
||||
|
||||
async def run(self):
|
||||
while True:
|
||||
if self.enabled:
|
||||
self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
await self._send_heartbeat()
|
||||
await asyncio.sleep(1800)
|
||||
|
||||
def stop(self):
|
||||
if self.task is not None and not self.task.done():
|
||||
self.task.cancel()
|
||||
|
||||
async def _post(self, data: typing.Dict):
|
||||
request_kwargs = {
|
||||
'method': 'POST',
|
||||
'url': self.url + '/track',
|
||||
'headers': {'Connection': 'Close'},
|
||||
'auth': aiohttp.BasicAuth(self._write_key, ''),
|
||||
'json': data,
|
||||
'cookies': self.cookies
|
||||
}
|
||||
try:
|
||||
async with utils.aiohttp_request(**request_kwargs) as response:
|
||||
self.cookies.update(response.cookies)
|
||||
except Exception as e:
|
||||
log.debug('Encountered an exception while POSTing to %s: ', self.url + '/track', exc_info=e)
|
||||
|
||||
async def track(self, event: typing.Dict):
|
||||
"""Send a single tracking event"""
|
||||
if self.enabled:
|
||||
log.debug('Sending track event: %s', event)
|
||||
await self._post(event)
|
||||
|
||||
async def send_upnp_setup_success_fail(self, success, status):
|
||||
await self.track(
|
||||
self._event(UPNP_SETUP, {
|
||||
'success': success,
|
||||
'status': status,
|
||||
})
|
||||
)
|
||||
|
||||
async def send_server_startup(self):
|
||||
await self.track(self._event(SERVER_STARTUP))
|
||||
|
||||
async def send_server_startup_success(self):
|
||||
await self.track(self._event(SERVER_STARTUP_SUCCESS))
|
||||
|
||||
async def send_server_startup_error(self, message):
|
||||
await self.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
|
||||
|
||||
async def send_time_to_first_bytes(self, resolve_duration: typing.Optional[float],
|
||||
total_duration: typing.Optional[float], download_id: str,
|
||||
name: str, outpoint: typing.Optional[str],
|
||||
found_peers_count: typing.Optional[int],
|
||||
tried_peers_count: typing.Optional[int],
|
||||
connection_failures_count: typing.Optional[int],
|
||||
added_fixed_peers: bool,
|
||||
fixed_peers_delay: float, sd_hash: str,
|
||||
sd_download_duration: typing.Optional[float] = None,
|
||||
head_blob_hash: typing.Optional[str] = None,
|
||||
head_blob_length: typing.Optional[int] = None,
|
||||
head_blob_duration: typing.Optional[int] = None,
|
||||
error: typing.Optional[str] = None,
|
||||
error_msg: typing.Optional[str] = None,
|
||||
wallet_server: typing.Optional[str] = None):
|
||||
await self.track(self._event(TIME_TO_FIRST_BYTES, _download_properties(
|
||||
self.conf, self.external_ip, resolve_duration, total_duration, download_id, name, outpoint,
|
||||
found_peers_count, tried_peers_count, connection_failures_count, added_fixed_peers, fixed_peers_delay,
|
||||
sd_hash, sd_download_duration, head_blob_hash, head_blob_length, head_blob_duration, error, error_msg,
|
||||
wallet_server
|
||||
)))
|
||||
|
||||
async def send_download_finished(self, download_id, name, sd_hash):
|
||||
await self.track(
|
||||
self._event(
|
||||
DOWNLOAD_FINISHED, {
|
||||
'download_id': download_id,
|
||||
'name': name,
|
||||
'stream_info': sd_hash
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
async def send_claim_action(self, action):
|
||||
await self.track(self._event(CLAIM_ACTION, {'action': action}))
|
||||
|
||||
async def send_new_channel(self):
|
||||
await self.track(self._event(NEW_CHANNEL))
|
||||
|
||||
async def send_credits_sent(self):
|
||||
await self.track(self._event(CREDITS_SENT))
|
||||
|
||||
async def _send_heartbeat(self):
|
||||
await self.track(self._event(HEARTBEAT))
|
||||
|
||||
def _event(self, event, properties: typing.Optional[typing.Dict] = None):
|
||||
return {
|
||||
'userId': 'lbry',
|
||||
'event': event,
|
||||
'properties': _event_properties(self.installation_id, self.session_id, properties),
|
||||
'context': self.context,
|
||||
'timestamp': utils.isonow()
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
from lbry.conf import Config
|
||||
from lbry.extras.cli import execute_command
|
||||
|
||||
|
||||
def daemon_rpc(conf: Config, method: str, **kwargs):
|
||||
return execute_command(conf, method, kwargs, callback=lambda data: data)
|
||||
@@ -0,0 +1,79 @@
|
||||
import logging
|
||||
import time
|
||||
import hashlib
|
||||
import binascii
|
||||
|
||||
import ecdsa
|
||||
from lbry import utils
|
||||
from lbry.crypto.hash import sha256
|
||||
from lbry.wallet.transaction import Output
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_encoded_signature(signature):
|
||||
signature = signature.encode() if isinstance(signature, str) else signature
|
||||
r = int(signature[:int(len(signature) / 2)], 16)
|
||||
s = int(signature[int(len(signature) / 2):], 16)
|
||||
return ecdsa.util.sigencode_der(r, s, len(signature) * 4)
|
||||
|
||||
|
||||
def cid2hash(claim_id: str) -> bytes:
|
||||
return binascii.unhexlify(claim_id.encode())[::-1]
|
||||
|
||||
|
||||
def is_comment_signed_by_channel(comment: dict, channel: Output, sign_comment_id=False):
|
||||
if isinstance(channel, Output):
|
||||
try:
|
||||
signing_field = comment['comment_id'] if sign_comment_id else comment['comment']
|
||||
return verify(channel, signing_field.encode(), comment, cid2hash(comment['channel_id']))
|
||||
except KeyError:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def verify(channel, data, signature, channel_hash=None):
|
||||
pieces = [
|
||||
signature['signing_ts'].encode(),
|
||||
channel_hash or channel.claim_hash,
|
||||
data
|
||||
]
|
||||
return Output.is_signature_valid(
|
||||
get_encoded_signature(signature['signature']),
|
||||
sha256(b''.join(pieces)),
|
||||
channel.claim.channel.public_key_bytes
|
||||
)
|
||||
|
||||
|
||||
def sign_comment(comment: dict, channel: Output, sign_comment_id=False):
|
||||
signing_field = comment['comment_id'] if sign_comment_id else comment['comment']
|
||||
comment.update(sign(channel, signing_field.encode()))
|
||||
|
||||
|
||||
def sign(channel, data):
|
||||
timestamp = str(int(time.time()))
|
||||
pieces = [timestamp.encode(), channel.claim_hash, data]
|
||||
digest = sha256(b''.join(pieces))
|
||||
signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
|
||||
return {
|
||||
'signature': binascii.hexlify(signature).decode(),
|
||||
'signing_ts': timestamp
|
||||
}
|
||||
|
||||
|
||||
def sign_reaction(reaction: dict, channel: Output):
|
||||
signing_field = reaction['channel_name']
|
||||
reaction.update(sign(channel, signing_field.encode()))
|
||||
|
||||
|
||||
async def jsonrpc_post(url: str, method: str, params: dict = None, **kwargs) -> any:
|
||||
params = params or {}
|
||||
params.update(kwargs)
|
||||
json_body = {'jsonrpc': '2.0', 'id': 1, 'method': method, 'params': params}
|
||||
async with utils.aiohttp_request('POST', url, json=json_body) as response:
|
||||
try:
|
||||
result = await response.json()
|
||||
return result['result'] if 'result' in result else result
|
||||
except Exception as cte:
|
||||
log.exception('Unable to decode response from server: %s', cte)
|
||||
return await response.text()
|
||||
@@ -0,0 +1,75 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from lbry.conf import Config
|
||||
from lbry.extras.daemon.componentmanager import ComponentManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ComponentType(type):
|
||||
def __new__(mcs, name, bases, newattrs):
|
||||
klass = type.__new__(mcs, name, bases, newattrs)
|
||||
if name != "Component" and newattrs['__module__'] != 'lbry.testcase':
|
||||
ComponentManager.default_component_classes[klass.component_name] = klass
|
||||
return klass
|
||||
|
||||
|
||||
class Component(metaclass=ComponentType):
|
||||
"""
|
||||
lbry-daemon component helper
|
||||
|
||||
Inheriting classes will be automatically registered with the ComponentManager and must implement setup and stop
|
||||
methods
|
||||
"""
|
||||
|
||||
depends_on = []
|
||||
component_name = None
|
||||
|
||||
def __init__(self, component_manager):
|
||||
self.conf: Config = component_manager.conf
|
||||
self.component_manager = component_manager
|
||||
self._running = False
|
||||
|
||||
def __lt__(self, other):
|
||||
return self.component_name < other.component_name
|
||||
|
||||
@property
|
||||
def running(self):
|
||||
return self._running
|
||||
|
||||
async def get_status(self):
|
||||
return
|
||||
|
||||
async def start(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
async def stop(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
async def _setup(self):
|
||||
try:
|
||||
result = await self.start()
|
||||
self._running = True
|
||||
return result
|
||||
except asyncio.CancelledError:
|
||||
log.info("Cancelled setup of %s component", self.__class__.__name__)
|
||||
raise
|
||||
except Exception as err:
|
||||
log.exception("Error setting up %s", self.component_name or self.__class__.__name__)
|
||||
raise err
|
||||
|
||||
async def _stop(self):
|
||||
try:
|
||||
result = await self.stop()
|
||||
self._running = False
|
||||
return result
|
||||
except asyncio.CancelledError:
|
||||
log.info("Cancelled stop of %s component", self.__class__.__name__)
|
||||
raise
|
||||
except Exception as err:
|
||||
log.exception("Error stopping %s", self.__class__.__name__)
|
||||
raise err
|
||||
@@ -0,0 +1,171 @@
|
||||
import logging
|
||||
import asyncio
|
||||
from lbry.conf import Config
|
||||
from lbry.error import ComponentStartConditionNotMetError
|
||||
from lbry.dht.peer import PeerManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RegisteredConditions:
|
||||
conditions = {}
|
||||
|
||||
|
||||
class RequiredConditionType(type):
|
||||
def __new__(mcs, name, bases, newattrs):
|
||||
klass = type.__new__(mcs, name, bases, newattrs)
|
||||
if name != "RequiredCondition":
|
||||
if klass.name in RegisteredConditions.conditions:
|
||||
raise SyntaxError("already have a component registered for \"%s\"" % klass.name)
|
||||
RegisteredConditions.conditions[klass.name] = klass
|
||||
return klass
|
||||
|
||||
|
||||
class RequiredCondition(metaclass=RequiredConditionType):
|
||||
name = ""
|
||||
component = ""
|
||||
message = ""
|
||||
|
||||
@staticmethod
|
||||
def evaluate(component):
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class ComponentManager:
|
||||
default_component_classes = {}
|
||||
|
||||
def __init__(self, conf: Config, analytics_manager=None, skip_components=None,
|
||||
peer_manager=None, **override_components):
|
||||
self.conf = conf
|
||||
self.skip_components = skip_components or []
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.analytics_manager = analytics_manager
|
||||
self.component_classes = {}
|
||||
self.components = set()
|
||||
self.started = asyncio.Event(loop=self.loop)
|
||||
self.peer_manager = peer_manager or PeerManager(asyncio.get_event_loop_policy().get_event_loop())
|
||||
|
||||
for component_name, component_class in self.default_component_classes.items():
|
||||
if component_name in override_components:
|
||||
component_class = override_components.pop(component_name)
|
||||
if component_name not in self.skip_components:
|
||||
self.component_classes[component_name] = component_class
|
||||
|
||||
if override_components:
|
||||
raise SyntaxError("unexpected components: %s" % override_components)
|
||||
|
||||
for component_class in self.component_classes.values():
|
||||
self.components.add(component_class(self))
|
||||
|
||||
def evaluate_condition(self, condition_name):
|
||||
if condition_name not in RegisteredConditions.conditions:
|
||||
raise NameError(condition_name)
|
||||
condition = RegisteredConditions.conditions[condition_name]
|
||||
try:
|
||||
component = self.get_component(condition.component)
|
||||
result = condition.evaluate(component)
|
||||
except Exception:
|
||||
log.exception('failed to evaluate condition:')
|
||||
result = False
|
||||
return result, "" if result else condition.message
|
||||
|
||||
def sort_components(self, reverse=False):
|
||||
"""
|
||||
Sort components by requirements
|
||||
"""
|
||||
steps = []
|
||||
staged = set()
|
||||
components = set(self.components)
|
||||
|
||||
# components with no requirements
|
||||
step = []
|
||||
for component in set(components):
|
||||
if not component.depends_on:
|
||||
step.append(component)
|
||||
staged.add(component.component_name)
|
||||
components.remove(component)
|
||||
|
||||
if step:
|
||||
step.sort()
|
||||
steps.append(step)
|
||||
|
||||
while components:
|
||||
step = []
|
||||
to_stage = set()
|
||||
for component in set(components):
|
||||
reqs_met = 0
|
||||
for needed in component.depends_on:
|
||||
if needed in staged:
|
||||
reqs_met += 1
|
||||
if reqs_met == len(component.depends_on):
|
||||
step.append(component)
|
||||
to_stage.add(component.component_name)
|
||||
components.remove(component)
|
||||
if step:
|
||||
step.sort()
|
||||
staged.update(to_stage)
|
||||
steps.append(step)
|
||||
elif components:
|
||||
raise ComponentStartConditionNotMetError(components)
|
||||
if reverse:
|
||||
steps.reverse()
|
||||
return steps
|
||||
|
||||
async def start(self):
|
||||
""" Start Components in sequence sorted by requirements """
|
||||
for stage in self.sort_components():
|
||||
needing_start = [
|
||||
component._setup() for component in stage if not component.running
|
||||
]
|
||||
if needing_start:
|
||||
await asyncio.wait(needing_start)
|
||||
self.started.set()
|
||||
|
||||
async def stop(self):
|
||||
"""
|
||||
Stop Components in reversed startup order
|
||||
"""
|
||||
stages = self.sort_components(reverse=True)
|
||||
for stage in stages:
|
||||
needing_stop = [
|
||||
component._stop() for component in stage if component.running
|
||||
]
|
||||
if needing_stop:
|
||||
await asyncio.wait(needing_stop)
|
||||
|
||||
def all_components_running(self, *component_names):
|
||||
"""
|
||||
Check if components are running
|
||||
|
||||
:return: (bool) True if all specified components are running
|
||||
"""
|
||||
components = {component.component_name: component for component in self.components}
|
||||
for component in component_names:
|
||||
if component not in components:
|
||||
raise NameError("%s is not a known Component" % component)
|
||||
if not components[component].running:
|
||||
return False
|
||||
return True
|
||||
|
||||
def get_components_status(self):
|
||||
"""
|
||||
List status of all the components, whether they are running or not
|
||||
|
||||
:return: (dict) {(str) component_name: (bool) True is running else False}
|
||||
"""
|
||||
return {
|
||||
component.component_name: component.running
|
||||
for component in self.components
|
||||
}
|
||||
|
||||
def get_actual_component(self, component_name):
|
||||
for component in self.components:
|
||||
if component.component_name == component_name:
|
||||
return component
|
||||
raise NameError(component_name)
|
||||
|
||||
def get_component(self, component_name):
|
||||
return self.get_actual_component(component_name).component
|
||||
|
||||
def has_component(self, component_name):
|
||||
return any(component for component in self.components if component_name == component.component_name)
|
||||
@@ -0,0 +1,606 @@
|
||||
import math
|
||||
import os
|
||||
import asyncio
|
||||
import logging
|
||||
import binascii
|
||||
import typing
|
||||
import base58
|
||||
|
||||
from aioupnp import __version__ as aioupnp_version
|
||||
from aioupnp.upnp import UPnP
|
||||
from aioupnp.fault import UPnPError
|
||||
|
||||
from lbry import utils
|
||||
from lbry.dht.node import Node
|
||||
from lbry.dht.peer import is_valid_public_ipv4
|
||||
from lbry.dht.blob_announcer import BlobAnnouncer
|
||||
from lbry.blob.blob_manager import BlobManager
|
||||
from lbry.blob_exchange.server import BlobServer
|
||||
from lbry.stream.stream_manager import StreamManager
|
||||
from lbry.file.file_manager import FileManager
|
||||
from lbry.extras.daemon.component import Component
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from lbry.torrent.torrent_manager import TorrentManager
|
||||
from lbry.wallet import WalletManager
|
||||
from lbry.wallet.usage_payment import WalletServerPayer
|
||||
try:
|
||||
from lbry.torrent.session import TorrentSession
|
||||
except ImportError:
|
||||
TorrentSession = None
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# settings must be initialized before this file is imported
|
||||
|
||||
DATABASE_COMPONENT = "database"
|
||||
BLOB_COMPONENT = "blob_manager"
|
||||
WALLET_COMPONENT = "wallet"
|
||||
WALLET_SERVER_PAYMENTS_COMPONENT = "wallet_server_payments"
|
||||
DHT_COMPONENT = "dht"
|
||||
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
|
||||
FILE_MANAGER_COMPONENT = "file_manager"
|
||||
PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
|
||||
UPNP_COMPONENT = "upnp"
|
||||
EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager"
|
||||
LIBTORRENT_COMPONENT = "libtorrent_component"
|
||||
|
||||
|
||||
class DatabaseComponent(Component):
|
||||
component_name = DATABASE_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.storage = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.storage
|
||||
|
||||
@staticmethod
|
||||
def get_current_db_revision():
|
||||
return 14
|
||||
|
||||
@property
|
||||
def revision_filename(self):
|
||||
return os.path.join(self.conf.data_dir, 'db_revision')
|
||||
|
||||
def _write_db_revision_file(self, version_num):
|
||||
with open(self.revision_filename, mode='w') as db_revision:
|
||||
db_revision.write(str(version_num))
|
||||
|
||||
async def start(self):
|
||||
# check directories exist, create them if they don't
|
||||
log.info("Loading databases")
|
||||
|
||||
if not os.path.exists(self.revision_filename):
|
||||
log.info("db_revision file not found. Creating it")
|
||||
self._write_db_revision_file(self.get_current_db_revision())
|
||||
|
||||
# check the db migration and run any needed migrations
|
||||
with open(self.revision_filename, "r") as revision_read_handle:
|
||||
old_revision = int(revision_read_handle.read().strip())
|
||||
|
||||
if old_revision > self.get_current_db_revision():
|
||||
raise Exception('This version of lbrynet is not compatible with the database\n'
|
||||
'Your database is revision %i, expected %i' %
|
||||
(old_revision, self.get_current_db_revision()))
|
||||
if old_revision < self.get_current_db_revision():
|
||||
from lbry.extras.daemon.migrator import dbmigrator # pylint: disable=import-outside-toplevel
|
||||
log.info("Upgrading your databases (revision %i to %i)", old_revision, self.get_current_db_revision())
|
||||
await asyncio.get_event_loop().run_in_executor(
|
||||
None, dbmigrator.migrate_db, self.conf, old_revision, self.get_current_db_revision()
|
||||
)
|
||||
self._write_db_revision_file(self.get_current_db_revision())
|
||||
log.info("Finished upgrading the databases.")
|
||||
|
||||
self.storage = SQLiteStorage(
|
||||
self.conf, os.path.join(self.conf.data_dir, "lbrynet.sqlite")
|
||||
)
|
||||
await self.storage.open()
|
||||
|
||||
async def stop(self):
|
||||
await self.storage.close()
|
||||
self.storage = None
|
||||
|
||||
|
||||
class WalletComponent(Component):
|
||||
component_name = WALLET_COMPONENT
|
||||
depends_on = [DATABASE_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.wallet_manager = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.wallet_manager
|
||||
|
||||
async def get_status(self):
|
||||
if self.wallet_manager is None:
|
||||
return
|
||||
is_connected = self.wallet_manager.ledger.network.is_connected
|
||||
sessions = []
|
||||
connected = None
|
||||
if is_connected:
|
||||
addr, port = self.wallet_manager.ledger.network.client.server
|
||||
connected = f"{addr}:{port}"
|
||||
sessions.append(self.wallet_manager.ledger.network.client)
|
||||
|
||||
result = {
|
||||
'connected': connected,
|
||||
'connected_features': self.wallet_manager.ledger.network.server_features,
|
||||
'servers': [
|
||||
{
|
||||
'host': session.server[0],
|
||||
'port': session.server[1],
|
||||
'latency': session.connection_latency,
|
||||
'availability': session.available,
|
||||
} for session in sessions
|
||||
],
|
||||
'known_servers': len(self.wallet_manager.ledger.network.known_hubs),
|
||||
'available_servers': 1 if is_connected else 0
|
||||
}
|
||||
|
||||
if self.wallet_manager.ledger.network.remote_height:
|
||||
local_height = self.wallet_manager.ledger.local_height_including_downloaded_height
|
||||
disk_height = len(self.wallet_manager.ledger.headers)
|
||||
remote_height = self.wallet_manager.ledger.network.remote_height
|
||||
download_height, target_height = local_height - disk_height, remote_height - disk_height
|
||||
if target_height > 0:
|
||||
progress = min(max(math.ceil(float(download_height) / float(target_height) * 100), 0), 100)
|
||||
else:
|
||||
progress = 100
|
||||
best_hash = await self.wallet_manager.get_best_blockhash()
|
||||
result.update({
|
||||
'headers_synchronization_progress': progress,
|
||||
'blocks': max(local_height, 0),
|
||||
'blocks_behind': max(remote_height - local_height, 0),
|
||||
'best_blockhash': best_hash,
|
||||
})
|
||||
|
||||
return result
|
||||
|
||||
async def start(self):
|
||||
log.info("Starting wallet")
|
||||
self.wallet_manager = await WalletManager.from_lbrynet_config(self.conf)
|
||||
await self.wallet_manager.start()
|
||||
|
||||
async def stop(self):
|
||||
await self.wallet_manager.stop()
|
||||
self.wallet_manager = None
|
||||
|
||||
|
||||
class WalletServerPaymentsComponent(Component):
|
||||
component_name = WALLET_SERVER_PAYMENTS_COMPONENT
|
||||
depends_on = [WALLET_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.usage_payment_service = WalletServerPayer(
|
||||
max_fee=self.conf.max_wallet_server_fee, analytics_manager=self.component_manager.analytics_manager,
|
||||
)
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[WalletServerPayer]:
|
||||
return self.usage_payment_service
|
||||
|
||||
async def start(self):
|
||||
wallet_manager = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
await self.usage_payment_service.start(wallet_manager.ledger, wallet_manager.default_wallet)
|
||||
|
||||
async def stop(self):
|
||||
await self.usage_payment_service.stop()
|
||||
|
||||
async def get_status(self):
|
||||
return {
|
||||
'max_fee': self.usage_payment_service.max_fee,
|
||||
'running': self.usage_payment_service.running
|
||||
}
|
||||
|
||||
|
||||
class BlobComponent(Component):
|
||||
component_name = BLOB_COMPONENT
|
||||
depends_on = [DATABASE_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.blob_manager: typing.Optional[BlobManager] = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[BlobManager]:
|
||||
return self.blob_manager
|
||||
|
||||
async def start(self):
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
data_store = None
|
||||
if DHT_COMPONENT not in self.component_manager.skip_components:
|
||||
dht_node: Node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
if dht_node:
|
||||
data_store = dht_node.protocol.data_store
|
||||
blob_dir = os.path.join(self.conf.data_dir, 'blobfiles')
|
||||
if not os.path.isdir(blob_dir):
|
||||
os.mkdir(blob_dir)
|
||||
self.blob_manager = BlobManager(self.component_manager.loop, blob_dir, storage, self.conf, data_store)
|
||||
return await self.blob_manager.setup()
|
||||
|
||||
async def stop(self):
|
||||
self.blob_manager.stop()
|
||||
|
||||
async def get_status(self):
|
||||
count = 0
|
||||
if self.blob_manager:
|
||||
count = len(self.blob_manager.completed_blob_hashes)
|
||||
return {
|
||||
'finished_blobs': count,
|
||||
'connections': {} if not self.blob_manager else self.blob_manager.connection_manager.status
|
||||
}
|
||||
|
||||
|
||||
class DHTComponent(Component):
|
||||
component_name = DHT_COMPONENT
|
||||
depends_on = [UPNP_COMPONENT, DATABASE_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.dht_node: typing.Optional[Node] = None
|
||||
self.external_udp_port = None
|
||||
self.external_peer_port = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[Node]:
|
||||
return self.dht_node
|
||||
|
||||
async def get_status(self):
|
||||
return {
|
||||
'node_id': None if not self.dht_node else binascii.hexlify(self.dht_node.protocol.node_id),
|
||||
'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.protocol.routing_table.get_peers())
|
||||
}
|
||||
|
||||
def get_node_id(self):
|
||||
node_id_filename = os.path.join(self.conf.data_dir, "node_id")
|
||||
if os.path.isfile(node_id_filename):
|
||||
with open(node_id_filename, "r") as node_id_file:
|
||||
return base58.b58decode(str(node_id_file.read()).strip())
|
||||
node_id = utils.generate_id()
|
||||
with open(node_id_filename, "w") as node_id_file:
|
||||
node_id_file.write(base58.b58encode(node_id).decode())
|
||||
return node_id
|
||||
|
||||
async def start(self):
|
||||
log.info("start the dht")
|
||||
upnp_component = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
self.external_peer_port = upnp_component.upnp_redirects.get("TCP", self.conf.tcp_port)
|
||||
self.external_udp_port = upnp_component.upnp_redirects.get("UDP", self.conf.udp_port)
|
||||
external_ip = upnp_component.external_ip
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
if not external_ip:
|
||||
external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
if not external_ip:
|
||||
log.warning("failed to get external ip")
|
||||
|
||||
self.dht_node = Node(
|
||||
self.component_manager.loop,
|
||||
self.component_manager.peer_manager,
|
||||
node_id=self.get_node_id(),
|
||||
internal_udp_port=self.conf.udp_port,
|
||||
udp_port=self.external_udp_port,
|
||||
external_ip=external_ip,
|
||||
peer_port=self.external_peer_port,
|
||||
rpc_timeout=self.conf.node_rpc_timeout,
|
||||
split_buckets_under_index=self.conf.split_buckets_under_index,
|
||||
storage=storage
|
||||
)
|
||||
self.dht_node.start(self.conf.network_interface, self.conf.known_dht_nodes)
|
||||
log.info("Started the dht")
|
||||
|
||||
async def stop(self):
|
||||
self.dht_node.stop()
|
||||
|
||||
|
||||
class HashAnnouncerComponent(Component):
|
||||
component_name = HASH_ANNOUNCER_COMPONENT
|
||||
depends_on = [DHT_COMPONENT, DATABASE_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.hash_announcer: typing.Optional[BlobAnnouncer] = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[BlobAnnouncer]:
|
||||
return self.hash_announcer
|
||||
|
||||
async def start(self):
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
self.hash_announcer = BlobAnnouncer(self.component_manager.loop, dht_node, storage)
|
||||
self.hash_announcer.start(self.conf.concurrent_blob_announcers)
|
||||
log.info("Started blob announcer")
|
||||
|
||||
async def stop(self):
|
||||
self.hash_announcer.stop()
|
||||
log.info("Stopped blob announcer")
|
||||
|
||||
async def get_status(self):
|
||||
return {
|
||||
'announce_queue_size': 0 if not self.hash_announcer else len(self.hash_announcer.announce_queue)
|
||||
}
|
||||
|
||||
|
||||
class FileManagerComponent(Component):
|
||||
component_name = FILE_MANAGER_COMPONENT
|
||||
depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.file_manager: typing.Optional[FileManager] = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[FileManager]:
|
||||
return self.file_manager
|
||||
|
||||
async def get_status(self):
|
||||
if not self.file_manager:
|
||||
return
|
||||
return {
|
||||
'managed_files': len(self.file_manager.get_filtered()),
|
||||
}
|
||||
|
||||
async def start(self):
|
||||
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
node = self.component_manager.get_component(DHT_COMPONENT) \
|
||||
if self.component_manager.has_component(DHT_COMPONENT) else None
|
||||
try:
|
||||
torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT) if TorrentSession else None
|
||||
except NameError:
|
||||
torrent = None
|
||||
log.info('Starting the file manager')
|
||||
loop = asyncio.get_event_loop()
|
||||
self.file_manager = FileManager(
|
||||
loop, self.conf, wallet, storage, self.component_manager.analytics_manager
|
||||
)
|
||||
self.file_manager.source_managers['stream'] = StreamManager(
|
||||
loop, self.conf, blob_manager, wallet, storage, node,
|
||||
)
|
||||
if TorrentSession and LIBTORRENT_COMPONENT not in self.conf.components_to_skip:
|
||||
self.file_manager.source_managers['torrent'] = TorrentManager(
|
||||
loop, self.conf, torrent, storage, self.component_manager.analytics_manager
|
||||
)
|
||||
await self.file_manager.start()
|
||||
log.info('Done setting up file manager')
|
||||
|
||||
async def stop(self):
|
||||
self.file_manager.stop()
|
||||
|
||||
|
||||
class TorrentComponent(Component):
|
||||
component_name = LIBTORRENT_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.torrent_session = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[TorrentSession]:
|
||||
return self.torrent_session
|
||||
|
||||
async def get_status(self):
|
||||
if not self.torrent_session:
|
||||
return
|
||||
return {
|
||||
'running': True, # TODO: what to return here?
|
||||
}
|
||||
|
||||
async def start(self):
|
||||
if TorrentSession:
|
||||
self.torrent_session = TorrentSession(asyncio.get_event_loop(), None)
|
||||
await self.torrent_session.bind() # TODO: specify host/port
|
||||
|
||||
async def stop(self):
|
||||
if self.torrent_session:
|
||||
await self.torrent_session.pause()
|
||||
|
||||
|
||||
class PeerProtocolServerComponent(Component):
|
||||
component_name = PEER_PROTOCOL_SERVER_COMPONENT
|
||||
depends_on = [UPNP_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.blob_server: typing.Optional[BlobServer] = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[BlobServer]:
|
||||
return self.blob_server
|
||||
|
||||
async def start(self):
|
||||
log.info("start blob server")
|
||||
blob_manager: BlobManager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
wallet: WalletManager = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
peer_port = self.conf.tcp_port
|
||||
address = await wallet.get_unused_address()
|
||||
self.blob_server = BlobServer(asyncio.get_event_loop(), blob_manager, address)
|
||||
self.blob_server.start_server(peer_port, interface=self.conf.network_interface)
|
||||
await self.blob_server.started_listening.wait()
|
||||
|
||||
async def stop(self):
|
||||
if self.blob_server:
|
||||
self.blob_server.stop_server()
|
||||
|
||||
|
||||
class UPnPComponent(Component):
|
||||
component_name = UPNP_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self._int_peer_port = self.conf.tcp_port
|
||||
self._int_dht_node_port = self.conf.udp_port
|
||||
self.use_upnp = self.conf.use_upnp
|
||||
self.upnp: typing.Optional[UPnP] = None
|
||||
self.upnp_redirects = {}
|
||||
self.external_ip: typing.Optional[str] = None
|
||||
self._maintain_redirects_task = None
|
||||
|
||||
@property
|
||||
def component(self) -> 'UPnPComponent':
|
||||
return self
|
||||
|
||||
async def _repeatedly_maintain_redirects(self, now=True):
|
||||
while True:
|
||||
if now:
|
||||
await self._maintain_redirects()
|
||||
await asyncio.sleep(360, loop=self.component_manager.loop)
|
||||
|
||||
async def _maintain_redirects(self):
|
||||
# setup the gateway if necessary
|
||||
if not self.upnp:
|
||||
try:
|
||||
self.upnp = await UPnP.discover(loop=self.component_manager.loop)
|
||||
log.info("found upnp gateway: %s", self.upnp.gateway.manufacturer_string)
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
|
||||
raise
|
||||
log.warning("upnp discovery failed: %s", err)
|
||||
self.upnp = None
|
||||
|
||||
# update the external ip
|
||||
external_ip = None
|
||||
if self.upnp:
|
||||
try:
|
||||
external_ip = await self.upnp.get_external_ip()
|
||||
if external_ip != "0.0.0.0" and not self.external_ip:
|
||||
log.info("got external ip from UPnP: %s", external_ip)
|
||||
except (asyncio.TimeoutError, UPnPError, NotImplementedError):
|
||||
pass
|
||||
if external_ip and not is_valid_public_ipv4(external_ip):
|
||||
log.warning("UPnP returned a private/reserved ip - %s, checking lbry.com fallback", external_ip)
|
||||
external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
if self.external_ip and self.external_ip != external_ip:
|
||||
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
|
||||
if external_ip:
|
||||
self.external_ip = external_ip
|
||||
dht_component = self.component_manager.get_component(DHT_COMPONENT)
|
||||
if dht_component:
|
||||
dht_node = dht_component.component
|
||||
dht_node.protocol.external_ip = external_ip
|
||||
# assert self.external_ip is not None # TODO: handle going/starting offline
|
||||
|
||||
if not self.upnp_redirects and self.upnp: # setup missing redirects
|
||||
log.info("add UPnP port mappings")
|
||||
upnp_redirects = {}
|
||||
if PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components:
|
||||
try:
|
||||
upnp_redirects["TCP"] = await self.upnp.get_next_mapping(
|
||||
self._int_peer_port, "TCP", "LBRY peer port", self._int_peer_port
|
||||
)
|
||||
except (UPnPError, asyncio.TimeoutError, NotImplementedError):
|
||||
pass
|
||||
if DHT_COMPONENT not in self.component_manager.skip_components:
|
||||
try:
|
||||
upnp_redirects["UDP"] = await self.upnp.get_next_mapping(
|
||||
self._int_dht_node_port, "UDP", "LBRY DHT port", self._int_dht_node_port
|
||||
)
|
||||
except (UPnPError, asyncio.TimeoutError, NotImplementedError):
|
||||
pass
|
||||
if upnp_redirects:
|
||||
log.info("set up redirects: %s", upnp_redirects)
|
||||
self.upnp_redirects.update(upnp_redirects)
|
||||
elif self.upnp: # check existing redirects are still active
|
||||
found = set()
|
||||
mappings = await self.upnp.get_redirects()
|
||||
for mapping in mappings:
|
||||
proto = mapping.protocol
|
||||
if proto in self.upnp_redirects and mapping.external_port == self.upnp_redirects[proto]:
|
||||
if mapping.lan_address == self.upnp.lan_address:
|
||||
found.add(proto)
|
||||
if 'UDP' not in found and DHT_COMPONENT not in self.component_manager.skip_components:
|
||||
try:
|
||||
udp_port = await self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")
|
||||
self.upnp_redirects['UDP'] = udp_port
|
||||
log.info("refreshed upnp redirect for dht port: %i", udp_port)
|
||||
except (asyncio.TimeoutError, UPnPError, NotImplementedError):
|
||||
del self.upnp_redirects['UDP']
|
||||
if 'TCP' not in found and PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components:
|
||||
try:
|
||||
tcp_port = await self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")
|
||||
self.upnp_redirects['TCP'] = tcp_port
|
||||
log.info("refreshed upnp redirect for peer port: %i", tcp_port)
|
||||
except (asyncio.TimeoutError, UPnPError, NotImplementedError):
|
||||
del self.upnp_redirects['TCP']
|
||||
if ('TCP' in self.upnp_redirects and
|
||||
PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components) and \
|
||||
('UDP' in self.upnp_redirects and DHT_COMPONENT not in self.component_manager.skip_components):
|
||||
if self.upnp_redirects:
|
||||
log.debug("upnp redirects are still active")
|
||||
|
||||
async def start(self):
|
||||
log.info("detecting external ip")
|
||||
if not self.use_upnp:
|
||||
self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
return
|
||||
success = False
|
||||
await self._maintain_redirects()
|
||||
if self.upnp:
|
||||
if not self.upnp_redirects and not all(
|
||||
x in self.component_manager.skip_components
|
||||
for x in (DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT)
|
||||
):
|
||||
log.error("failed to setup upnp")
|
||||
else:
|
||||
success = True
|
||||
if self.upnp_redirects:
|
||||
log.debug("set up upnp port redirects for gateway: %s", self.upnp.gateway.manufacturer_string)
|
||||
else:
|
||||
log.error("failed to setup upnp")
|
||||
if not self.external_ip:
|
||||
self.external_ip, probed_url = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
if self.external_ip:
|
||||
log.info("detected external ip using %s fallback", probed_url)
|
||||
if self.component_manager.analytics_manager:
|
||||
self.component_manager.loop.create_task(
|
||||
self.component_manager.analytics_manager.send_upnp_setup_success_fail(
|
||||
success, await self.get_status()
|
||||
)
|
||||
)
|
||||
self._maintain_redirects_task = self.component_manager.loop.create_task(
|
||||
self._repeatedly_maintain_redirects(now=False)
|
||||
)
|
||||
|
||||
async def stop(self):
|
||||
if self.upnp_redirects:
|
||||
log.info("Removing upnp redirects: %s", self.upnp_redirects)
|
||||
await asyncio.wait([
|
||||
self.upnp.delete_port_mapping(port, protocol) for protocol, port in self.upnp_redirects.items()
|
||||
], loop=self.component_manager.loop)
|
||||
if self._maintain_redirects_task and not self._maintain_redirects_task.done():
|
||||
self._maintain_redirects_task.cancel()
|
||||
|
||||
async def get_status(self):
|
||||
return {
|
||||
'aioupnp_version': aioupnp_version,
|
||||
'redirects': self.upnp_redirects,
|
||||
'gateway': 'No gateway found' if not self.upnp else self.upnp.gateway.manufacturer_string,
|
||||
'dht_redirect_set': 'UDP' in self.upnp_redirects,
|
||||
'peer_redirect_set': 'TCP' in self.upnp_redirects,
|
||||
'external_ip': self.external_ip
|
||||
}
|
||||
|
||||
|
||||
class ExchangeRateManagerComponent(Component):
|
||||
component_name = EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.exchange_rate_manager = ExchangeRateManager()
|
||||
|
||||
@property
|
||||
def component(self) -> ExchangeRateManager:
|
||||
return self.exchange_rate_manager
|
||||
|
||||
async def start(self):
|
||||
self.exchange_rate_manager.start()
|
||||
|
||||
async def stop(self):
|
||||
self.exchange_rate_manager.stop()
|
||||
File diff suppressed because it is too large.
Load diff
+92
-58
@@ -2,12 +2,13 @@ import json
|
||||
import time
|
||||
import asyncio
|
||||
import logging
|
||||
from statistics import median
|
||||
from decimal import Decimal
|
||||
from typing import Optional, Iterable, Type
|
||||
from aiohttp.client_exceptions import ContentTypeError
|
||||
from aiohttp.client_exceptions import ContentTypeError, ClientConnectionError
|
||||
from lbry.error import InvalidExchangeRateResponseError, CurrencyConversionError
|
||||
from lbry.utils import aiohttp_request
|
||||
from lbry.blockchain.dewies import lbc_to_dewies
|
||||
from lbry.wallet.dewies import lbc_to_dewies
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -58,9 +59,12 @@ class MarketFeed:
|
||||
raise NotImplementedError()
|
||||
|
||||
async def get_response(self):
|
||||
async with aiohttp_request('get', self.url, params=self.params, timeout=self.request_timeout) as response:
|
||||
async with aiohttp_request(
|
||||
'get', self.url, params=self.params,
|
||||
timeout=self.request_timeout, headers={"User-Agent": "lbrynet"}
|
||||
) as response:
|
||||
try:
|
||||
self._last_response = await response.json()
|
||||
self._last_response = await response.json(content_type=None)
|
||||
except ContentTypeError as e:
|
||||
self._last_response = {}
|
||||
log.warning("Could not parse exchange rate response from %s: %s", self.name, e.message)
|
||||
@@ -75,18 +79,23 @@ class MarketFeed:
|
||||
log.debug("Saving rate update %f for %s from %s", rate, self.market, self.name)
|
||||
self.rate = ExchangeRate(self.market, rate, int(time.time()))
|
||||
self.last_check = time.time()
|
||||
self.event.set()
|
||||
return self.rate
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("Timed out fetching exchange rate from %s.", self.name)
|
||||
except json.JSONDecodeError as e:
|
||||
log.warning("Could not parse exchange rate response from %s: %s", self.name, e.doc)
|
||||
msg = e.doc if '<html>' not in e.doc else 'unexpected content type.'
|
||||
log.warning("Could not parse exchange rate response from %s: %s", self.name, msg)
|
||||
log.debug(e.doc)
|
||||
except InvalidExchangeRateResponseError as e:
|
||||
log.warning(str(e))
|
||||
except ClientConnectionError as e:
|
||||
log.warning("Error trying to connect to exchange rate %s: %s", self.name, str(e))
|
||||
except Exception as e:
|
||||
log.exception("Exchange rate error (%s from %s):", self.market, self.name)
|
||||
finally:
|
||||
self.event.set()
|
||||
|
||||
async def keep_updated(self):
|
||||
while True:
|
||||
@@ -104,70 +113,92 @@ class MarketFeed:
|
||||
self.event.clear()
|
||||
|
||||
|
||||
class BittrexFeed(MarketFeed):
|
||||
class BaseBittrexFeed(MarketFeed):
|
||||
name = "Bittrex"
|
||||
market = "BTCLBC"
|
||||
url = "https://bittrex.com/api/v1.1/public/getmarkethistory"
|
||||
params = {'market': 'BTC-LBC', 'count': 50}
|
||||
market = None
|
||||
url = None
|
||||
fee = 0.0025
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'lastTradeRate' not in json_response:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return 1.0 / float(json_response['lastTradeRate'])
|
||||
|
||||
|
||||
class BittrexBTCFeed(BaseBittrexFeed):
|
||||
market = "BTCLBC"
|
||||
url = "https://api.bittrex.com/v3/markets/LBC-BTC/ticker"
|
||||
|
||||
|
||||
class BittrexUSDFeed(BaseBittrexFeed):
|
||||
market = "USDLBC"
|
||||
url = "https://api.bittrex.com/v3/markets/LBC-USD/ticker"
|
||||
|
||||
|
||||
class BaseCoinExFeed(MarketFeed):
|
||||
name = "CoinEx"
|
||||
market = None
|
||||
url = None
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'data' not in json_response or \
|
||||
'ticker' not in json_response['data'] or \
|
||||
'last' not in json_response['data']['ticker']:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return 1.0 / float(json_response['data']['ticker']['last'])
|
||||
|
||||
|
||||
class CoinExBTCFeed(BaseCoinExFeed):
|
||||
market = "BTCLBC"
|
||||
url = "https://api.coinex.com/v1/market/ticker?market=LBCBTC"
|
||||
|
||||
|
||||
class CoinExUSDFeed(BaseCoinExFeed):
|
||||
market = "USDLBC"
|
||||
url = "https://api.coinex.com/v1/market/ticker?market=LBCUSDT"
|
||||
|
||||
|
||||
class BaseHotbitFeed(MarketFeed):
|
||||
name = "hotbit"
|
||||
market = None
|
||||
url = "https://api.hotbit.io/api/v1/market.last"
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'result' not in json_response:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
trades = json_response['result']
|
||||
if len(trades) == 0:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'trades not found')
|
||||
totals = sum([i['Total'] for i in trades])
|
||||
qtys = sum([i['Quantity'] for i in trades])
|
||||
if totals <= 0 or qtys <= 0:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'quantities were not positive')
|
||||
vwap = totals / qtys
|
||||
return float(1.0 / vwap)
|
||||
return 1.0 / float(json_response['result'])
|
||||
|
||||
|
||||
class LBRYFeed(MarketFeed):
|
||||
name = "lbry.com"
|
||||
class HotbitBTCFeed(BaseHotbitFeed):
|
||||
market = "BTCLBC"
|
||||
url = "https://api.lbry.com/lbc/exchange_rate"
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'data' not in json_response:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return 1.0 / json_response['data']['lbc_btc']
|
||||
params = {"market": "LBC/BTC"}
|
||||
|
||||
|
||||
class LBRYBTCFeed(LBRYFeed):
|
||||
market = "USDBTC"
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'data' not in json_response:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return 1.0 / json_response['data']['btc_usd']
|
||||
class HotbitUSDFeed(BaseHotbitFeed):
|
||||
market = "USDLBC"
|
||||
params = {"market": "LBC/USDT"}
|
||||
|
||||
|
||||
class CryptonatorFeed(MarketFeed):
|
||||
name = "cryptonator.com"
|
||||
class UPbitBTCFeed(MarketFeed):
|
||||
name = "UPbit"
|
||||
market = "BTCLBC"
|
||||
url = "https://api.cryptonator.com/api/ticker/btc-lbc"
|
||||
url = "https://api.upbit.com/v1/ticker"
|
||||
params = {"markets": "BTC-LBC"}
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \
|
||||
'success' not in json_response or json_response['success'] is not True:
|
||||
if "error" in json_response or len(json_response) != 1 or 'trade_price' not in json_response[0]:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return float(json_response['ticker']['price'])
|
||||
|
||||
|
||||
class CryptonatorBTCFeed(CryptonatorFeed):
|
||||
market = "USDBTC"
|
||||
url = "https://api.cryptonator.com/api/ticker/usd-btc"
|
||||
return 1.0 / float(json_response[0]['trade_price'])
|
||||
|
||||
|
||||
FEEDS: Iterable[Type[MarketFeed]] = (
|
||||
LBRYFeed,
|
||||
LBRYBTCFeed,
|
||||
BittrexFeed,
|
||||
# CryptonatorFeed,
|
||||
# CryptonatorBTCFeed,
|
||||
BittrexBTCFeed,
|
||||
BittrexUSDFeed,
|
||||
CoinExBTCFeed,
|
||||
CoinExUSDFeed,
|
||||
HotbitBTCFeed,
|
||||
HotbitUSDFeed,
|
||||
UPbitBTCFeed,
|
||||
)
|
||||
|
||||
|
||||
@@ -191,20 +222,23 @@ class ExchangeRateManager:
|
||||
source.stop()
|
||||
|
||||
def convert_currency(self, from_currency, to_currency, amount):
|
||||
rates = [market.rate for market in self.market_feeds]
|
||||
log.debug("Converting %f %s to %s, rates: %s", amount, from_currency, to_currency, rates)
|
||||
log.debug(
|
||||
"Converting %f %s to %s, rates: %s",
|
||||
amount, from_currency, to_currency,
|
||||
[market.rate for market in self.market_feeds]
|
||||
)
|
||||
if from_currency == to_currency:
|
||||
return round(amount, 8)
|
||||
|
||||
rates = []
|
||||
for market in self.market_feeds:
|
||||
if (market.has_rate and market.is_online and
|
||||
market.rate.currency_pair == (from_currency, to_currency)):
|
||||
return round(amount * Decimal(market.rate.spot), 8)
|
||||
for market in self.market_feeds:
|
||||
if (market.has_rate and market.is_online and
|
||||
market.rate.currency_pair[0] == from_currency):
|
||||
return round(self.convert_currency(
|
||||
market.rate.currency_pair[1], to_currency, amount * Decimal(market.rate.spot)), 8)
|
||||
rates.append(market.rate.spot)
|
||||
|
||||
if rates:
|
||||
return round(amount * Decimal(median(rates)), 8)
|
||||
|
||||
raise CurrencyConversionError(
|
||||
f'Unable to convert {amount} from {from_currency} to {to_currency}')
|
||||
|
||||
@@ -0,0 +1,361 @@
|
||||
import logging
|
||||
from decimal import Decimal
|
||||
from binascii import hexlify, unhexlify
|
||||
from datetime import datetime
|
||||
from json import JSONEncoder
|
||||
|
||||
from google.protobuf.message import DecodeError
|
||||
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.schema.support import Support
|
||||
from lbry.torrent.torrent_manager import TorrentSource
|
||||
from lbry.wallet import Wallet, Ledger, Account, Transaction, Output
|
||||
from lbry.wallet.bip32 import PubKey
|
||||
from lbry.wallet.dewies import dewies_to_lbc
|
||||
from lbry.stream.managed_stream import ManagedStream
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def encode_txo_doc():
|
||||
return {
|
||||
'txid': "hash of transaction in hex",
|
||||
'nout': "position in the transaction",
|
||||
'height': "block where transaction was recorded",
|
||||
'amount': "value of the txo as a decimal",
|
||||
'address': "address of who can spend the txo",
|
||||
'confirmations': "number of confirmed blocks",
|
||||
'is_change': "payment to change address, only available when it can be determined",
|
||||
'is_received': "true if txo was sent from external account to this account",
|
||||
'is_spent': "true if txo is spent",
|
||||
'is_mine': "payment to one of your accounts, only available when it can be determined",
|
||||
'type': "one of 'claim', 'support' or 'purchase'",
|
||||
'name': "when type is 'claim' or 'support', this is the claim name",
|
||||
'claim_id': "when type is 'claim', 'support' or 'purchase', this is the claim id",
|
||||
'claim_op': "when type is 'claim', this determines if it is 'create' or 'update'",
|
||||
'value': "when type is 'claim' or 'support' with payload, this is the decoded protobuf payload",
|
||||
'value_type': "determines the type of the 'value' field: 'channel', 'stream', etc",
|
||||
'protobuf': "hex encoded raw protobuf version of 'value' field",
|
||||
'permanent_url': "when type is 'claim' or 'support', this is the long permanent claim URL",
|
||||
'claim': "for purchase outputs only, metadata of purchased claim",
|
||||
'reposted_claim': "for repost claims only, metadata of claim being reposted",
|
||||
'signing_channel': "for signed claims only, metadata of signing channel",
|
||||
'is_channel_signature_valid': "for signed claims only, whether signature is valid",
|
||||
'purchase_receipt': "metadata for the purchase transaction associated with this claim"
|
||||
}
|
||||
|
||||
|
||||
def encode_tx_doc():
|
||||
return {
|
||||
'txid': "hash of transaction in hex",
|
||||
'height': "block where transaction was recorded",
|
||||
'inputs': [encode_txo_doc()],
|
||||
'outputs': [encode_txo_doc()],
|
||||
'total_input': "sum of inputs as a decimal",
|
||||
'total_output': "sum of outputs, sans fee, as a decimal",
|
||||
'total_fee': "fee amount",
|
||||
'hex': "entire transaction encoded in hex",
|
||||
}
|
||||
|
||||
|
||||
def encode_account_doc():
|
||||
return {
|
||||
'id': 'account_id',
|
||||
'is_default': 'this account is used by default',
|
||||
'ledger': 'name of crypto currency and network',
|
||||
'name': 'optional account name',
|
||||
'seed': 'human friendly words from which account can be recreated',
|
||||
'encrypted': 'if account is encrypted',
|
||||
'private_key': 'extended private key',
|
||||
'public_key': 'extended public key',
|
||||
'address_generator': 'settings for generating addresses',
|
||||
'modified_on': 'date of last modification to account settings'
|
||||
}
|
||||
|
||||
|
||||
def encode_wallet_doc():
|
||||
return {
|
||||
'id': 'wallet_id',
|
||||
'name': 'optional wallet name',
|
||||
}
|
||||
|
||||
|
||||
def encode_file_doc():
|
||||
return {
|
||||
'streaming_url': '(str) url to stream the file using range requests',
|
||||
'completed': '(bool) true if download is completed',
|
||||
'file_name': '(str) name of file',
|
||||
'download_directory': '(str) download directory',
|
||||
'points_paid': '(float) credit paid to download file',
|
||||
'stopped': '(bool) true if download is stopped',
|
||||
'stream_hash': '(str) stream hash of file',
|
||||
'stream_name': '(str) stream name',
|
||||
'suggested_file_name': '(str) suggested file name',
|
||||
'sd_hash': '(str) sd hash of file',
|
||||
'download_path': '(str) download path of file',
|
||||
'mime_type': '(str) mime type of file',
|
||||
'key': '(str) key attached to file',
|
||||
'total_bytes_lower_bound': '(int) lower bound file size in bytes',
|
||||
'total_bytes': '(int) file upper bound size in bytes',
|
||||
'written_bytes': '(int) written size in bytes',
|
||||
'blobs_completed': '(int) number of fully downloaded blobs',
|
||||
'blobs_in_stream': '(int) total blobs on stream',
|
||||
'blobs_remaining': '(int) total blobs remaining to download',
|
||||
'status': '(str) downloader status',
|
||||
'claim_id': '(str) None if claim is not found else the claim id',
|
||||
'txid': '(str) None if claim is not found else the transaction id',
|
||||
'nout': '(int) None if claim is not found else the transaction output index',
|
||||
'outpoint': '(str) None if claim is not found else the tx and output',
|
||||
'metadata': '(dict) None if claim is not found else the claim metadata',
|
||||
'channel_claim_id': '(str) None if claim is not found or not signed',
|
||||
'channel_name': '(str) None if claim is not found or not signed',
|
||||
'claim_name': '(str) None if claim is not found else the claim name',
|
||||
'reflector_progress': '(int) reflector upload progress, 0 to 100',
|
||||
'uploading_to_reflector': '(bool) set to True when currently uploading to reflector'
|
||||
}
|
||||
|
||||
|
||||
class JSONResponseEncoder(JSONEncoder):
|
||||
|
||||
def __init__(self, *args, ledger: Ledger, include_protobuf=False, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.ledger = ledger
|
||||
self.include_protobuf = include_protobuf
|
||||
|
||||
def default(self, obj): # pylint: disable=method-hidden,arguments-differ,too-many-return-statements
|
||||
if isinstance(obj, Account):
|
||||
return self.encode_account(obj)
|
||||
if isinstance(obj, Wallet):
|
||||
return self.encode_wallet(obj)
|
||||
if isinstance(obj, (ManagedStream, TorrentSource)):
|
||||
return self.encode_file(obj)
|
||||
if isinstance(obj, Transaction):
|
||||
return self.encode_transaction(obj)
|
||||
if isinstance(obj, Output):
|
||||
return self.encode_output(obj)
|
||||
if isinstance(obj, Claim):
|
||||
return self.encode_claim(obj)
|
||||
if isinstance(obj, Support):
|
||||
return obj.to_dict()
|
||||
if isinstance(obj, PubKey):
|
||||
return obj.extended_key_string()
|
||||
if isinstance(obj, datetime):
|
||||
return obj.strftime("%Y%m%dT%H:%M:%S")
|
||||
if isinstance(obj, Decimal):
|
||||
return float(obj)
|
||||
if isinstance(obj, bytes):
|
||||
return obj.decode()
|
||||
return super().default(obj)
|
||||
|
||||
def encode_transaction(self, tx):
|
||||
return {
|
||||
'txid': tx.id,
|
||||
'height': tx.height,
|
||||
'inputs': [self.encode_input(txo) for txo in tx.inputs],
|
||||
'outputs': [self.encode_output(txo) for txo in tx.outputs],
|
||||
'total_input': dewies_to_lbc(tx.input_sum),
|
||||
'total_output': dewies_to_lbc(tx.input_sum - tx.fee),
|
||||
'total_fee': dewies_to_lbc(tx.fee),
|
||||
'hex': hexlify(tx.raw).decode(),
|
||||
}
|
||||
|
||||
def encode_output(self, txo, check_signature=True):
|
||||
if not txo:
|
||||
return
|
||||
tx_height = txo.tx_ref.height
|
||||
best_height = self.ledger.headers.height
|
||||
output = {
|
||||
'txid': txo.tx_ref.id,
|
||||
'nout': txo.position,
|
||||
'height': tx_height,
|
||||
'amount': dewies_to_lbc(txo.amount),
|
||||
'address': txo.get_address(self.ledger) if txo.has_address else None,
|
||||
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
|
||||
'timestamp': self.ledger.headers.estimated_timestamp(tx_height)
|
||||
}
|
||||
if txo.is_spent is not None:
|
||||
output['is_spent'] = txo.is_spent
|
||||
if txo.is_my_output is not None:
|
||||
output['is_my_output'] = txo.is_my_output
|
||||
if txo.is_my_input is not None:
|
||||
output['is_my_input'] = txo.is_my_input
|
||||
if txo.sent_supports is not None:
|
||||
output['sent_supports'] = dewies_to_lbc(txo.sent_supports)
|
||||
if txo.sent_tips is not None:
|
||||
output['sent_tips'] = dewies_to_lbc(txo.sent_tips)
|
||||
if txo.received_tips is not None:
|
||||
output['received_tips'] = dewies_to_lbc(txo.received_tips)
|
||||
if txo.is_internal_transfer is not None:
|
||||
output['is_internal_transfer'] = txo.is_internal_transfer
|
||||
|
||||
if txo.script.is_claim_name:
|
||||
output['type'] = 'claim'
|
||||
output['claim_op'] = 'create'
|
||||
elif txo.script.is_update_claim:
|
||||
output['type'] = 'claim'
|
||||
output['claim_op'] = 'update'
|
||||
elif txo.script.is_support_claim:
|
||||
output['type'] = 'support'
|
||||
elif txo.script.is_return_data:
|
||||
output['type'] = 'data'
|
||||
elif txo.purchase is not None:
|
||||
output['type'] = 'purchase'
|
||||
output['claim_id'] = txo.purchased_claim_id
|
||||
if txo.purchased_claim is not None:
|
||||
output['claim'] = self.encode_output(txo.purchased_claim)
|
||||
else:
|
||||
output['type'] = 'payment'
|
||||
|
||||
if txo.script.is_claim_involved:
|
||||
output.update({
|
||||
'name': txo.claim_name,
|
||||
'normalized_name': txo.normalized_name,
|
||||
'claim_id': txo.claim_id,
|
||||
'permanent_url': txo.permanent_url,
|
||||
'meta': self.encode_claim_meta(txo.meta.copy())
|
||||
})
|
||||
if 'short_url' in output['meta']:
|
||||
output['short_url'] = output['meta'].pop('short_url')
|
||||
if 'canonical_url' in output['meta']:
|
||||
output['canonical_url'] = output['meta'].pop('canonical_url')
|
||||
if txo.claims is not None:
|
||||
output['claims'] = [self.encode_output(o) for o in txo.claims]
|
||||
if txo.reposted_claim is not None:
|
||||
output['reposted_claim'] = self.encode_output(txo.reposted_claim)
|
||||
if txo.script.is_claim_name or txo.script.is_update_claim or txo.script.is_support_claim_data:
|
||||
try:
|
||||
output['value'] = txo.signable
|
||||
if self.include_protobuf:
|
||||
output['protobuf'] = hexlify(txo.signable.to_bytes())
|
||||
if txo.purchase_receipt is not None:
|
||||
output['purchase_receipt'] = self.encode_output(txo.purchase_receipt)
|
||||
if txo.script.is_claim_name or txo.script.is_update_claim:
|
||||
output['value_type'] = txo.claim.claim_type
|
||||
if txo.claim.is_channel:
|
||||
output['has_signing_key'] = txo.has_private_key
|
||||
if check_signature and txo.signable.is_signed:
|
||||
if txo.channel is not None:
|
||||
output['signing_channel'] = self.encode_output(txo.channel)
|
||||
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
|
||||
else:
|
||||
output['signing_channel'] = {'channel_id': txo.signable.signing_channel_id}
|
||||
output['is_channel_signature_valid'] = False
|
||||
except DecodeError:
|
||||
pass
|
||||
return output
|
||||
|
||||
def encode_claim_meta(self, meta):
|
||||
for key, value in meta.items():
|
||||
if key.endswith('_amount'):
|
||||
if isinstance(value, int):
|
||||
meta[key] = dewies_to_lbc(value)
|
||||
if 0 < meta.get('creation_height', 0) <= self.ledger.headers.height:
|
||||
meta['creation_timestamp'] = self.ledger.headers.estimated_timestamp(meta['creation_height'])
|
||||
return meta
|
||||
|
||||
def encode_input(self, txi):
|
||||
return self.encode_output(txi.txo_ref.txo, False) if txi.txo_ref.txo is not None else {
|
||||
'txid': txi.txo_ref.tx_ref.id,
|
||||
'nout': txi.txo_ref.position
|
||||
}
|
||||
|
||||
def encode_account(self, account):
|
||||
result = account.to_dict()
|
||||
result['id'] = account.id
|
||||
result.pop('certificates', None)
|
||||
result['is_default'] = self.ledger.accounts[0] == account
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def encode_wallet(wallet):
|
||||
return {
|
||||
'id': wallet.id,
|
||||
'name': wallet.name
|
||||
}
|
||||
|
||||
def encode_file(self, managed_stream):
|
||||
output_exists = managed_stream.output_file_exists
|
||||
tx_height = managed_stream.stream_claim_info.height
|
||||
best_height = self.ledger.headers.height
|
||||
is_stream = hasattr(managed_stream, 'stream_hash')
|
||||
if is_stream:
|
||||
total_bytes_lower_bound = managed_stream.descriptor.lower_bound_decrypted_length()
|
||||
total_bytes = managed_stream.descriptor.upper_bound_decrypted_length()
|
||||
else:
|
||||
total_bytes_lower_bound = total_bytes = managed_stream.torrent_length
|
||||
result = {
|
||||
'streaming_url': None,
|
||||
'completed': managed_stream.completed,
|
||||
'file_name': None,
|
||||
'download_directory': None,
|
||||
'download_path': None,
|
||||
'points_paid': 0.0,
|
||||
'stopped': not managed_stream.running,
|
||||
'stream_hash': None,
|
||||
'stream_name': None,
|
||||
'suggested_file_name': None,
|
||||
'sd_hash': None,
|
||||
'mime_type': None,
|
||||
'key': None,
|
||||
'total_bytes_lower_bound': total_bytes_lower_bound,
|
||||
'total_bytes': total_bytes,
|
||||
'written_bytes': managed_stream.written_bytes,
|
||||
'blobs_completed': None,
|
||||
'blobs_in_stream': None,
|
||||
'blobs_remaining': None,
|
||||
'status': managed_stream.status,
|
||||
'claim_id': managed_stream.claim_id,
|
||||
'txid': managed_stream.txid,
|
||||
'nout': managed_stream.nout,
|
||||
'outpoint': managed_stream.outpoint,
|
||||
'metadata': managed_stream.metadata,
|
||||
'protobuf': managed_stream.metadata_protobuf,
|
||||
'channel_claim_id': managed_stream.channel_claim_id,
|
||||
'channel_name': managed_stream.channel_name,
|
||||
'claim_name': managed_stream.claim_name,
|
||||
'content_fee': managed_stream.content_fee,
|
||||
'purchase_receipt': self.encode_output(managed_stream.purchase_receipt),
|
||||
'added_on': managed_stream.added_on,
|
||||
'height': tx_height,
|
||||
'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height,
|
||||
'timestamp': self.ledger.headers.estimated_timestamp(tx_height),
|
||||
'is_fully_reflected': False,
|
||||
'reflector_progress': False,
|
||||
'uploading_to_reflector': False
|
||||
}
|
||||
if is_stream:
|
||||
result.update({
|
||||
'streaming_url': managed_stream.stream_url,
|
||||
'stream_hash': managed_stream.stream_hash,
|
||||
'stream_name': managed_stream.descriptor.stream_name,
|
||||
'suggested_file_name': managed_stream.descriptor.suggested_file_name,
|
||||
'sd_hash': managed_stream.descriptor.sd_hash,
|
||||
'mime_type': managed_stream.mime_type,
|
||||
'key': managed_stream.descriptor.key,
|
||||
'blobs_completed': managed_stream.blobs_completed,
|
||||
'blobs_in_stream': managed_stream.blobs_in_stream,
|
||||
'blobs_remaining': managed_stream.blobs_remaining,
|
||||
'is_fully_reflected': managed_stream.is_fully_reflected,
|
||||
'reflector_progress': managed_stream.reflector_progress,
|
||||
'uploading_to_reflector': managed_stream.uploading_to_reflector
|
||||
})
|
||||
else:
|
||||
result.update({
|
||||
'streaming_url': f'file://{managed_stream.full_path}',
|
||||
})
|
||||
if output_exists:
|
||||
result.update({
|
||||
'file_name': managed_stream.file_name,
|
||||
'download_directory': managed_stream.download_directory,
|
||||
'download_path': managed_stream.full_path,
|
||||
})
|
||||
return result
|
||||
|
||||
def encode_claim(self, claim):
|
||||
encoded = getattr(claim, claim.claim_type).to_dict()
|
||||
if 'public_key' in encoded:
|
||||
encoded['public_key_id'] = self.ledger.public_key_to_address(
|
||||
unhexlify(encoded['public_key'])
|
||||
)
|
||||
return encoded
|
||||
File renamed without changes.
@@ -0,0 +1,70 @@
|
||||
# pylint: skip-file
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def migrate_db(conf, start, end):
|
||||
current = start
|
||||
while current < end:
|
||||
if current == 1:
|
||||
from .migrate1to2 import do_migration
|
||||
elif current == 2:
|
||||
from .migrate2to3 import do_migration
|
||||
elif current == 3:
|
||||
from .migrate3to4 import do_migration
|
||||
elif current == 4:
|
||||
from .migrate4to5 import do_migration
|
||||
elif current == 5:
|
||||
from .migrate5to6 import do_migration
|
||||
elif current == 6:
|
||||
from .migrate6to7 import do_migration
|
||||
elif current == 7:
|
||||
from .migrate7to8 import do_migration
|
||||
elif current == 8:
|
||||
from .migrate8to9 import do_migration
|
||||
elif current == 9:
|
||||
from .migrate9to10 import do_migration
|
||||
elif current == 10:
|
||||
from .migrate10to11 import do_migration
|
||||
elif current == 11:
|
||||
from .migrate11to12 import do_migration
|
||||
elif current == 12:
|
||||
from .migrate12to13 import do_migration
|
||||
elif current == 13:
|
||||
from .migrate13to14 import do_migration
|
||||
else:
|
||||
raise Exception(f"DB migration of version {current} to {current+1} is not available")
|
||||
try:
|
||||
do_migration(conf)
|
||||
except Exception:
|
||||
log.exception("failed to migrate database")
|
||||
if os.path.exists(os.path.join(conf.data_dir, "lbrynet.sqlite")):
|
||||
backup_name = f"rev_{current}_unmigrated_database"
|
||||
count = 0
|
||||
while os.path.exists(os.path.join(conf.data_dir, backup_name + ".sqlite")):
|
||||
count += 1
|
||||
backup_name = f"rev_{current}_unmigrated_database_{count}"
|
||||
backup_path = os.path.join(conf.data_dir, backup_name + ".sqlite")
|
||||
os.rename(os.path.join(conf.data_dir, "lbrynet.sqlite"), backup_path)
|
||||
log.info("made a backup of the unmigrated database: %s", backup_path)
|
||||
if os.path.isfile(os.path.join(conf.data_dir, "db_revision")):
|
||||
os.remove(os.path.join(conf.data_dir, "db_revision"))
|
||||
return None
|
||||
current += 1
|
||||
log.info("successfully migrated the database from revision %i to %i", current - 1, current)
|
||||
return None
|
||||
|
||||
|
||||
def run_migration_script():
|
||||
log_format = "(%(asctime)s)[%(filename)s:%(lineno)s] %(funcName)s(): %(message)s"
|
||||
logging.basicConfig(level=logging.DEBUG, format=log_format, filename="migrator.log")
|
||||
sys.stdout = open("migrator.out.log", 'w')
|
||||
sys.stderr = open("migrator.err.log", 'w')
|
||||
migrate_db(sys.argv[1], int(sys.argv[2]), int(sys.argv[3]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_migration_script()
|
||||
@@ -0,0 +1,54 @@
|
||||
import sqlite3
|
||||
import os
|
||||
import binascii
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
current_columns = []
|
||||
for col_info in cursor.execute("pragma table_info('file');").fetchall():
|
||||
current_columns.append(col_info[1])
|
||||
if 'content_fee' in current_columns or 'saved_file' in current_columns:
|
||||
connection.close()
|
||||
print("already migrated")
|
||||
return
|
||||
|
||||
cursor.execute(
|
||||
"pragma foreign_keys=off;"
|
||||
)
|
||||
|
||||
cursor.execute("""
|
||||
create table if not exists new_file (
|
||||
stream_hash text primary key not null references stream,
|
||||
file_name text,
|
||||
download_directory text,
|
||||
blob_data_rate real not null,
|
||||
status text not null,
|
||||
saved_file integer not null,
|
||||
content_fee text
|
||||
);
|
||||
""")
|
||||
for (stream_hash, file_name, download_dir, data_rate, status) in cursor.execute("select * from file").fetchall():
|
||||
saved_file = 0
|
||||
if download_dir != '{stream}' and file_name != '{stream}':
|
||||
try:
|
||||
if os.path.isfile(os.path.join(binascii.unhexlify(download_dir).decode(),
|
||||
binascii.unhexlify(file_name).decode())):
|
||||
saved_file = 1
|
||||
else:
|
||||
download_dir, file_name = None, None
|
||||
except Exception:
|
||||
download_dir, file_name = None, None
|
||||
else:
|
||||
download_dir, file_name = None, None
|
||||
cursor.execute(
|
||||
"insert into new_file values (?, ?, ?, ?, ?, ?, NULL)",
|
||||
(stream_hash, file_name, download_dir, data_rate, status, saved_file)
|
||||
)
|
||||
cursor.execute("drop table file")
|
||||
cursor.execute("alter table new_file rename to file")
|
||||
connection.commit()
|
||||
connection.close()
|
||||
@@ -0,0 +1,69 @@
|
||||
import sqlite3
|
||||
import os
|
||||
import time
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, 'lbrynet.sqlite')
|
||||
connection = sqlite3.connect(db_path)
|
||||
connection.row_factory = sqlite3.Row
|
||||
cursor = connection.cursor()
|
||||
|
||||
current_columns = []
|
||||
for col_info in cursor.execute("pragma table_info('file');").fetchall():
|
||||
current_columns.append(col_info[1])
|
||||
|
||||
if 'added_on' in current_columns:
|
||||
connection.close()
|
||||
print('already migrated')
|
||||
return
|
||||
|
||||
# follow 12 step schema change procedure
|
||||
cursor.execute("pragma foreign_keys=off")
|
||||
|
||||
# we don't have any indexes, views or triggers, so step 3 is skipped.
|
||||
cursor.execute("drop table if exists new_file")
|
||||
cursor.execute("""
|
||||
create table if not exists new_file (
|
||||
stream_hash text not null primary key references stream,
|
||||
file_name text,
|
||||
download_directory text,
|
||||
blob_data_rate text not null,
|
||||
status text not null,
|
||||
saved_file integer not null,
|
||||
content_fee text,
|
||||
added_on integer not null
|
||||
);
|
||||
|
||||
|
||||
""")
|
||||
|
||||
# step 5: transfer content from old to new
|
||||
select = "select * from file"
|
||||
for (stream_hash, file_name, download_dir, blob_rate, status, saved_file, fee) \
|
||||
in cursor.execute(select).fetchall():
|
||||
added_on = int(time.time())
|
||||
cursor.execute(
|
||||
"insert into new_file values (?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(stream_hash, file_name, download_dir, blob_rate, status, saved_file, fee, added_on)
|
||||
)
|
||||
|
||||
# step 6: drop old table
|
||||
cursor.execute("drop table file")
|
||||
|
||||
# step 7: rename new table to old table
|
||||
cursor.execute("alter table new_file rename to file")
|
||||
|
||||
# step 8: we aren't using indexes, views or triggers so skip
|
||||
# step 9: no views so skip
|
||||
# step 10: foreign key check
|
||||
cursor.execute("pragma foreign_key_check;")
|
||||
|
||||
# step 11: commit transaction
|
||||
connection.commit()
|
||||
|
||||
# step 12: re-enable foreign keys
|
||||
connection.execute("pragma foreign_keys=on;")
|
||||
|
||||
# done :)
|
||||
connection.close()
|
||||
@@ -0,0 +1,80 @@
|
||||
import os
|
||||
import sqlite3
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
current_columns = []
|
||||
for col_info in cursor.execute("pragma table_info('file');").fetchall():
|
||||
current_columns.append(col_info[1])
|
||||
if 'bt_infohash' in current_columns:
|
||||
connection.close()
|
||||
print("already migrated")
|
||||
return
|
||||
|
||||
cursor.executescript("""
|
||||
pragma foreign_keys=off;
|
||||
|
||||
create table if not exists torrent (
|
||||
bt_infohash char(20) not null primary key,
|
||||
tracker text,
|
||||
length integer not null,
|
||||
name text not null
|
||||
);
|
||||
|
||||
create table if not exists torrent_node ( -- BEP-0005
|
||||
bt_infohash char(20) not null references torrent,
|
||||
host text not null,
|
||||
port integer not null
|
||||
);
|
||||
|
||||
create table if not exists torrent_tracker ( -- BEP-0012
|
||||
bt_infohash char(20) not null references torrent,
|
||||
tracker text not null
|
||||
);
|
||||
|
||||
create table if not exists torrent_http_seed ( -- BEP-0017
|
||||
bt_infohash char(20) not null references torrent,
|
||||
http_seed text not null
|
||||
);
|
||||
|
||||
create table if not exists new_file (
|
||||
stream_hash char(96) references stream,
|
||||
bt_infohash char(20) references torrent,
|
||||
file_name text,
|
||||
download_directory text,
|
||||
blob_data_rate real not null,
|
||||
status text not null,
|
||||
saved_file integer not null,
|
||||
content_fee text,
|
||||
added_on integer not null
|
||||
);
|
||||
|
||||
create table if not exists new_content_claim (
|
||||
stream_hash char(96) references stream,
|
||||
bt_infohash char(20) references torrent,
|
||||
claim_outpoint text unique not null references claim
|
||||
);
|
||||
|
||||
insert into new_file (stream_hash, bt_infohash, file_name, download_directory, blob_data_rate, status,
|
||||
saved_file, content_fee, added_on) select
|
||||
stream_hash, NULL, file_name, download_directory, blob_data_rate, status, saved_file, content_fee,
|
||||
added_on
|
||||
from file;
|
||||
|
||||
insert or ignore into new_content_claim (stream_hash, bt_infohash, claim_outpoint)
|
||||
select stream_hash, NULL, claim_outpoint from content_claim;
|
||||
|
||||
drop table file;
|
||||
drop table content_claim;
|
||||
alter table new_file rename to file;
|
||||
alter table new_content_claim rename to content_claim;
|
||||
|
||||
pragma foreign_keys=on;
|
||||
""")
|
||||
|
||||
connection.commit()
|
||||
connection.close()
|
||||
@@ -0,0 +1,21 @@
|
||||
import os
|
||||
import sqlite3
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
cursor.executescript("""
|
||||
create table if not exists peer (
|
||||
node_id char(96) not null primary key,
|
||||
address text not null,
|
||||
udp_port integer not null,
|
||||
tcp_port integer,
|
||||
unique (address, udp_port)
|
||||
);
|
||||
""")
|
||||
|
||||
connection.commit()
|
||||
connection.close()
|
||||
@@ -0,0 +1,77 @@
|
||||
import sqlite3
|
||||
import os
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
UNSET_NOUT = -1
|
||||
|
||||
def do_migration(conf):
|
||||
log.info("Doing the migration")
|
||||
migrate_blockchainname_db(conf.data_dir)
|
||||
log.info("Migration succeeded")
|
||||
|
||||
|
||||
def migrate_blockchainname_db(db_dir):
|
||||
blockchainname_db = os.path.join(db_dir, "blockchainname.db")
|
||||
# skip migration on fresh installs
|
||||
if not os.path.isfile(blockchainname_db):
|
||||
return
|
||||
temp_db = sqlite3.connect(":memory:")
|
||||
db_file = sqlite3.connect(blockchainname_db)
|
||||
file_cursor = db_file.cursor()
|
||||
mem_cursor = temp_db.cursor()
|
||||
|
||||
mem_cursor.execute("create table if not exists name_metadata ("
|
||||
" name text, "
|
||||
" txid text, "
|
||||
" n integer, "
|
||||
" sd_hash text)")
|
||||
mem_cursor.execute("create table if not exists claim_ids ("
|
||||
" claimId text, "
|
||||
" name text, "
|
||||
" txid text, "
|
||||
" n integer)")
|
||||
temp_db.commit()
|
||||
|
||||
name_metadata = file_cursor.execute("select * from name_metadata").fetchall()
|
||||
claim_metadata = file_cursor.execute("select * from claim_ids").fetchall()
|
||||
|
||||
# fill n as V1_UNSET_NOUT, Wallet.py will be responsible for filling in correct n
|
||||
for name, txid, sd_hash in name_metadata:
|
||||
mem_cursor.execute(
|
||||
"insert into name_metadata values (?, ?, ?, ?) ",
|
||||
(name, txid, UNSET_NOUT, sd_hash))
|
||||
|
||||
for claim_id, name, txid in claim_metadata:
|
||||
mem_cursor.execute(
|
||||
"insert into claim_ids values (?, ?, ?, ?)",
|
||||
(claim_id, name, txid, UNSET_NOUT))
|
||||
temp_db.commit()
|
||||
|
||||
new_name_metadata = mem_cursor.execute("select * from name_metadata").fetchall()
|
||||
new_claim_metadata = mem_cursor.execute("select * from claim_ids").fetchall()
|
||||
|
||||
file_cursor.execute("drop table name_metadata")
|
||||
file_cursor.execute("create table name_metadata ("
|
||||
" name text, "
|
||||
" txid text, "
|
||||
" n integer, "
|
||||
" sd_hash text)")
|
||||
|
||||
for name, txid, n, sd_hash in new_name_metadata:
|
||||
file_cursor.execute(
|
||||
"insert into name_metadata values (?, ?, ?, ?) ", (name, txid, n, sd_hash))
|
||||
|
||||
file_cursor.execute("drop table claim_ids")
|
||||
file_cursor.execute("create table claim_ids ("
|
||||
" claimId text, "
|
||||
" name text, "
|
||||
" txid text, "
|
||||
" n integer)")
|
||||
|
||||
for claim_id, name, txid, n in new_claim_metadata:
|
||||
file_cursor.execute("insert into claim_ids values (?, ?, ?, ?)", (claim_id, name, txid, n))
|
||||
|
||||
db_file.commit()
|
||||
db_file.close()
|
||||
temp_db.close()
|
||||
@@ -0,0 +1,42 @@
|
||||
import sqlite3
|
||||
import os
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
log.info("Doing the migration")
|
||||
migrate_blockchainname_db(conf.data_dir)
|
||||
log.info("Migration succeeded")
|
||||
|
||||
|
||||
def migrate_blockchainname_db(db_dir):
|
||||
blockchainname_db = os.path.join(db_dir, "blockchainname.db")
|
||||
# skip migration on fresh installs
|
||||
if not os.path.isfile(blockchainname_db):
|
||||
return
|
||||
|
||||
db_file = sqlite3.connect(blockchainname_db)
|
||||
file_cursor = db_file.cursor()
|
||||
|
||||
tables = file_cursor.execute("SELECT tbl_name FROM sqlite_master "
|
||||
"WHERE type='table'").fetchall()
|
||||
|
||||
if 'tmp_name_metadata_table' in tables and 'name_metadata' not in tables:
|
||||
file_cursor.execute("ALTER TABLE tmp_name_metadata_table RENAME TO name_metadata")
|
||||
else:
|
||||
file_cursor.executescript(
|
||||
"CREATE TABLE IF NOT EXISTS tmp_name_metadata_table "
|
||||
" (name TEXT UNIQUE NOT NULL, "
|
||||
" txid TEXT NOT NULL, "
|
||||
" n INTEGER NOT NULL, "
|
||||
" sd_hash TEXT NOT NULL); "
|
||||
"INSERT OR IGNORE INTO tmp_name_metadata_table "
|
||||
" (name, txid, n, sd_hash) "
|
||||
" SELECT name, txid, n, sd_hash FROM name_metadata; "
|
||||
"DROP TABLE name_metadata; "
|
||||
"ALTER TABLE tmp_name_metadata_table RENAME TO name_metadata;"
|
||||
)
|
||||
db_file.commit()
|
||||
db_file.close()
|
||||
@@ -0,0 +1,85 @@
|
||||
import sqlite3
|
||||
import os
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
log.info("Doing the migration")
|
||||
migrate_blobs_db(conf.data_dir)
|
||||
log.info("Migration succeeded")
|
||||
|
||||
|
||||
def migrate_blobs_db(db_dir):
|
||||
"""
|
||||
We migrate the blobs.db used in BlobManager to have a "should_announce" column,
|
||||
and set this to True for blobs that are sd_hash's or head blobs (first blob in stream)
|
||||
"""
|
||||
|
||||
blobs_db = os.path.join(db_dir, "blobs.db")
|
||||
lbryfile_info_db = os.path.join(db_dir, 'lbryfile_info.db')
|
||||
|
||||
# skip migration on fresh installs
|
||||
if not os.path.isfile(blobs_db) and not os.path.isfile(lbryfile_info_db):
|
||||
return
|
||||
|
||||
# if blobs.db doesn't exist, skip migration
|
||||
if not os.path.isfile(blobs_db):
|
||||
log.info("blobs.db was not found but lbryfile_info.db was found, skipping migration")
|
||||
return
|
||||
|
||||
blobs_db_file = sqlite3.connect(blobs_db)
|
||||
blobs_db_cursor = blobs_db_file.cursor()
|
||||
|
||||
# check if new columns exist (it shouldn't) and create it
|
||||
try:
|
||||
blobs_db_cursor.execute("SELECT should_announce FROM blobs")
|
||||
except sqlite3.OperationalError:
|
||||
blobs_db_cursor.execute(
|
||||
"ALTER TABLE blobs ADD COLUMN should_announce integer NOT NULL DEFAULT 0")
|
||||
else:
|
||||
log.warning("should_announce already exists somehow, proceeding anyways")
|
||||
|
||||
# if lbryfile_info.db doesn't exist, skip marking blobs as should_announce = True
|
||||
if not os.path.isfile(lbryfile_info_db):
|
||||
log.error("lbryfile_info.db was not found, skipping check for should_announce")
|
||||
return
|
||||
|
||||
lbryfile_info_file = sqlite3.connect(lbryfile_info_db)
|
||||
lbryfile_info_cursor = lbryfile_info_file.cursor()
|
||||
|
||||
# find blobs that are stream descriptors
|
||||
lbryfile_info_cursor.execute('SELECT * FROM lbry_file_descriptors')
|
||||
descriptors = lbryfile_info_cursor.fetchall()
|
||||
should_announce_blob_hashes = []
|
||||
for d in descriptors:
|
||||
sd_blob_hash = (d[0],)
|
||||
should_announce_blob_hashes.append(sd_blob_hash)
|
||||
|
||||
# find blobs that are the first blob in a stream
|
||||
lbryfile_info_cursor.execute('SELECT * FROM lbry_file_blobs WHERE position = 0')
|
||||
blobs = lbryfile_info_cursor.fetchall()
|
||||
head_blob_hashes = []
|
||||
for b in blobs:
|
||||
blob_hash = (b[0],)
|
||||
should_announce_blob_hashes.append(blob_hash)
|
||||
|
||||
# now mark them as should_announce = True
|
||||
blobs_db_cursor.executemany('UPDATE blobs SET should_announce=1 WHERE blob_hash=?',
|
||||
should_announce_blob_hashes)
|
||||
|
||||
# Now run some final checks here to make sure migration succeeded
|
||||
try:
|
||||
blobs_db_cursor.execute("SELECT should_announce FROM blobs")
|
||||
except sqlite3.OperationalError:
|
||||
raise Exception('Migration failed, cannot find should_announce')
|
||||
|
||||
blobs_db_cursor.execute("SELECT * FROM blobs WHERE should_announce=1")
|
||||
blobs = blobs_db_cursor.fetchall()
|
||||
if len(blobs) != len(should_announce_blob_hashes):
|
||||
log.error("Some how not all blobs were marked as announceable")
|
||||
|
||||
blobs_db_file.commit()
|
||||
blobs_db_file.close()
|
||||
lbryfile_info_file.close()
|
||||
@@ -0,0 +1,62 @@
|
||||
import sqlite3
|
||||
import os
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
log.info("Doing the migration")
|
||||
add_lbry_file_metadata(conf.data_dir)
|
||||
log.info("Migration succeeded")
|
||||
|
||||
|
||||
def add_lbry_file_metadata(db_dir):
|
||||
"""
|
||||
We migrate the blobs.db used in BlobManager to have a "should_announce" column,
|
||||
and set this to True for blobs that are sd_hash's or head blobs (first blob in stream)
|
||||
"""
|
||||
|
||||
name_metadata = os.path.join(db_dir, "blockchainname.db")
|
||||
lbryfile_info_db = os.path.join(db_dir, 'lbryfile_info.db')
|
||||
|
||||
if not os.path.isfile(name_metadata) and not os.path.isfile(lbryfile_info_db):
|
||||
return
|
||||
|
||||
if not os.path.isfile(lbryfile_info_db):
|
||||
log.info("blockchainname.db was not found but lbryfile_info.db was found, skipping migration")
|
||||
return
|
||||
|
||||
name_metadata_db = sqlite3.connect(name_metadata)
|
||||
lbryfile_db = sqlite3.connect(lbryfile_info_db)
|
||||
name_metadata_cursor = name_metadata_db.cursor()
|
||||
lbryfile_cursor = lbryfile_db.cursor()
|
||||
|
||||
lbryfile_db.executescript(
|
||||
"create table if not exists lbry_file_metadata (" +
|
||||
" lbry_file integer primary key, " +
|
||||
" txid text, " +
|
||||
" n integer, " +
|
||||
" foreign key(lbry_file) references lbry_files(rowid)"
|
||||
")")
|
||||
|
||||
_files = lbryfile_cursor.execute("select rowid, stream_hash from lbry_files").fetchall()
|
||||
|
||||
lbry_files = {x[1]: x[0] for x in _files}
|
||||
for (sd_hash, stream_hash) in lbryfile_cursor.execute("select * "
|
||||
"from lbry_file_descriptors").fetchall():
|
||||
lbry_file_id = lbry_files[stream_hash]
|
||||
outpoint = name_metadata_cursor.execute("select txid, n from name_metadata "
|
||||
"where sd_hash=?",
|
||||
(sd_hash,)).fetchall()
|
||||
if outpoint:
|
||||
txid, nout = outpoint[0]
|
||||
lbryfile_cursor.execute("insert into lbry_file_metadata values (?, ?, ?)",
|
||||
(lbry_file_id, txid, nout))
|
||||
else:
|
||||
lbryfile_cursor.execute("insert into lbry_file_metadata values (?, ?, ?)",
|
||||
(lbry_file_id, None, None))
|
||||
lbryfile_db.commit()
|
||||
|
||||
lbryfile_db.close()
|
||||
name_metadata_db.close()
|
||||
@@ -0,0 +1,326 @@
|
||||
import sqlite3
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
from binascii import hexlify
|
||||
from lbry.schema.claim import Claim
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
CREATE_TABLES_QUERY = """
|
||||
pragma foreign_keys=on;
|
||||
pragma journal_mode=WAL;
|
||||
|
||||
create table if not exists blob (
|
||||
blob_hash char(96) primary key not null,
|
||||
blob_length integer not null,
|
||||
next_announce_time integer not null,
|
||||
should_announce integer not null default 0,
|
||||
status text not null
|
||||
);
|
||||
|
||||
create table if not exists stream (
|
||||
stream_hash char(96) not null primary key,
|
||||
sd_hash char(96) not null references blob,
|
||||
stream_key text not null,
|
||||
stream_name text not null,
|
||||
suggested_filename text not null
|
||||
);
|
||||
|
||||
create table if not exists stream_blob (
|
||||
stream_hash char(96) not null references stream,
|
||||
blob_hash char(96) references blob,
|
||||
position integer not null,
|
||||
iv char(32) not null,
|
||||
primary key (stream_hash, blob_hash)
|
||||
);
|
||||
|
||||
create table if not exists claim (
|
||||
claim_outpoint text not null primary key,
|
||||
claim_id char(40) not null,
|
||||
claim_name text not null,
|
||||
amount integer not null,
|
||||
height integer not null,
|
||||
serialized_metadata blob not null,
|
||||
channel_claim_id text,
|
||||
address text not null,
|
||||
claim_sequence integer not null
|
||||
);
|
||||
|
||||
create table if not exists file (
|
||||
stream_hash text primary key not null references stream,
|
||||
file_name text not null,
|
||||
download_directory text not null,
|
||||
blob_data_rate real not null,
|
||||
status text not null
|
||||
);
|
||||
|
||||
create table if not exists content_claim (
|
||||
stream_hash text unique not null references file,
|
||||
claim_outpoint text not null references claim,
|
||||
primary key (stream_hash, claim_outpoint)
|
||||
);
|
||||
|
||||
create table if not exists support (
|
||||
support_outpoint text not null primary key,
|
||||
claim_id text not null,
|
||||
amount integer not null,
|
||||
address text not null
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
def run_operation(db):
|
||||
def _decorate(fn):
|
||||
def _wrapper(*args):
|
||||
cursor = db.cursor()
|
||||
try:
|
||||
result = fn(cursor, *args)
|
||||
db.commit()
|
||||
return result
|
||||
except sqlite3.IntegrityError:
|
||||
db.rollback()
|
||||
raise
|
||||
return _wrapper
|
||||
return _decorate
|
||||
|
||||
|
||||
def verify_sd_blob(sd_hash, blob_dir):
|
||||
with open(os.path.join(blob_dir, sd_hash), "r") as sd_file:
|
||||
data = sd_file.read()
|
||||
sd_length = len(data)
|
||||
decoded = json.loads(data)
|
||||
assert set(decoded.keys()) == {
|
||||
'stream_name', 'blobs', 'stream_type', 'key', 'suggested_file_name', 'stream_hash'
|
||||
}, "invalid sd blob"
|
||||
for blob in sorted(decoded['blobs'], key=lambda x: int(x['blob_num']), reverse=True):
|
||||
if blob['blob_num'] == len(decoded['blobs']) - 1:
|
||||
assert {'length', 'blob_num', 'iv'} == set(blob.keys()), 'invalid stream terminator'
|
||||
assert blob['length'] == 0, 'non zero length stream terminator'
|
||||
else:
|
||||
assert {'blob_hash', 'length', 'blob_num', 'iv'} == set(blob.keys()), 'invalid stream blob'
|
||||
assert blob['length'] > 0, 'zero length stream blob'
|
||||
return decoded, sd_length
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
new_db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(new_db_path)
|
||||
|
||||
metadata_db = sqlite3.connect(os.path.join(conf.data_dir, "blockchainname.db"))
|
||||
lbryfile_db = sqlite3.connect(os.path.join(conf.data_dir, 'lbryfile_info.db'))
|
||||
blobs_db = sqlite3.connect(os.path.join(conf.data_dir, 'blobs.db'))
|
||||
|
||||
name_metadata_cursor = metadata_db.cursor()
|
||||
lbryfile_cursor = lbryfile_db.cursor()
|
||||
blobs_db_cursor = blobs_db.cursor()
|
||||
|
||||
old_rowid_to_outpoint = {
|
||||
rowid: (txid, nout) for (rowid, txid, nout) in
|
||||
lbryfile_cursor.execute("select * from lbry_file_metadata").fetchall()
|
||||
}
|
||||
|
||||
old_sd_hash_to_outpoint = {
|
||||
sd_hash: (txid, nout) for (txid, nout, sd_hash) in
|
||||
name_metadata_cursor.execute("select txid, n, sd_hash from name_metadata").fetchall()
|
||||
}
|
||||
|
||||
sd_hash_to_stream_hash = dict(
|
||||
lbryfile_cursor.execute("select sd_blob_hash, stream_hash from lbry_file_descriptors").fetchall()
|
||||
)
|
||||
|
||||
stream_hash_to_stream_blobs = {}
|
||||
|
||||
for (blob_hash, stream_hash, position, iv, length) in lbryfile_db.execute(
|
||||
"select * from lbry_file_blobs").fetchall():
|
||||
stream_blobs = stream_hash_to_stream_blobs.get(stream_hash, [])
|
||||
stream_blobs.append((blob_hash, length, position, iv))
|
||||
stream_hash_to_stream_blobs[stream_hash] = stream_blobs
|
||||
|
||||
claim_outpoint_queries = {}
|
||||
|
||||
for claim_query in metadata_db.execute(
|
||||
"select distinct c.txid, c.n, c.claimId, c.name, claim_cache.claim_sequence, claim_cache.claim_address, "
|
||||
"claim_cache.height, claim_cache.amount, claim_cache.claim_pb "
|
||||
"from claim_cache inner join claim_ids c on claim_cache.claim_id=c.claimId"):
|
||||
txid, nout = claim_query[0], claim_query[1]
|
||||
if (txid, nout) in claim_outpoint_queries:
|
||||
continue
|
||||
claim_outpoint_queries[(txid, nout)] = claim_query
|
||||
|
||||
@run_operation(connection)
|
||||
def _populate_blobs(transaction, blob_infos):
|
||||
transaction.executemany(
|
||||
"insert into blob values (?, ?, ?, ?, ?)",
|
||||
[(blob_hash, blob_length, int(next_announce_time), should_announce, "finished")
|
||||
for (blob_hash, blob_length, _, next_announce_time, should_announce) in blob_infos]
|
||||
)
|
||||
|
||||
@run_operation(connection)
|
||||
def _import_file(transaction, sd_hash, stream_hash, key, stream_name, suggested_file_name, data_rate,
|
||||
status, stream_blobs):
|
||||
try:
|
||||
transaction.execute(
|
||||
"insert or ignore into stream values (?, ?, ?, ?, ?)",
|
||||
(stream_hash, sd_hash, key, stream_name, suggested_file_name)
|
||||
)
|
||||
except sqlite3.IntegrityError:
|
||||
# failed because the sd isn't a known blob, we'll try to read the blob file and recover it
|
||||
return sd_hash
|
||||
|
||||
# insert any stream blobs that were missing from the blobs table
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?)",
|
||||
[
|
||||
(blob_hash, length, 0, 0, "pending")
|
||||
for (blob_hash, length, position, iv) in stream_blobs
|
||||
]
|
||||
)
|
||||
|
||||
# insert the stream blobs
|
||||
for blob_hash, length, position, iv in stream_blobs:
|
||||
transaction.execute(
|
||||
"insert or ignore into stream_blob values (?, ?, ?, ?)",
|
||||
(stream_hash, blob_hash, position, iv)
|
||||
)
|
||||
|
||||
download_dir = conf.download_dir
|
||||
if not isinstance(download_dir, bytes):
|
||||
download_dir = download_dir.encode()
|
||||
|
||||
# insert the file
|
||||
transaction.execute(
|
||||
"insert or ignore into file values (?, ?, ?, ?, ?)",
|
||||
(stream_hash, stream_name, hexlify(download_dir),
|
||||
data_rate, status)
|
||||
)
|
||||
|
||||
@run_operation(connection)
|
||||
def _add_recovered_blobs(transaction, blob_infos, sd_hash, sd_length):
|
||||
transaction.execute(
|
||||
"insert or replace into blob values (?, ?, ?, ?, ?)", (sd_hash, sd_length, 0, 1, "finished")
|
||||
)
|
||||
for blob in sorted(blob_infos, key=lambda x: x['blob_num'], reverse=True):
|
||||
if blob['blob_num'] < len(blob_infos) - 1:
|
||||
transaction.execute(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?)",
|
||||
(blob['blob_hash'], blob['length'], 0, 0, "pending")
|
||||
)
|
||||
|
||||
@run_operation(connection)
|
||||
def _make_db(new_db):
|
||||
# create the new tables
|
||||
new_db.executescript(CREATE_TABLES_QUERY)
|
||||
|
||||
# first migrate the blobs
|
||||
blobs = blobs_db_cursor.execute("select * from blobs").fetchall()
|
||||
_populate_blobs(blobs) # pylint: disable=no-value-for-parameter
|
||||
log.info("migrated %i blobs", new_db.execute("select count(*) from blob").fetchone()[0])
|
||||
|
||||
# used to store the query arguments if we need to try re-importing the lbry file later
|
||||
file_args = {} # <sd_hash>: args tuple
|
||||
|
||||
file_outpoints = {} # <outpoint tuple>: sd_hash
|
||||
|
||||
# get the file and stream queries ready
|
||||
for (rowid, sd_hash, stream_hash, key, stream_name, suggested_file_name, data_rate, status) in \
|
||||
lbryfile_db.execute(
|
||||
"select distinct lbry_files.rowid, d.sd_blob_hash, lbry_files.*, o.blob_data_rate, o.status "
|
||||
"from lbry_files "
|
||||
"inner join lbry_file_descriptors d on lbry_files.stream_hash=d.stream_hash "
|
||||
"inner join lbry_file_options o on lbry_files.stream_hash=o.stream_hash"):
|
||||
|
||||
# this is try to link the file to a content claim after we've imported all the files
|
||||
if rowid in old_rowid_to_outpoint:
|
||||
file_outpoints[old_rowid_to_outpoint[rowid]] = sd_hash
|
||||
elif sd_hash in old_sd_hash_to_outpoint:
|
||||
file_outpoints[old_sd_hash_to_outpoint[sd_hash]] = sd_hash
|
||||
|
||||
sd_hash_to_stream_hash[sd_hash] = stream_hash
|
||||
if stream_hash in stream_hash_to_stream_blobs:
|
||||
file_args[sd_hash] = (
|
||||
sd_hash, stream_hash, key, stream_name,
|
||||
suggested_file_name, data_rate or 0.0,
|
||||
status, stream_hash_to_stream_blobs.pop(stream_hash)
|
||||
)
|
||||
|
||||
# used to store the query arguments if we need to try re-importing the claim
|
||||
claim_queries = {} # <sd_hash>: claim query tuple
|
||||
|
||||
# get the claim queries ready, only keep those with associated files
|
||||
for outpoint, sd_hash in file_outpoints.items():
|
||||
if outpoint in claim_outpoint_queries:
|
||||
claim_queries[sd_hash] = claim_outpoint_queries[outpoint]
|
||||
|
||||
# insert the claims
|
||||
new_db.executemany(
|
||||
"insert or ignore into claim values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
[
|
||||
(
|
||||
"%s:%i" % (claim_arg_tup[0], claim_arg_tup[1]), claim_arg_tup[2], claim_arg_tup[3],
|
||||
claim_arg_tup[7], claim_arg_tup[6], claim_arg_tup[8],
|
||||
Claim.from_bytes(claim_arg_tup[8]).signing_channel_id, claim_arg_tup[5], claim_arg_tup[4]
|
||||
)
|
||||
for sd_hash, claim_arg_tup in claim_queries.items() if claim_arg_tup
|
||||
] # sd_hash, (txid, nout, claim_id, name, sequence, address, height, amount, serialized)
|
||||
)
|
||||
|
||||
log.info("migrated %i claims", new_db.execute("select count(*) from claim").fetchone()[0])
|
||||
|
||||
damaged_stream_sds = []
|
||||
# import the files and get sd hashes of streams to attempt recovering
|
||||
for sd_hash, file_query in file_args.items():
|
||||
failed_sd = _import_file(*file_query)
|
||||
if failed_sd:
|
||||
damaged_stream_sds.append(failed_sd)
|
||||
|
||||
# recover damaged streams
|
||||
if damaged_stream_sds:
|
||||
blob_dir = os.path.join(conf.data_dir, "blobfiles")
|
||||
damaged_sds_on_disk = [] if not os.path.isdir(blob_dir) else list({p for p in os.listdir(blob_dir)
|
||||
if p in damaged_stream_sds})
|
||||
for damaged_sd in damaged_sds_on_disk:
|
||||
try:
|
||||
decoded, sd_length = verify_sd_blob(damaged_sd, blob_dir)
|
||||
blobs = decoded['blobs']
|
||||
_add_recovered_blobs(blobs, damaged_sd, sd_length) # pylint: disable=no-value-for-parameter
|
||||
_import_file(*file_args[damaged_sd])
|
||||
damaged_stream_sds.remove(damaged_sd)
|
||||
except (OSError, ValueError, TypeError, AssertionError, sqlite3.IntegrityError):
|
||||
continue
|
||||
|
||||
log.info("migrated %i files", new_db.execute("select count(*) from file").fetchone()[0])
|
||||
|
||||
# associate the content claims to their respective files
|
||||
for claim_arg_tup in claim_queries.values():
|
||||
if claim_arg_tup and (claim_arg_tup[0], claim_arg_tup[1]) in file_outpoints \
|
||||
and file_outpoints[(claim_arg_tup[0], claim_arg_tup[1])] in sd_hash_to_stream_hash:
|
||||
try:
|
||||
new_db.execute(
|
||||
"insert or ignore into content_claim values (?, ?)",
|
||||
(
|
||||
sd_hash_to_stream_hash.get(file_outpoints.get((claim_arg_tup[0], claim_arg_tup[1]))),
|
||||
"%s:%i" % (claim_arg_tup[0], claim_arg_tup[1])
|
||||
)
|
||||
)
|
||||
except sqlite3.IntegrityError:
|
||||
continue
|
||||
|
||||
log.info("migrated %i content claims", new_db.execute("select count(*) from content_claim").fetchone()[0])
|
||||
try:
|
||||
_make_db() # pylint: disable=no-value-for-parameter
|
||||
except sqlite3.OperationalError as err:
|
||||
if err.message == "table blob has 7 columns but 5 values were supplied":
|
||||
log.warning("detected a failed previous migration to revision 6, repairing it")
|
||||
connection.close()
|
||||
os.remove(new_db_path)
|
||||
return do_migration(conf)
|
||||
raise err
|
||||
|
||||
connection.close()
|
||||
blobs_db.close()
|
||||
lbryfile_db.close()
|
||||
metadata_db.close()
|
||||
# os.remove(os.path.join(db_dir, "blockchainname.db"))
|
||||
# os.remove(os.path.join(db_dir, 'lbryfile_info.db'))
|
||||
# os.remove(os.path.join(db_dir, 'blobs.db'))
|
||||
@@ -0,0 +1,13 @@
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
cursor.executescript("alter table blob add last_announced_time integer;")
|
||||
cursor.executescript("alter table blob add single_announce integer;")
|
||||
cursor.execute("update blob set next_announce_time=0")
|
||||
connection.commit()
|
||||
connection.close()
|
||||
@@ -0,0 +1,21 @@
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
cursor.executescript(
|
||||
"""
|
||||
create table reflected_stream (
|
||||
sd_hash text not null,
|
||||
reflector_address text not null,
|
||||
timestamp integer,
|
||||
primary key (sd_hash, reflector_address)
|
||||
);
|
||||
"""
|
||||
)
|
||||
connection.commit()
|
||||
connection.close()
|
||||
@@ -0,0 +1,47 @@
|
||||
import sqlite3
|
||||
import logging
|
||||
import os
|
||||
from lbry.blob.blob_info import BlobInfo
|
||||
from lbry.stream.descriptor import StreamDescriptor
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
blob_dir = os.path.join(conf.data_dir, "blobfiles")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
query = "select stream_name, stream_key, suggested_filename, sd_hash, stream_hash from stream"
|
||||
streams = cursor.execute(query).fetchall()
|
||||
|
||||
blobs = cursor.execute("select s.stream_hash, s.position, s.iv, b.blob_hash, b.blob_length from stream_blob s "
|
||||
"left outer join blob b ON b.blob_hash=s.blob_hash order by s.position").fetchall()
|
||||
blobs_by_stream = {}
|
||||
for stream_hash, position, iv, blob_hash, blob_length in blobs:
|
||||
blobs_by_stream.setdefault(stream_hash, []).append(BlobInfo(position, blob_length or 0, iv, blob_hash))
|
||||
|
||||
for stream_name, stream_key, suggested_filename, sd_hash, stream_hash in streams:
|
||||
sd = StreamDescriptor(None, blob_dir, stream_name, stream_key, suggested_filename,
|
||||
blobs_by_stream[stream_hash], stream_hash, sd_hash)
|
||||
if sd_hash != sd.calculate_sd_hash():
|
||||
log.info("Stream for descriptor %s is invalid, cleaning it up", sd_hash)
|
||||
blob_hashes = [blob.blob_hash for blob in blobs_by_stream[stream_hash]]
|
||||
delete_stream(cursor, stream_hash, sd_hash, blob_hashes, blob_dir)
|
||||
|
||||
connection.commit()
|
||||
connection.close()
|
||||
|
||||
|
||||
def delete_stream(transaction, stream_hash, sd_hash, blob_hashes, blob_dir):
|
||||
transaction.execute("delete from content_claim where stream_hash=? ", (stream_hash,))
|
||||
transaction.execute("delete from file where stream_hash=? ", (stream_hash, ))
|
||||
transaction.execute("delete from stream_blob where stream_hash=?", (stream_hash, ))
|
||||
transaction.execute("delete from stream where stream_hash=? ", (stream_hash, ))
|
||||
transaction.execute("delete from blob where blob_hash=?", (sd_hash, ))
|
||||
for blob_hash in blob_hashes:
|
||||
transaction.execute("delete from blob where blob_hash=?", (blob_hash, ))
|
||||
file_path = os.path.join(blob_dir, blob_hash)
|
||||
if os.path.isfile(file_path):
|
||||
os.unlink(file_path)
|
||||
@@ -0,0 +1,20 @@
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
query = "select stream_hash, sd_hash from main.stream"
|
||||
for stream_hash, sd_hash in cursor.execute(query).fetchall():
|
||||
head_blob_hash = cursor.execute(
|
||||
"select blob_hash from stream_blob where position = 0 and stream_hash = ?",
|
||||
(stream_hash,)
|
||||
).fetchone()
|
||||
if not head_blob_hash:
|
||||
continue
|
||||
cursor.execute("update blob set should_announce=1 where blob_hash in (?, ?)", (sd_hash, head_blob_hash[0],))
|
||||
connection.commit()
|
||||
connection.close()
|
||||
@@ -0,0 +1,31 @@
|
||||
import logging
|
||||
from aiohttp import web
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def ensure_request_allowed(request, conf):
|
||||
if is_request_allowed(request, conf):
|
||||
return
|
||||
if conf.allowed_origin:
|
||||
log.warning(
|
||||
"API requests with Origin '%s' are not allowed, "
|
||||
"configuration 'allowed_origin' limits requests to: '%s'",
|
||||
request.headers.get('Origin'), conf.allowed_origin
|
||||
)
|
||||
else:
|
||||
log.warning(
|
||||
"API requests with Origin '%s' are not allowed, "
|
||||
"update configuration 'allowed_origin' to enable this origin.",
|
||||
request.headers.get('Origin')
|
||||
)
|
||||
raise web.HTTPForbidden()
|
||||
|
||||
|
||||
def is_request_allowed(request, conf) -> bool:
|
||||
origin = request.headers.get('Origin')
|
||||
return (
|
||||
origin is None or
|
||||
origin == conf.allowed_origin or
|
||||
conf.allowed_origin == '*'
|
||||
)
|
||||
@@ -0,0 +1,866 @@
|
||||
import os
|
||||
import logging
|
||||
import sqlite3
|
||||
import typing
|
||||
import asyncio
|
||||
import binascii
|
||||
import time
|
||||
from typing import Optional
|
||||
from lbry.wallet import SQLiteMixin
|
||||
from lbry.conf import Config
|
||||
from lbry.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
||||
from lbry.wallet.transaction import Transaction, Output
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.dht.constants import DATA_EXPIRATION
|
||||
from lbry.blob.blob_info import BlobInfo
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.blob.blob_file import BlobFile
|
||||
from lbry.stream.descriptor import StreamDescriptor
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def calculate_effective_amount(amount: str, supports: typing.Optional[typing.List[typing.Dict]] = None) -> str:
|
||||
return dewies_to_lbc(
|
||||
lbc_to_dewies(amount) + sum([lbc_to_dewies(support['amount']) for support in supports])
|
||||
)
|
||||
|
||||
|
||||
class StoredContentClaim:
|
||||
def __init__(self, outpoint: Optional[str] = None, claim_id: Optional[str] = None, name: Optional[str] = None,
|
||||
amount: Optional[int] = None, height: Optional[int] = None, serialized: Optional[str] = None,
|
||||
channel_claim_id: Optional[str] = None, address: Optional[str] = None,
|
||||
claim_sequence: Optional[int] = None, channel_name: Optional[str] = None):
|
||||
self.claim_id = claim_id
|
||||
self.outpoint = outpoint
|
||||
self.claim_name = name
|
||||
self.amount = amount
|
||||
self.height = height
|
||||
self.claim: typing.Optional[Claim] = None if not serialized else Claim.from_bytes(
|
||||
binascii.unhexlify(serialized)
|
||||
)
|
||||
self.claim_address = address
|
||||
self.claim_sequence = claim_sequence
|
||||
self.channel_claim_id = channel_claim_id
|
||||
self.channel_name = channel_name
|
||||
|
||||
@property
|
||||
def txid(self) -> typing.Optional[str]:
|
||||
return None if not self.outpoint else self.outpoint.split(":")[0]
|
||||
|
||||
@property
|
||||
def nout(self) -> typing.Optional[int]:
|
||||
return None if not self.outpoint else int(self.outpoint.split(":")[1])
|
||||
|
||||
def as_dict(self) -> typing.Dict:
|
||||
return {
|
||||
"name": self.claim_name,
|
||||
"claim_id": self.claim_id,
|
||||
"address": self.claim_address,
|
||||
"claim_sequence": self.claim_sequence,
|
||||
"value": self.claim,
|
||||
"height": self.height,
|
||||
"amount": dewies_to_lbc(self.amount),
|
||||
"nout": self.nout,
|
||||
"txid": self.txid,
|
||||
"channel_claim_id": self.channel_claim_id,
|
||||
"channel_name": self.channel_name
|
||||
}
|
||||
|
||||
|
||||
def _get_content_claims(transaction: sqlite3.Connection, query: str,
|
||||
source_hashes: typing.List[str]) -> typing.Dict[str, StoredContentClaim]:
|
||||
claims = {}
|
||||
for claim_info in _batched_select(transaction, query, source_hashes):
|
||||
claims[claim_info[0]] = StoredContentClaim(*claim_info[1:])
|
||||
return claims
|
||||
|
||||
|
||||
def get_claims_from_stream_hashes(transaction: sqlite3.Connection,
|
||||
stream_hashes: typing.List[str]) -> typing.Dict[str, StoredContentClaim]:
|
||||
query = (
|
||||
"select content_claim.stream_hash, c.*, case when c.channel_claim_id is not null then "
|
||||
" (select claim_name from claim where claim_id==c.channel_claim_id) "
|
||||
" else null end as channel_name "
|
||||
" from content_claim "
|
||||
" inner join claim c on c.claim_outpoint=content_claim.claim_outpoint and content_claim.stream_hash in {}"
|
||||
" order by c.rowid desc"
|
||||
)
|
||||
return _get_content_claims(transaction, query, stream_hashes)
|
||||
|
||||
|
||||
def get_claims_from_torrent_info_hashes(transaction: sqlite3.Connection,
|
||||
info_hashes: typing.List[str]) -> typing.Dict[str, StoredContentClaim]:
|
||||
query = (
|
||||
"select content_claim.bt_infohash, c.*, case when c.channel_claim_id is not null then "
|
||||
" (select claim_name from claim where claim_id==c.channel_claim_id) "
|
||||
" else null end as channel_name "
|
||||
" from content_claim "
|
||||
" inner join claim c on c.claim_outpoint=content_claim.claim_outpoint and content_claim.bt_infohash in {}"
|
||||
" order by c.rowid desc"
|
||||
)
|
||||
return _get_content_claims(transaction, query, info_hashes)
|
||||
|
||||
|
||||
def _batched_select(transaction, query, parameters, batch_size=900):
|
||||
for start_index in range(0, len(parameters), batch_size):
|
||||
current_batch = parameters[start_index:start_index+batch_size]
|
||||
bind = "({})".format(','.join(['?'] * len(current_batch)))
|
||||
yield from transaction.execute(query.format(bind), current_batch)
|
||||
|
||||
|
||||
def _get_lbry_file_stream_dict(rowid, added_on, stream_hash, file_name, download_dir, data_rate, status,
|
||||
sd_hash, stream_key, stream_name, suggested_file_name, claim, saved_file,
|
||||
raw_content_fee, fully_reflected):
|
||||
return {
|
||||
"rowid": rowid,
|
||||
"added_on": added_on,
|
||||
"stream_hash": stream_hash,
|
||||
"file_name": file_name, # hex
|
||||
"download_directory": download_dir, # hex
|
||||
"blob_data_rate": data_rate,
|
||||
"status": status,
|
||||
"sd_hash": sd_hash,
|
||||
"key": stream_key,
|
||||
"stream_name": stream_name, # hex
|
||||
"suggested_file_name": suggested_file_name, # hex
|
||||
"claim": claim,
|
||||
"saved_file": bool(saved_file),
|
||||
"content_fee": None if not raw_content_fee else Transaction(
|
||||
binascii.unhexlify(raw_content_fee)
|
||||
),
|
||||
"fully_reflected": fully_reflected
|
||||
}
|
||||
|
||||
|
||||
def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
|
||||
files = []
|
||||
signed_claims = {}
|
||||
for (rowid, stream_hash, _, file_name, download_dir, data_rate, status, saved_file, raw_content_fee,
|
||||
added_on, _, sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in transaction.execute(
|
||||
"select file.rowid, file.*, stream.*, c.*, "
|
||||
" case when (SELECT 1 FROM reflected_stream r WHERE r.sd_hash=stream.sd_hash) "
|
||||
" is null then 0 else 1 end as fully_reflected "
|
||||
"from file inner join stream on file.stream_hash=stream.stream_hash "
|
||||
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
|
||||
"inner join claim c on cc.claim_outpoint=c.claim_outpoint "
|
||||
"order by c.rowid desc").fetchall():
|
||||
claim_args, fully_reflected = tuple(claim_args[:-1]), claim_args[-1]
|
||||
claim = StoredContentClaim(*claim_args)
|
||||
if claim.channel_claim_id:
|
||||
if claim.channel_claim_id not in signed_claims:
|
||||
signed_claims[claim.channel_claim_id] = []
|
||||
signed_claims[claim.channel_claim_id].append(claim)
|
||||
files.append(
|
||||
_get_lbry_file_stream_dict(
|
||||
rowid, added_on, stream_hash, file_name, download_dir, data_rate, status,
|
||||
sd_hash, stream_key, stream_name, suggested_file_name, claim, saved_file,
|
||||
raw_content_fee, fully_reflected
|
||||
)
|
||||
)
|
||||
for claim_name, claim_id in _batched_select(
|
||||
transaction, "select c.claim_name, c.claim_id from claim c where c.claim_id in {}",
|
||||
tuple(signed_claims.keys())):
|
||||
for claim in signed_claims[claim_id]:
|
||||
claim.channel_name = claim_name
|
||||
return files
|
||||
|
||||
|
||||
def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'):
|
||||
# add all blobs, except the last one, which is empty
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
((blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
|
||||
for blob in (descriptor.blobs[:-1] if len(descriptor.blobs) > 1 else descriptor.blobs) + [sd_blob])
|
||||
).fetchall()
|
||||
# associate the blobs to the stream
|
||||
transaction.execute("insert or ignore into stream values (?, ?, ?, ?, ?)",
|
||||
(descriptor.stream_hash, sd_blob.blob_hash, descriptor.key,
|
||||
binascii.hexlify(descriptor.stream_name.encode()).decode(),
|
||||
binascii.hexlify(descriptor.suggested_file_name.encode()).decode())).fetchall()
|
||||
# add the stream
|
||||
transaction.executemany(
|
||||
"insert or ignore into stream_blob values (?, ?, ?, ?)",
|
||||
((descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
|
||||
for blob in descriptor.blobs)
|
||||
).fetchall()
|
||||
# ensure should_announce is set regardless if insert was ignored
|
||||
transaction.execute(
|
||||
"update blob set should_announce=1 where blob_hash in (?, ?)",
|
||||
(sd_blob.blob_hash, descriptor.blobs[0].blob_hash,)
|
||||
).fetchall()
|
||||
|
||||
|
||||
def delete_stream(transaction: sqlite3.Connection, descriptor: 'StreamDescriptor'):
|
||||
blob_hashes = [(blob.blob_hash, ) for blob in descriptor.blobs[:-1]]
|
||||
blob_hashes.append((descriptor.sd_hash, ))
|
||||
transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,)).fetchall()
|
||||
transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash,)).fetchall()
|
||||
transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash,)).fetchall()
|
||||
transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash,)).fetchall()
|
||||
transaction.executemany("delete from blob where blob_hash=?", blob_hashes).fetchall()
|
||||
|
||||
|
||||
def delete_torrent(transaction: sqlite3.Connection, bt_infohash: str):
|
||||
transaction.execute("delete from content_claim where bt_infohash=?", (bt_infohash, )).fetchall()
|
||||
transaction.execute("delete from torrent_tracker where bt_infohash=?", (bt_infohash,)).fetchall()
|
||||
transaction.execute("delete from torrent_node where bt_infohash=?", (bt_infohash,)).fetchall()
|
||||
transaction.execute("delete from torrent_http_seed where bt_infohash=?", (bt_infohash,)).fetchall()
|
||||
transaction.execute("delete from file where bt_infohash=?", (bt_infohash,)).fetchall()
|
||||
transaction.execute("delete from torrent where bt_infohash=?", (bt_infohash,)).fetchall()
|
||||
|
||||
|
||||
def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: typing.Optional[str],
|
||||
download_directory: typing.Optional[str], data_payment_rate: float, status: str,
|
||||
content_fee: typing.Optional[Transaction], added_on: typing.Optional[int] = None) -> int:
|
||||
if not file_name and not download_directory:
|
||||
encoded_file_name, encoded_download_dir = None, None
|
||||
else:
|
||||
encoded_file_name = binascii.hexlify(file_name.encode()).decode()
|
||||
encoded_download_dir = binascii.hexlify(download_directory.encode()).decode()
|
||||
time_added = added_on or int(time.time())
|
||||
transaction.execute(
|
||||
"insert or replace into file values (?, NULL, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(stream_hash, encoded_file_name, encoded_download_dir, data_payment_rate, status,
|
||||
1 if (file_name and download_directory and os.path.isfile(os.path.join(download_directory, file_name))) else 0,
|
||||
None if not content_fee else binascii.hexlify(content_fee.raw).decode(), time_added)
|
||||
).fetchall()
|
||||
|
||||
return transaction.execute("select rowid from file where stream_hash=?", (stream_hash, )).fetchone()[0]
|
||||
|
||||
|
||||
class SQLiteStorage(SQLiteMixin):
|
||||
CREATE_TABLES_QUERY = """
|
||||
pragma foreign_keys=on;
|
||||
pragma journal_mode=WAL;
|
||||
|
||||
create table if not exists blob (
|
||||
blob_hash char(96) primary key not null,
|
||||
blob_length integer not null,
|
||||
next_announce_time integer not null,
|
||||
should_announce integer not null default 0,
|
||||
status text not null,
|
||||
last_announced_time integer,
|
||||
single_announce integer
|
||||
);
|
||||
|
||||
create table if not exists stream (
|
||||
stream_hash char(96) not null primary key,
|
||||
sd_hash char(96) not null references blob,
|
||||
stream_key text not null,
|
||||
stream_name text not null,
|
||||
suggested_filename text not null
|
||||
);
|
||||
|
||||
create table if not exists stream_blob (
|
||||
stream_hash char(96) not null references stream,
|
||||
blob_hash char(96) references blob,
|
||||
position integer not null,
|
||||
iv char(32) not null,
|
||||
primary key (stream_hash, blob_hash)
|
||||
);
|
||||
|
||||
create table if not exists claim (
|
||||
claim_outpoint text not null primary key,
|
||||
claim_id char(40) not null,
|
||||
claim_name text not null,
|
||||
amount integer not null,
|
||||
height integer not null,
|
||||
serialized_metadata blob not null,
|
||||
channel_claim_id text,
|
||||
address text not null,
|
||||
claim_sequence integer not null
|
||||
);
|
||||
|
||||
create table if not exists torrent (
|
||||
bt_infohash char(20) not null primary key,
|
||||
tracker text,
|
||||
length integer not null,
|
||||
name text not null
|
||||
);
|
||||
|
||||
create table if not exists torrent_node ( -- BEP-0005
|
||||
bt_infohash char(20) not null references torrent,
|
||||
host text not null,
|
||||
port integer not null
|
||||
);
|
||||
|
||||
create table if not exists torrent_tracker ( -- BEP-0012
|
||||
bt_infohash char(20) not null references torrent,
|
||||
tracker text not null
|
||||
);
|
||||
|
||||
create table if not exists torrent_http_seed ( -- BEP-0017
|
||||
bt_infohash char(20) not null references torrent,
|
||||
http_seed text not null
|
||||
);
|
||||
|
||||
create table if not exists file (
|
||||
stream_hash char(96) references stream,
|
||||
bt_infohash char(20) references torrent,
|
||||
file_name text,
|
||||
download_directory text,
|
||||
blob_data_rate real not null,
|
||||
status text not null,
|
||||
saved_file integer not null,
|
||||
content_fee text,
|
||||
added_on integer not null
|
||||
);
|
||||
|
||||
create table if not exists content_claim (
|
||||
stream_hash char(96) references stream,
|
||||
bt_infohash char(20) references torrent,
|
||||
claim_outpoint text unique not null references claim
|
||||
);
|
||||
|
||||
create table if not exists support (
|
||||
support_outpoint text not null primary key,
|
||||
claim_id text not null,
|
||||
amount integer not null,
|
||||
address text not null
|
||||
);
|
||||
|
||||
create table if not exists reflected_stream (
|
||||
sd_hash text not null,
|
||||
reflector_address text not null,
|
||||
timestamp integer,
|
||||
primary key (sd_hash, reflector_address)
|
||||
);
|
||||
|
||||
create table if not exists peer (
|
||||
node_id char(96) not null primary key,
|
||||
address text not null,
|
||||
udp_port integer not null,
|
||||
tcp_port integer,
|
||||
unique (address, udp_port)
|
||||
);
|
||||
"""
|
||||
|
||||
def __init__(self, conf: Config, path, loop=None, time_getter: typing.Optional[typing.Callable[[], float]] = None):
|
||||
super().__init__(path)
|
||||
self.conf = conf
|
||||
self.content_claim_callbacks = {}
|
||||
self.loop = loop or asyncio.get_event_loop()
|
||||
self.time_getter = time_getter or time.time
|
||||
|
||||
async def run_and_return_one_or_none(self, query, *args):
|
||||
for row in await self.db.execute_fetchall(query, args):
|
||||
if len(row) == 1:
|
||||
return row[0]
|
||||
return row
|
||||
|
||||
async def run_and_return_list(self, query, *args):
|
||||
rows = list(await self.db.execute_fetchall(query, args))
|
||||
return [col[0] for col in rows] if rows else []
|
||||
|
||||
# # # # # # # # # blob functions # # # # # # # # #
|
||||
|
||||
async def add_blobs(self, *blob_hashes_and_lengths: typing.Tuple[str, int], finished=False):
|
||||
def _add_blobs(transaction: sqlite3.Connection):
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
(blob_hash, length, 0, 0, "pending" if not finished else "finished", 0, 0)
|
||||
for blob_hash, length in blob_hashes_and_lengths
|
||||
)
|
||||
).fetchall()
|
||||
if finished:
|
||||
transaction.executemany(
|
||||
"update blob set status='finished' where blob.blob_hash=?", (
|
||||
(blob_hash, ) for blob_hash, _ in blob_hashes_and_lengths
|
||||
)
|
||||
).fetchall()
|
||||
return await self.db.run(_add_blobs)
|
||||
|
||||
def get_blob_status(self, blob_hash: str):
|
||||
return self.run_and_return_one_or_none(
|
||||
"select status from blob where blob_hash=?", blob_hash
|
||||
)
|
||||
|
||||
def update_last_announced_blobs(self, blob_hashes: typing.List[str]):
|
||||
def _update_last_announced_blobs(transaction: sqlite3.Connection):
|
||||
last_announced = self.time_getter()
|
||||
return transaction.executemany(
|
||||
"update blob set next_announce_time=?, last_announced_time=?, single_announce=0 "
|
||||
"where blob_hash=?",
|
||||
((int(last_announced + (DATA_EXPIRATION / 2)), int(last_announced), blob_hash)
|
||||
for blob_hash in blob_hashes)
|
||||
).fetchall()
|
||||
return self.db.run(_update_last_announced_blobs)
|
||||
|
||||
def should_single_announce_blobs(self, blob_hashes, immediate=False):
|
||||
def set_single_announce(transaction):
|
||||
now = int(self.time_getter())
|
||||
for blob_hash in blob_hashes:
|
||||
if immediate:
|
||||
transaction.execute(
|
||||
"update blob set single_announce=1, next_announce_time=? "
|
||||
"where blob_hash=? and status='finished'", (int(now), blob_hash)
|
||||
).fetchall()
|
||||
else:
|
||||
transaction.execute(
|
||||
"update blob set single_announce=1 where blob_hash=? and status='finished'", (blob_hash,)
|
||||
).fetchall()
|
||||
return self.db.run(set_single_announce)
|
||||
|
||||
def get_blobs_to_announce(self):
|
||||
def get_and_update(transaction):
|
||||
timestamp = int(self.time_getter())
|
||||
if self.conf.announce_head_and_sd_only:
|
||||
r = transaction.execute(
|
||||
"select blob_hash from blob "
|
||||
"where blob_hash is not null and "
|
||||
"(should_announce=1 or single_announce=1) and next_announce_time<? and status='finished' "
|
||||
"order by next_announce_time asc limit ?",
|
||||
(timestamp, int(self.conf.concurrent_blob_announcers * 10))
|
||||
).fetchall()
|
||||
else:
|
||||
r = transaction.execute(
|
||||
"select blob_hash from blob where blob_hash is not null "
|
||||
"and next_announce_time<? and status='finished' "
|
||||
"order by next_announce_time asc limit ?",
|
||||
(timestamp, int(self.conf.concurrent_blob_announcers * 10))
|
||||
).fetchall()
|
||||
return [b[0] for b in r]
|
||||
return self.db.run(get_and_update)
|
||||
|
||||
def delete_blobs_from_db(self, blob_hashes):
|
||||
def delete_blobs(transaction):
|
||||
transaction.executemany(
|
||||
"delete from blob where blob_hash=?;", ((blob_hash,) for blob_hash in blob_hashes)
|
||||
).fetchall()
|
||||
return self.db.run_with_foreign_keys_disabled(delete_blobs)
|
||||
|
||||
def get_all_blob_hashes(self):
|
||||
return self.run_and_return_list("select blob_hash from blob")
|
||||
|
||||
def sync_missing_blobs(self, blob_files: typing.Set[str]) -> typing.Awaitable[typing.Set[str]]:
|
||||
def _sync_blobs(transaction: sqlite3.Connection) -> typing.Set[str]:
|
||||
finished_blob_hashes = tuple(
|
||||
blob_hash for (blob_hash, ) in transaction.execute(
|
||||
"select blob_hash from blob where status='finished'"
|
||||
).fetchall()
|
||||
)
|
||||
finished_blobs_set = set(finished_blob_hashes)
|
||||
to_update_set = finished_blobs_set.difference(blob_files)
|
||||
transaction.executemany(
|
||||
"update blob set status='pending' where blob_hash=?",
|
||||
((blob_hash, ) for blob_hash in to_update_set)
|
||||
).fetchall()
|
||||
return blob_files.intersection(finished_blobs_set)
|
||||
return self.db.run(_sync_blobs)
|
||||
|
||||
# # # # # # # # # stream functions # # # # # # # # #
|
||||
|
||||
async def stream_exists(self, sd_hash: str) -> bool:
|
||||
streams = await self.run_and_return_one_or_none("select stream_hash from stream where sd_hash=?", sd_hash)
|
||||
return streams is not None
|
||||
|
||||
async def file_exists(self, sd_hash: str) -> bool:
|
||||
streams = await self.run_and_return_one_or_none("select f.stream_hash from file f "
|
||||
"inner join stream s on "
|
||||
"s.stream_hash=f.stream_hash and s.sd_hash=?", sd_hash)
|
||||
return streams is not None
|
||||
|
||||
def store_stream(self, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'):
|
||||
return self.db.run(store_stream, sd_blob, descriptor)
|
||||
|
||||
def get_blobs_for_stream(self, stream_hash, only_completed=False) -> typing.Awaitable[typing.List[BlobInfo]]:
|
||||
def _get_blobs_for_stream(transaction):
|
||||
crypt_blob_infos = []
|
||||
stream_blobs = transaction.execute(
|
||||
"select blob_hash, position, iv from stream_blob where stream_hash=? "
|
||||
"order by position asc", (stream_hash, )
|
||||
).fetchall()
|
||||
if only_completed:
|
||||
lengths = transaction.execute(
|
||||
"select b.blob_hash, b.blob_length from blob b "
|
||||
"inner join stream_blob s ON b.blob_hash=s.blob_hash and b.status='finished' and s.stream_hash=?",
|
||||
(stream_hash, )
|
||||
).fetchall()
|
||||
else:
|
||||
lengths = transaction.execute(
|
||||
"select b.blob_hash, b.blob_length from blob b "
|
||||
"inner join stream_blob s ON b.blob_hash=s.blob_hash and s.stream_hash=?",
|
||||
(stream_hash, )
|
||||
).fetchall()
|
||||
|
||||
blob_length_dict = {}
|
||||
for blob_hash, length in lengths:
|
||||
blob_length_dict[blob_hash] = length
|
||||
|
||||
for blob_hash, position, iv in stream_blobs:
|
||||
blob_length = blob_length_dict.get(blob_hash, 0)
|
||||
crypt_blob_infos.append(BlobInfo(position, blob_length, iv, blob_hash))
|
||||
if not blob_hash:
|
||||
break
|
||||
return crypt_blob_infos
|
||||
return self.db.run(_get_blobs_for_stream)
|
||||
|
||||
def get_sd_blob_hash_for_stream(self, stream_hash):
|
||||
return self.run_and_return_one_or_none(
|
||||
"select sd_hash from stream where stream_hash=?", stream_hash
|
||||
)
|
||||
|
||||
def get_stream_hash_for_sd_hash(self, sd_blob_hash):
|
||||
return self.run_and_return_one_or_none(
|
||||
"select stream_hash from stream where sd_hash = ?", sd_blob_hash
|
||||
)
|
||||
|
||||
def delete_stream(self, descriptor: 'StreamDescriptor'):
|
||||
return self.db.run_with_foreign_keys_disabled(delete_stream, descriptor)
|
||||
|
||||
async def delete_torrent(self, bt_infohash: str):
|
||||
return await self.db.run(delete_torrent, bt_infohash)
|
||||
|
||||
# # # # # # # # # file stuff # # # # # # # # #
|
||||
|
||||
def save_downloaded_file(self, stream_hash: str, file_name: typing.Optional[str],
|
||||
download_directory: typing.Optional[str], data_payment_rate: float,
|
||||
content_fee: typing.Optional[Transaction] = None,
|
||||
added_on: typing.Optional[int] = None) -> typing.Awaitable[int]:
|
||||
return self.save_published_file(
|
||||
stream_hash, file_name, download_directory, data_payment_rate, status="running",
|
||||
content_fee=content_fee, added_on=added_on
|
||||
)
|
||||
|
||||
def save_published_file(self, stream_hash: str, file_name: typing.Optional[str],
|
||||
download_directory: typing.Optional[str], data_payment_rate: float,
|
||||
status: str = "finished",
|
||||
content_fee: typing.Optional[Transaction] = None,
|
||||
added_on: typing.Optional[int] = None) -> typing.Awaitable[int]:
|
||||
return self.db.run(store_file, stream_hash, file_name, download_directory, data_payment_rate, status,
|
||||
content_fee, added_on)
|
||||
|
||||
async def update_manually_removed_files_since_last_run(self):
|
||||
"""
|
||||
Update files that have been removed from the downloads directory since the last run
|
||||
"""
|
||||
def update_manually_removed_files(transaction: sqlite3.Connection):
|
||||
files = {}
|
||||
query = "select stream_hash, download_directory, file_name from file where saved_file=1 " \
|
||||
"and stream_hash is not null"
|
||||
for (stream_hash, download_directory, file_name) in transaction.execute(query).fetchall():
|
||||
if download_directory and file_name:
|
||||
files[stream_hash] = download_directory, file_name
|
||||
return files
|
||||
|
||||
def detect_removed(files):
|
||||
return [
|
||||
stream_hash for stream_hash, (download_directory, file_name) in files.items()
|
||||
if not os.path.isfile(os.path.join(binascii.unhexlify(download_directory).decode(),
|
||||
binascii.unhexlify(file_name).decode()))
|
||||
]
|
||||
|
||||
def update_db_removed(transaction: sqlite3.Connection, removed):
|
||||
query = "update file set file_name=null, download_directory=null, saved_file=0 where stream_hash in {}"
|
||||
for cur in _batched_select(transaction, query, removed):
|
||||
cur.fetchall()
|
||||
|
||||
stream_and_file = await self.db.run(update_manually_removed_files)
|
||||
removed = await self.loop.run_in_executor(None, detect_removed, stream_and_file)
|
||||
if removed:
|
||||
await self.db.run(update_db_removed, removed)
|
||||
|
||||
def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]:
|
||||
return self.db.run(get_all_lbry_files)
|
||||
|
||||
def change_file_status(self, stream_hash: str, new_status: str):
|
||||
log.debug("update file status %s -> %s", stream_hash, new_status)
|
||||
return self.db.execute_fetchall("update file set status=? where stream_hash=?", (new_status, stream_hash))
|
||||
|
||||
async def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: typing.Optional[str],
|
||||
file_name: typing.Optional[str]):
|
||||
if not file_name or not download_dir:
|
||||
encoded_file_name, encoded_download_dir = None, None
|
||||
else:
|
||||
encoded_file_name = binascii.hexlify(file_name.encode()).decode()
|
||||
encoded_download_dir = binascii.hexlify(download_dir.encode()).decode()
|
||||
return await self.db.execute_fetchall("update file set download_directory=?, file_name=? where stream_hash=?", (
|
||||
encoded_download_dir, encoded_file_name, stream_hash,
|
||||
))
|
||||
|
||||
async def save_content_fee(self, stream_hash: str, content_fee: Transaction):
|
||||
return await self.db.execute_fetchall("update file set content_fee=? where stream_hash=?", (
|
||||
binascii.hexlify(content_fee.raw), stream_hash,
|
||||
))
|
||||
|
||||
async def set_saved_file(self, stream_hash: str):
|
||||
return await self.db.execute_fetchall("update file set saved_file=1 where stream_hash=?", (
|
||||
stream_hash,
|
||||
))
|
||||
|
||||
async def clear_saved_file(self, stream_hash: str):
|
||||
return await self.db.execute_fetchall("update file set saved_file=0 where stream_hash=?", (
|
||||
stream_hash,
|
||||
))
|
||||
|
||||
async def recover_streams(self, descriptors_and_sds: typing.List[typing.Tuple['StreamDescriptor', 'BlobFile',
|
||||
typing.Optional[Transaction]]],
|
||||
download_directory: str):
|
||||
def _recover(transaction: sqlite3.Connection):
|
||||
stream_hashes = [x[0].stream_hash for x in descriptors_and_sds]
|
||||
for descriptor, sd_blob, content_fee in descriptors_and_sds:
|
||||
content_claim = transaction.execute(
|
||||
"select * from content_claim where stream_hash=?", (descriptor.stream_hash, )
|
||||
).fetchone()
|
||||
delete_stream(transaction, descriptor) # this will also delete the content claim
|
||||
store_stream(transaction, sd_blob, descriptor)
|
||||
store_file(transaction, descriptor.stream_hash, os.path.basename(descriptor.suggested_file_name),
|
||||
download_directory, 0.0, 'stopped', content_fee=content_fee)
|
||||
if content_claim:
|
||||
transaction.execute("insert or ignore into content_claim values (?, ?, ?)", content_claim)
|
||||
transaction.executemany(
|
||||
"update file set status='stopped' where stream_hash=?",
|
||||
((stream_hash, ) for stream_hash in stream_hashes)
|
||||
).fetchall()
|
||||
download_dir = binascii.hexlify(self.conf.download_dir.encode()).decode()
|
||||
transaction.executemany(
|
||||
f"update file set download_directory=? where stream_hash=?",
|
||||
((download_dir, stream_hash) for stream_hash in stream_hashes)
|
||||
).fetchall()
|
||||
await self.db.run_with_foreign_keys_disabled(_recover)
|
||||
|
||||
def get_all_stream_hashes(self):
|
||||
return self.run_and_return_list("select stream_hash from stream")
|
||||
|
||||
# # # # # # # # # support functions # # # # # # # # #
|
||||
|
||||
def save_supports(self, claim_id_to_supports: dict):
|
||||
# TODO: add 'address' to support items returned for a claim from lbrycrdd and lbryum-server
|
||||
def _save_support(transaction):
|
||||
bind = "({})".format(','.join(['?'] * len(claim_id_to_supports)))
|
||||
transaction.execute(
|
||||
f"delete from support where claim_id in {bind}", tuple(claim_id_to_supports.keys())
|
||||
).fetchall()
|
||||
for claim_id, supports in claim_id_to_supports.items():
|
||||
for support in supports:
|
||||
transaction.execute(
|
||||
"insert into support values (?, ?, ?, ?)",
|
||||
("%s:%i" % (support['txid'], support['nout']), claim_id, lbc_to_dewies(support['amount']),
|
||||
support.get('address', ""))
|
||||
).fetchall()
|
||||
return self.db.run(_save_support)
|
||||
|
||||
def get_supports(self, *claim_ids):
|
||||
def _format_support(outpoint, supported_id, amount, address):
|
||||
return {
|
||||
"txid": outpoint.split(":")[0],
|
||||
"nout": int(outpoint.split(":")[1]),
|
||||
"claim_id": supported_id,
|
||||
"amount": dewies_to_lbc(amount),
|
||||
"address": address,
|
||||
}
|
||||
|
||||
def _get_supports(transaction):
|
||||
return [
|
||||
_format_support(*support_info)
|
||||
for support_info in _batched_select(
|
||||
transaction,
|
||||
"select * from support where claim_id in {}",
|
||||
claim_ids
|
||||
)
|
||||
]
|
||||
|
||||
return self.db.run(_get_supports)
|
||||
|
||||
# # # # # # # # # claim functions # # # # # # # # #
|
||||
|
||||
async def save_claims(self, claim_infos):
|
||||
claim_id_to_supports = {}
|
||||
update_file_callbacks = []
|
||||
|
||||
def _save_claims(transaction):
|
||||
content_claims_to_update = []
|
||||
for claim_info in claim_infos:
|
||||
outpoint = "%s:%i" % (claim_info['txid'], claim_info['nout'])
|
||||
claim_id = claim_info['claim_id']
|
||||
name = claim_info['name']
|
||||
amount = lbc_to_dewies(claim_info['amount'])
|
||||
height = claim_info['height']
|
||||
address = claim_info['address']
|
||||
sequence = claim_info['claim_sequence']
|
||||
certificate_id = claim_info['value'].signing_channel_id
|
||||
try:
|
||||
source_hash = claim_info['value'].stream.source.sd_hash
|
||||
except (AttributeError, ValueError):
|
||||
source_hash = None
|
||||
serialized = binascii.hexlify(claim_info['value'].to_bytes())
|
||||
transaction.execute(
|
||||
"insert or replace into claim values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(outpoint, claim_id, name, amount, height, serialized, certificate_id, address, sequence)
|
||||
).fetchall()
|
||||
# if this response doesn't have support info don't overwrite the existing
|
||||
# support info
|
||||
if 'supports' in claim_info:
|
||||
claim_id_to_supports[claim_id] = claim_info['supports']
|
||||
if not source_hash:
|
||||
continue
|
||||
stream_hash = transaction.execute(
|
||||
"select file.stream_hash from stream "
|
||||
"inner join file on file.stream_hash=stream.stream_hash where sd_hash=?", (source_hash,)
|
||||
).fetchone()
|
||||
if not stream_hash:
|
||||
continue
|
||||
stream_hash = stream_hash[0]
|
||||
known_outpoint = transaction.execute(
|
||||
"select claim_outpoint from content_claim where stream_hash=?", (stream_hash,)
|
||||
).fetchone()
|
||||
known_claim_id = transaction.execute(
|
||||
"select claim_id from claim "
|
||||
"inner join content_claim c3 ON claim.claim_outpoint=c3.claim_outpoint "
|
||||
"where c3.stream_hash=?", (stream_hash,)
|
||||
).fetchone()
|
||||
if not known_claim_id:
|
||||
content_claims_to_update.append((stream_hash, outpoint))
|
||||
elif known_outpoint != outpoint:
|
||||
content_claims_to_update.append((stream_hash, outpoint))
|
||||
for stream_hash, outpoint in content_claims_to_update:
|
||||
self._save_content_claim(transaction, outpoint, stream_hash)
|
||||
if stream_hash in self.content_claim_callbacks:
|
||||
update_file_callbacks.append(self.content_claim_callbacks[stream_hash]())
|
||||
|
||||
await self.db.run(_save_claims)
|
||||
if update_file_callbacks:
|
||||
await asyncio.wait(update_file_callbacks)
|
||||
if claim_id_to_supports:
|
||||
await self.save_supports(claim_id_to_supports)
|
||||
|
||||
def save_claim_from_output(self, ledger, *outputs: Output):
|
||||
return self.save_claims([{
|
||||
"claim_id": output.claim_id,
|
||||
"name": output.claim_name,
|
||||
"amount": dewies_to_lbc(output.amount),
|
||||
"address": output.get_address(ledger),
|
||||
"txid": output.tx_ref.id,
|
||||
"nout": output.position,
|
||||
"value": output.claim,
|
||||
"height": output.tx_ref.height,
|
||||
"claim_sequence": -1,
|
||||
} for output in outputs])
|
||||
|
||||
def save_claims_for_resolve(self, claim_infos):
|
||||
to_save = {}
|
||||
for info in claim_infos:
|
||||
if 'value' in info:
|
||||
if info['value']:
|
||||
to_save[info['claim_id']] = info
|
||||
else:
|
||||
for key in ('certificate', 'claim'):
|
||||
if info.get(key, {}).get('value'):
|
||||
to_save[info[key]['claim_id']] = info[key]
|
||||
return self.save_claims(to_save.values())
|
||||
|
||||
@staticmethod
|
||||
def _save_content_claim(transaction, claim_outpoint, stream_hash=None, bt_infohash=None):
|
||||
assert stream_hash or bt_infohash
|
||||
# get the claim id and serialized metadata
|
||||
claim_info = transaction.execute(
|
||||
"select claim_id, serialized_metadata from claim where claim_outpoint=?", (claim_outpoint,)
|
||||
).fetchone()
|
||||
if not claim_info:
|
||||
raise Exception("claim not found")
|
||||
new_claim_id, claim = claim_info[0], Claim.from_bytes(binascii.unhexlify(claim_info[1]))
|
||||
|
||||
# certificate claims should not be in the content_claim table
|
||||
if not claim.is_stream:
|
||||
raise Exception("claim does not contain a stream")
|
||||
|
||||
# get the known sd hash for this stream
|
||||
known_sd_hash = transaction.execute(
|
||||
"select sd_hash from stream where stream_hash=?", (stream_hash,)
|
||||
).fetchone()
|
||||
if not known_sd_hash:
|
||||
raise Exception("stream not found")
|
||||
# check the claim contains the same sd hash
|
||||
if known_sd_hash[0] != claim.stream.source.sd_hash:
|
||||
raise Exception("stream mismatch")
|
||||
|
||||
# if there is a current claim associated to the file, check that the new claim is an update to it
|
||||
current_associated_content = transaction.execute(
|
||||
"select claim_outpoint from content_claim where stream_hash=?", (stream_hash,)
|
||||
).fetchone()
|
||||
if current_associated_content:
|
||||
current_associated_claim_id = transaction.execute(
|
||||
"select claim_id from claim where claim_outpoint=?", current_associated_content
|
||||
).fetchone()[0]
|
||||
if current_associated_claim_id != new_claim_id:
|
||||
raise Exception(
|
||||
f"mismatching claim ids when updating stream {current_associated_claim_id} vs {new_claim_id}"
|
||||
)
|
||||
|
||||
# update the claim associated to the file
|
||||
transaction.execute("delete from content_claim where stream_hash=?", (stream_hash, )).fetchall()
|
||||
transaction.execute(
|
||||
"insert into content_claim values (?, NULL, ?)", (stream_hash, claim_outpoint)
|
||||
).fetchall()
|
||||
|
||||
async def save_content_claim(self, stream_hash, claim_outpoint):
|
||||
await self.db.run(self._save_content_claim, claim_outpoint, stream_hash)
|
||||
# update corresponding ManagedEncryptedFileDownloader object
|
||||
if stream_hash in self.content_claim_callbacks:
|
||||
await self.content_claim_callbacks[stream_hash]()
|
||||
|
||||
async def save_torrent_content_claim(self, bt_infohash, claim_outpoint, length, name):
|
||||
def _save_torrent(transaction):
|
||||
transaction.execute(
|
||||
"insert or replace into torrent values (?, NULL, ?, ?)", (bt_infohash, length, name)
|
||||
).fetchall()
|
||||
transaction.execute(
|
||||
"insert or replace into content_claim values (NULL, ?, ?)", (bt_infohash, claim_outpoint)
|
||||
).fetchall()
|
||||
await self.db.run(_save_torrent)
|
||||
# update corresponding ManagedEncryptedFileDownloader object
|
||||
if bt_infohash in self.content_claim_callbacks:
|
||||
await self.content_claim_callbacks[bt_infohash]()
|
||||
|
||||
async def get_content_claim(self, stream_hash: str, include_supports: typing.Optional[bool] = True) -> typing.Dict:
|
||||
claims = await self.db.run(get_claims_from_stream_hashes, [stream_hash])
|
||||
claim = None
|
||||
if claims:
|
||||
claim = claims[stream_hash].as_dict()
|
||||
if include_supports:
|
||||
supports = await self.get_supports(claim['claim_id'])
|
||||
claim['supports'] = supports
|
||||
claim['effective_amount'] = calculate_effective_amount(claim['amount'], supports)
|
||||
return claim
|
||||
|
||||
async def get_content_claim_for_torrent(self, bt_infohash):
|
||||
claims = await self.db.run(get_claims_from_torrent_info_hashes, [bt_infohash])
|
||||
return claims[bt_infohash].as_dict() if claims else None
|
||||
|
||||
# # # # # # # # # reflector functions # # # # # # # # #
|
||||
|
||||
def update_reflected_stream(self, sd_hash, reflector_address, success=True):
|
||||
if success:
|
||||
return self.db.execute_fetchall(
|
||||
"insert or replace into reflected_stream values (?, ?, ?)",
|
||||
(sd_hash, reflector_address, self.time_getter())
|
||||
)
|
||||
return self.db.execute_fetchall(
|
||||
"delete from reflected_stream where sd_hash=? and reflector_address=?",
|
||||
(sd_hash, reflector_address)
|
||||
)
|
||||
|
||||
def get_streams_to_re_reflect(self):
|
||||
return self.run_and_return_list(
|
||||
"select s.sd_hash from stream s "
|
||||
"left outer join reflected_stream r on s.sd_hash=r.sd_hash "
|
||||
"where r.timestamp is null or r.timestamp < ?",
|
||||
int(self.time_getter()) - 86400
|
||||
)
|
||||
|
||||
# # # # # # # # # # dht functions # # # # # # # # # # #
|
||||
async def get_persisted_kademlia_peers(self) -> typing.List[typing.Tuple[bytes, str, int, int]]:
|
||||
query = 'select node_id, address, udp_port, tcp_port from peer'
|
||||
return [(binascii.unhexlify(n), a, u, t) for n, a, u, t in await self.db.execute_fetchall(query)]
|
||||
|
||||
async def save_kademlia_peers(self, peers: typing.List['KademliaPeer']):
|
||||
def _save_kademlia_peers(transaction: sqlite3.Connection):
|
||||
transaction.execute('delete from peer').fetchall()
|
||||
transaction.executemany(
|
||||
'insert into peer(node_id, address, udp_port, tcp_port) values (?, ?, ?, ?)',
|
||||
tuple([(binascii.hexlify(p.node_id), p.address, p.udp_port, p.tcp_port) for p in peers])
|
||||
).fetchall()
|
||||
return await self.db.run(_save_kademlia_peers)
|
||||
@@ -0,0 +1,62 @@
|
||||
# Copyright 2016-2017 Ionuț Arțăriși <ionut@artarisi.eu>
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
# This came from https://github.com/mapleoin/undecorated
|
||||
|
||||
from inspect import isfunction, ismethod, isclass
|
||||
|
||||
__version__ = '0.3.0'
|
||||
|
||||
|
||||
def undecorated(o):
|
||||
"""Remove all decorators from a function, method or class"""
|
||||
# class decorator
|
||||
if isinstance(o, type):
|
||||
return o
|
||||
|
||||
try:
|
||||
# python2
|
||||
closure = o.func_closure
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
try:
|
||||
# python3
|
||||
closure = o.__closure__
|
||||
except AttributeError:
|
||||
return
|
||||
|
||||
if closure:
|
||||
for cell in closure:
|
||||
# avoid infinite recursion
|
||||
if cell.cell_contents is o:
|
||||
continue
|
||||
|
||||
# check if the contents looks like a decorator; in that case
|
||||
# we need to go one level down into the dream, otherwise it
|
||||
# might just be a different closed-over variable, which we
|
||||
# can ignore.
|
||||
|
||||
# Note: this favors supporting decorators defined without
|
||||
# @wraps to the detriment of function/method/class closures
|
||||
if looks_like_a_decorator(cell.cell_contents):
|
||||
undecd = undecorated(cell.cell_contents)
|
||||
if undecd:
|
||||
return undecd
|
||||
return o
|
||||
|
||||
|
||||
def looks_like_a_decorator(a):
|
||||
return isfunction(a) or ismethod(a) or isclass(a)
|
||||
@@ -0,0 +1,29 @@
|
||||
import platform
|
||||
import os
|
||||
import logging.handlers
|
||||
|
||||
from lbry import build_info, __version__ as lbrynet_version
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_platform() -> dict:
|
||||
os_system = platform.system()
|
||||
if os.environ and 'ANDROID_ARGUMENT' in os.environ:
|
||||
os_system = 'android'
|
||||
d = {
|
||||
"processor": platform.processor(),
|
||||
"python_version": platform.python_version(),
|
||||
"platform": platform.platform(),
|
||||
"os_release": platform.release(),
|
||||
"os_system": os_system,
|
||||
"lbrynet_version": lbrynet_version,
|
||||
"version": lbrynet_version,
|
||||
"build": build_info.BUILD, # CI server sets this during build step
|
||||
}
|
||||
if d["os_system"] == "Linux":
|
||||
import distro # pylint: disable=import-outside-toplevel
|
||||
d["distro"] = distro.info()
|
||||
d["desktop"] = os.environ.get('XDG_CURRENT_DESKTOP', 'Unknown')
|
||||
|
||||
return d
|
||||
File renamed without changes.
@@ -0,0 +1,302 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import typing
|
||||
from typing import Optional
|
||||
from aiohttp.web import Request
|
||||
from lbry.error import ResolveError, DownloadSDTimeoutError, InsufficientFundsError
|
||||
from lbry.error import ResolveTimeoutError, DownloadDataTimeoutError, KeyFeeAboveMaxAllowedError
|
||||
from lbry.stream.managed_stream import ManagedStream
|
||||
from lbry.torrent.torrent_manager import TorrentSource
|
||||
from lbry.utils import cache_concurrent
|
||||
from lbry.schema.url import URL
|
||||
from lbry.wallet.dewies import dewies_to_lbc
|
||||
from lbry.file.source_manager import SourceManager
|
||||
from lbry.file.source import ManagedDownloadSource
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from lbry.wallet import WalletManager, Output
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FileManager:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', wallet_manager: 'WalletManager',
|
||||
storage: 'SQLiteStorage', analytics_manager: Optional['AnalyticsManager'] = None):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
self.wallet_manager = wallet_manager
|
||||
self.storage = storage
|
||||
self.analytics_manager = analytics_manager
|
||||
self.source_managers: typing.Dict[str, SourceManager] = {}
|
||||
self.started = asyncio.Event()
|
||||
|
||||
@property
|
||||
def streams(self):
|
||||
return self.source_managers['stream']._sources
|
||||
|
||||
async def create_stream(self, file_path: str, key: Optional[bytes] = None, **kwargs) -> ManagedDownloadSource:
|
||||
if 'stream' in self.source_managers:
|
||||
return await self.source_managers['stream'].create(file_path, key, **kwargs)
|
||||
raise NotImplementedError
|
||||
|
||||
async def start(self):
|
||||
await asyncio.gather(*(source_manager.start() for source_manager in self.source_managers.values()))
|
||||
for manager in self.source_managers.values():
|
||||
await manager.started.wait()
|
||||
self.started.set()
|
||||
|
||||
def stop(self):
|
||||
for manager in self.source_managers.values():
|
||||
# fixme: pop or not?
|
||||
manager.stop()
|
||||
self.started.clear()
|
||||
|
||||
@cache_concurrent
|
||||
async def download_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
|
||||
timeout: Optional[float] = None, file_name: Optional[str] = None,
|
||||
download_directory: Optional[str] = None,
|
||||
save_file: Optional[bool] = None, resolve_timeout: float = 3.0,
|
||||
wallet: Optional['Wallet'] = None) -> ManagedDownloadSource:
|
||||
|
||||
wallet = wallet or self.wallet_manager.default_wallet
|
||||
timeout = timeout or self.config.download_timeout
|
||||
start_time = self.loop.time()
|
||||
resolved_time = None
|
||||
stream = None
|
||||
claim = None
|
||||
error = None
|
||||
outpoint = None
|
||||
if save_file is None:
|
||||
save_file = self.config.save_files
|
||||
if file_name and not save_file:
|
||||
save_file = True
|
||||
if save_file:
|
||||
download_directory = download_directory or self.config.download_dir
|
||||
else:
|
||||
download_directory = None
|
||||
|
||||
payment = None
|
||||
try:
|
||||
# resolve the claim
|
||||
if not URL.parse(uri).has_stream:
|
||||
raise ResolveError("cannot download a channel claim, specify a /path")
|
||||
try:
|
||||
resolved_result = await asyncio.wait_for(
|
||||
self.wallet_manager.ledger.resolve(
|
||||
wallet.accounts, [uri],
|
||||
include_purchase_receipt=True,
|
||||
include_is_my_output=True
|
||||
), resolve_timeout
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
raise ResolveTimeoutError(uri)
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError):
|
||||
raise
|
||||
log.exception("Unexpected error resolving stream:")
|
||||
raise ResolveError(f"Unexpected error resolving stream: {str(err)}")
|
||||
if 'error' in resolved_result:
|
||||
raise ResolveError(f"Unexpected error resolving uri for download: {resolved_result['error']}")
|
||||
if not resolved_result or uri not in resolved_result:
|
||||
raise ResolveError(f"Failed to resolve stream at '{uri}'")
|
||||
txo = resolved_result[uri]
|
||||
if isinstance(txo, dict):
|
||||
raise ResolveError(f"Failed to resolve stream at '{uri}': {txo}")
|
||||
claim = txo.claim
|
||||
outpoint = f"{txo.tx_ref.id}:{txo.position}"
|
||||
resolved_time = self.loop.time() - start_time
|
||||
await self.storage.save_claim_from_output(self.wallet_manager.ledger, txo)
|
||||
|
||||
####################
|
||||
# update or replace
|
||||
####################
|
||||
|
||||
if claim.stream.source.bt_infohash:
|
||||
source_manager = self.source_managers['torrent']
|
||||
existing = source_manager.get_filtered(bt_infohash=claim.stream.source.bt_infohash)
|
||||
elif claim.stream.source.sd_hash:
|
||||
source_manager = self.source_managers['stream']
|
||||
existing = source_manager.get_filtered(sd_hash=claim.stream.source.sd_hash)
|
||||
else:
|
||||
raise ResolveError(f"There is nothing to download at {uri} - Source is unknown or unset")
|
||||
|
||||
# resume or update an existing stream, if the stream changed: download it and delete the old one after
|
||||
to_replace, updated_stream = None, None
|
||||
if existing and existing[0].claim_id != txo.claim_id:
|
||||
raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {txo.claim_id}")
|
||||
if existing:
|
||||
log.info("claim contains a metadata only update to a stream we have")
|
||||
if claim.stream.source.bt_infohash:
|
||||
await self.storage.save_torrent_content_claim(
|
||||
existing[0].identifier, outpoint, existing[0].torrent_length, existing[0].torrent_name
|
||||
)
|
||||
claim_info = await self.storage.get_content_claim_for_torrent(existing[0].identifier)
|
||||
existing[0].set_claim(claim_info, claim)
|
||||
else:
|
||||
await self.storage.save_content_claim(
|
||||
existing[0].stream_hash, outpoint
|
||||
)
|
||||
await source_manager._update_content_claim(existing[0])
|
||||
updated_stream = existing[0]
|
||||
else:
|
||||
existing_for_claim_id = self.get_filtered(claim_id=txo.claim_id)
|
||||
if existing_for_claim_id:
|
||||
log.info("claim contains an update to a stream we have, downloading it")
|
||||
if save_file and existing_for_claim_id[0].output_file_exists:
|
||||
save_file = False
|
||||
if not claim.stream.source.bt_infohash:
|
||||
existing_for_claim_id[0].downloader.node = source_manager.node
|
||||
await existing_for_claim_id[0].start(timeout=timeout, save_now=save_file)
|
||||
if not existing_for_claim_id[0].output_file_exists and (
|
||||
save_file or file_name or download_directory):
|
||||
await existing_for_claim_id[0].save_file(
|
||||
file_name=file_name, download_directory=download_directory
|
||||
)
|
||||
to_replace = existing_for_claim_id[0]
|
||||
|
||||
# resume or update an existing stream, if the stream changed: download it and delete the old one after
|
||||
if updated_stream:
|
||||
log.info("already have stream for %s", uri)
|
||||
if save_file and updated_stream.output_file_exists:
|
||||
save_file = False
|
||||
if not claim.stream.source.bt_infohash:
|
||||
updated_stream.downloader.node = source_manager.node
|
||||
await updated_stream.start(timeout=timeout, save_now=save_file)
|
||||
if not updated_stream.output_file_exists and (save_file or file_name or download_directory):
|
||||
await updated_stream.save_file(
|
||||
file_name=file_name, download_directory=download_directory
|
||||
)
|
||||
return updated_stream
|
||||
|
||||
####################
|
||||
# pay fee
|
||||
####################
|
||||
|
||||
needs_purchasing = (
|
||||
not to_replace and
|
||||
not txo.is_my_output and
|
||||
txo.has_price and
|
||||
not txo.purchase_receipt
|
||||
)
|
||||
|
||||
if needs_purchasing:
|
||||
payment = await self.wallet_manager.create_purchase_transaction(
|
||||
wallet.accounts, txo, exchange_rate_manager
|
||||
)
|
||||
|
||||
####################
|
||||
# make downloader and wait for start
|
||||
####################
|
||||
|
||||
if not claim.stream.source.bt_infohash:
|
||||
# fixme: this shouldnt be here
|
||||
stream = ManagedStream(
|
||||
self.loop, self.config, source_manager.blob_manager, claim.stream.source.sd_hash,
|
||||
download_directory, file_name, ManagedStream.STATUS_RUNNING, content_fee=payment,
|
||||
analytics_manager=self.analytics_manager
|
||||
)
|
||||
stream.downloader.node = source_manager.node
|
||||
else:
|
||||
stream = TorrentSource(
|
||||
self.loop, self.config, self.storage, identifier=claim.stream.source.bt_infohash,
|
||||
file_name=file_name, download_directory=download_directory or self.config.download_dir,
|
||||
status=ManagedStream.STATUS_RUNNING,
|
||||
analytics_manager=self.analytics_manager,
|
||||
torrent_session=source_manager.torrent_session
|
||||
)
|
||||
log.info("starting download for %s", uri)
|
||||
|
||||
before_download = self.loop.time()
|
||||
await stream.start(timeout, save_file)
|
||||
|
||||
####################
|
||||
# success case: delete to_replace if applicable, broadcast fee payment
|
||||
####################
|
||||
|
||||
if to_replace: # delete old stream now that the replacement has started downloading
|
||||
await source_manager.delete(to_replace)
|
||||
|
||||
if payment is not None:
|
||||
await self.wallet_manager.broadcast_or_release(payment)
|
||||
payment = None # to avoid releasing in `finally` later
|
||||
log.info("paid fee of %s for %s", dewies_to_lbc(stream.content_fee.outputs[0].amount), uri)
|
||||
await self.storage.save_content_fee(stream.stream_hash, stream.content_fee)
|
||||
|
||||
source_manager.add(stream)
|
||||
|
||||
if not claim.stream.source.bt_infohash:
|
||||
await self.storage.save_content_claim(stream.stream_hash, outpoint)
|
||||
else:
|
||||
await self.storage.save_torrent_content_claim(
|
||||
stream.identifier, outpoint, stream.torrent_length, stream.torrent_name
|
||||
)
|
||||
claim_info = await self.storage.get_content_claim_for_torrent(stream.identifier)
|
||||
stream.set_claim(claim_info, claim)
|
||||
if save_file:
|
||||
await asyncio.wait_for(stream.save_file(), timeout - (self.loop.time() - before_download),
|
||||
loop=self.loop)
|
||||
return stream
|
||||
except asyncio.TimeoutError:
|
||||
error = DownloadDataTimeoutError(stream.sd_hash)
|
||||
raise error
|
||||
except Exception as err: # forgive data timeout, don't delete stream
|
||||
expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError,
|
||||
KeyFeeAboveMaxAllowedError)
|
||||
if isinstance(err, expected):
|
||||
log.warning("Failed to download %s: %s", uri, str(err))
|
||||
elif isinstance(err, asyncio.CancelledError):
|
||||
pass
|
||||
else:
|
||||
log.exception("Unexpected error downloading stream:")
|
||||
error = err
|
||||
raise
|
||||
finally:
|
||||
if payment is not None:
|
||||
# payment is set to None after broadcasting, if we're here an exception probably happened
|
||||
await self.wallet_manager.ledger.release_tx(payment)
|
||||
if self.analytics_manager and claim and claim.stream.source.bt_infohash:
|
||||
# TODO: analytics for torrents
|
||||
pass
|
||||
elif self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or
|
||||
stream.downloader.time_to_first_bytes))):
|
||||
server = self.wallet_manager.ledger.network.client.server
|
||||
self.loop.create_task(
|
||||
self.analytics_manager.send_time_to_first_bytes(
|
||||
resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id,
|
||||
uri, outpoint,
|
||||
None if not stream else len(stream.downloader.blob_downloader.active_connections),
|
||||
None if not stream else len(stream.downloader.blob_downloader.scores),
|
||||
None if not stream else len(stream.downloader.blob_downloader.connection_failures),
|
||||
False if not stream else stream.downloader.added_fixed_peers,
|
||||
self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay,
|
||||
None if not stream else stream.sd_hash,
|
||||
None if not stream else stream.downloader.time_to_descriptor,
|
||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
|
||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
|
||||
None if not stream else stream.downloader.time_to_first_bytes,
|
||||
None if not error else error.__class__.__name__,
|
||||
None if not error else str(error),
|
||||
None if not server else f"{server[0]}:{server[1]}"
|
||||
)
|
||||
)
|
||||
|
||||
async def stream_partial_content(self, request: Request, sd_hash: str):
|
||||
return await self.source_managers['stream'].stream_partial_content(request, sd_hash)
|
||||
|
||||
def get_filtered(self, *args, **kwargs) -> typing.List[ManagedDownloadSource]:
|
||||
"""
|
||||
Get a list of filtered and sorted ManagedStream objects
|
||||
|
||||
:param sort_by: field to sort by
|
||||
:param reverse: reverse sorting
|
||||
:param comparison: comparison operator used for filtering
|
||||
:param search_by: fields and values to filter by
|
||||
"""
|
||||
return sum((manager.get_filtered(*args, **kwargs) for manager in self.source_managers.values()), [])
|
||||
|
||||
async def delete(self, source: ManagedDownloadSource, delete_file=False):
|
||||
for manager in self.source_managers.values():
|
||||
await manager.delete(source, delete_file)
|
||||
@@ -0,0 +1,161 @@
|
||||
import os
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
import binascii
|
||||
from typing import Optional
|
||||
from lbry.utils import generate_id
|
||||
from lbry.extras.daemon.storage import StoredContentClaim
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.wallet.transaction import Transaction
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ManagedDownloadSource:
|
||||
STATUS_RUNNING = "running"
|
||||
STATUS_STOPPED = "stopped"
|
||||
STATUS_FINISHED = "finished"
|
||||
|
||||
SAVING_ID = 1
|
||||
STREAMING_ID = 2
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage', identifier: str,
|
||||
file_name: Optional[str] = None, download_directory: Optional[str] = None,
|
||||
status: Optional[str] = STATUS_STOPPED, claim: Optional[StoredContentClaim] = None,
|
||||
download_id: Optional[str] = None, rowid: Optional[int] = None,
|
||||
content_fee: Optional['Transaction'] = None,
|
||||
analytics_manager: Optional['AnalyticsManager'] = None,
|
||||
added_on: Optional[int] = None):
|
||||
self.loop = loop
|
||||
self.storage = storage
|
||||
self.config = config
|
||||
self.identifier = identifier
|
||||
self.download_directory = download_directory
|
||||
self._file_name = file_name
|
||||
self._status = status
|
||||
self.stream_claim_info = claim
|
||||
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
|
||||
self.rowid = rowid
|
||||
self.content_fee = content_fee
|
||||
self.purchase_receipt = None
|
||||
self._added_on = added_on
|
||||
self.analytics_manager = analytics_manager
|
||||
|
||||
self.saving = asyncio.Event(loop=self.loop)
|
||||
self.finished_writing = asyncio.Event(loop=self.loop)
|
||||
self.started_writing = asyncio.Event(loop=self.loop)
|
||||
self.finished_write_attempt = asyncio.Event(loop=self.loop)
|
||||
|
||||
# @classmethod
|
||||
# async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', file_path: str,
|
||||
# key: Optional[bytes] = None,
|
||||
# iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedDownloadSource':
|
||||
# raise NotImplementedError()
|
||||
|
||||
async def start(self, timeout: Optional[float] = None, save_now: Optional[bool] = False):
|
||||
raise NotImplementedError()
|
||||
|
||||
async def stop(self, finished: bool = False):
|
||||
raise NotImplementedError()
|
||||
|
||||
async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
|
||||
raise NotImplementedError()
|
||||
|
||||
def stop_tasks(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
def set_claim(self, claim_info: typing.Dict, claim: 'Claim'):
|
||||
self.stream_claim_info = StoredContentClaim(
|
||||
f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
|
||||
claim_info['name'], claim_info['amount'], claim_info['height'],
|
||||
binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'],
|
||||
claim_info['claim_sequence'], claim_info.get('channel_name')
|
||||
)
|
||||
|
||||
# async def update_content_claim(self, claim_info: Optional[typing.Dict] = None):
|
||||
# if not claim_info:
|
||||
# claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash)
|
||||
# self.set_claim(claim_info, claim_info['value'])
|
||||
|
||||
@property
|
||||
def file_name(self) -> Optional[str]:
|
||||
return self._file_name
|
||||
|
||||
@property
|
||||
def added_on(self) -> Optional[int]:
|
||||
return self._added_on
|
||||
|
||||
@property
|
||||
def status(self) -> str:
|
||||
return self._status
|
||||
|
||||
@property
|
||||
def completed(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
# @property
|
||||
# def stream_url(self):
|
||||
# return f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash}
|
||||
|
||||
@property
|
||||
def finished(self) -> bool:
|
||||
return self.status == self.STATUS_FINISHED
|
||||
|
||||
@property
|
||||
def running(self) -> bool:
|
||||
return self.status == self.STATUS_RUNNING
|
||||
|
||||
@property
|
||||
def claim_id(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim_id
|
||||
|
||||
@property
|
||||
def txid(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.txid
|
||||
|
||||
@property
|
||||
def nout(self) -> Optional[int]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.nout
|
||||
|
||||
@property
|
||||
def outpoint(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.outpoint
|
||||
|
||||
@property
|
||||
def claim_height(self) -> Optional[int]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.height
|
||||
|
||||
@property
|
||||
def channel_claim_id(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.channel_claim_id
|
||||
|
||||
@property
|
||||
def channel_name(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.channel_name
|
||||
|
||||
@property
|
||||
def claim_name(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim_name
|
||||
|
||||
@property
|
||||
def metadata(self) -> Optional[typing.Dict]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict()
|
||||
|
||||
@property
|
||||
def metadata_protobuf(self) -> bytes:
|
||||
if self.stream_claim_info:
|
||||
return binascii.hexlify(self.stream_claim_info.claim.to_bytes())
|
||||
|
||||
@property
|
||||
def full_path(self) -> Optional[str]:
|
||||
return os.path.join(self.download_directory, os.path.basename(self.file_name)) \
|
||||
if self.file_name and self.download_directory else None
|
||||
|
||||
@property
|
||||
def output_file_exists(self):
|
||||
return os.path.isfile(self.full_path) if self.full_path else False
|
||||
Loaded 100 of 321 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user