Compare commits
551
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
69e7566eaa | ||
|
|
399ac561c4 | ||
|
|
588568752b | ||
|
|
a30e2daf05 | ||
|
|
adb7edd3b2 | ||
|
|
06a9770504 | ||
|
|
0da43d3e05 | ||
|
|
077dbb3138 | ||
|
|
3b8fd65fa9 | ||
|
|
d0ddaa5518 | ||
|
|
a60ab97552 | ||
|
|
078d09976c | ||
|
|
3770558694 | ||
|
|
4409b8c4f8 | ||
|
|
f5b7aaac18 | ||
|
|
6dbb6b9079 | ||
|
|
16b774a69e | ||
|
|
72ea19532e | ||
|
|
ad24946af3 | ||
|
|
b676907fd7 | ||
|
|
6965971613 | ||
|
|
6651111522 | ||
|
|
d8630a02d9 | ||
|
|
69345b4b0e | ||
|
|
f23aea9951 | ||
|
|
f96c46e84a | ||
|
|
bae2939943 | ||
|
|
7063138e3c | ||
|
|
798ba9acf8 | ||
|
|
c8dd1987e6 | ||
|
|
8d2c9e5785 | ||
|
|
9dbf47916b | ||
|
|
565ef5e454 | ||
|
|
9f1f2b1781 | ||
|
|
76df239d1a | ||
|
|
86ee6a78f5 | ||
|
|
4fb6f2bb27 | ||
|
|
ec039825e8 | ||
|
|
a10f09d169 | ||
|
|
76409600e1 | ||
|
|
d06a0f876a | ||
|
|
8e857de62a | ||
|
|
3d37f7c735 | ||
|
|
552b94bc65 | ||
|
|
17164601a1 | ||
|
|
26dd6d2ac7 | ||
|
|
5d5e36a2c2 | ||
|
|
77afbcded6 | ||
|
|
ae11d72f66 | ||
|
|
d2773f9713 | ||
|
|
a7fba569cd | ||
|
|
0559c64038 | ||
|
|
5ab2754bc6 | ||
|
|
6bfb8384e7 | ||
|
|
0156aa8b00 | ||
|
|
ad411cc157 | ||
|
|
6d462ad6d8 | ||
|
|
1b35cef77a | ||
|
|
a357208a77 | ||
|
|
7e1f8a56d5 | ||
|
|
37665f2df7 | ||
|
|
8f3b596385 | ||
|
|
e45c6fe42c | ||
|
|
56edf62cb9 | ||
|
|
d367ff6ac1 | ||
|
|
7a78d49e13 | ||
|
|
4dae1ae05f | ||
|
|
9004f240e6 | ||
|
|
1848462ad9 | ||
|
|
786efa06dd | ||
|
|
b6811d55cf | ||
|
|
42dc0eccd6 | ||
|
|
29bd936181 | ||
|
|
9d97b90ff4 | ||
|
|
370b34f860 | ||
|
|
affa46e0f6 | ||
|
|
6b93705599 | ||
|
|
925e39c5a8 | ||
|
|
1efa2c9406 | ||
|
|
4f8bcfcd18 | ||
|
|
3ed72a215e | ||
|
|
85d94d4ca3 | ||
|
|
b922ea8e7e | ||
|
|
cb6ddaf276 | ||
|
|
1c2029cddf | ||
|
|
d8d136f1d8 | ||
|
|
fedc0e5bc3 | ||
|
|
4fbccb9336 | ||
|
|
ef5dd5593f | ||
|
|
baed974be0 | ||
|
|
b2b3509b61 | ||
|
|
51720d9b73 | ||
|
|
6abb569ec8 | ||
|
|
c2fbbadc69 | ||
|
|
b8897223ec | ||
|
|
c2c184b4ef | ||
|
|
07dbdef3fe | ||
|
|
b841fb2f0c | ||
|
|
2b75982446 | ||
|
|
021ca3efd2 | ||
|
|
e56b72cf04 | ||
|
|
342f485ddf | ||
|
|
8fe16fbfc9 | ||
|
|
1e3035029e | ||
|
|
5f4c02f836 | ||
|
|
03a1185e0f | ||
|
|
0e1705d19e | ||
|
|
82c739daa6 | ||
|
|
4702bea9f0 | ||
|
|
75211b7e84 | ||
|
|
9f9273bd82 | ||
|
|
d2608bdf43 | ||
|
|
091bef1ead | ||
|
|
998ee9feba | ||
|
|
587f0b2ea8 | ||
|
|
048aa07e80 | ||
|
|
7ca01511f4 | ||
|
|
8dee83aac0 | ||
|
|
879ae5cc72 | ||
|
|
74a9e17178 | ||
|
|
93d2173fad | ||
|
|
a459ee34be | ||
|
|
2e6297f834 | ||
|
|
9accbfcf8b | ||
|
|
6946860521 | ||
|
|
9b8be8682c | ||
|
|
03455310ae | ||
|
|
e5edaed677 | ||
|
|
5e97215eaa | ||
|
|
266f94bc95 | ||
|
|
d26caded53 | ||
|
|
2260608bb6 | ||
|
|
bb1f42057e | ||
|
|
695f6658a0 | ||
|
|
eca677a720 | ||
|
|
51f301785f | ||
|
|
41a6e288aa | ||
|
|
97d8d0ef4c | ||
|
|
5baeb5f538 | ||
|
|
cc49c8407c | ||
|
|
3f37963f7f | ||
|
|
12eb83f773 | ||
|
|
ca1d135ba4 | ||
|
|
107e57e3ff | ||
|
|
928689a10d | ||
|
|
6ecc22f2c7 | ||
|
|
079c81f298 | ||
|
|
8afe178d62 | ||
|
|
8777dcee7c | ||
|
|
1cca4a91dd | ||
|
|
41abfbdd9b | ||
|
|
46ee65c8f2 | ||
|
|
fd19daad9e | ||
|
|
26d183cbab | ||
|
|
71a4be02dd | ||
|
|
124fdb7d31 | ||
|
|
2439743804 | ||
|
|
6f79875135 | ||
|
|
b91d2190f4 | ||
|
|
d024433d1b | ||
|
|
7c4eb416d6 | ||
|
|
fb457c820a | ||
|
|
18af2dcd4e | ||
|
|
0075dcc2c0 | ||
|
|
6e327d729b | ||
|
|
20022d49c1 | ||
|
|
32d1563885 | ||
|
|
f5cef14d8d | ||
|
|
bac7d99b8a | ||
|
|
6f06026511 | ||
|
|
f02df86709 | ||
|
|
b7d76fd09f | ||
|
|
e675f1387c | ||
|
|
0da2827c78 | ||
|
|
309cfc840c | ||
|
|
68ee31b62a | ||
|
|
ad2da24d64 | ||
|
|
648dbcc36b | ||
|
|
e59e96190e | ||
|
|
7d2f592bb5 | ||
|
|
1a182bc9bd | ||
|
|
324cbdcdb2 | ||
|
|
6c0c0ee322 | ||
|
|
1e54271d8a | ||
|
|
54b9ed6cb2 | ||
|
|
558a02d187 | ||
|
|
3567b331c0 | ||
|
|
9e4cf3c995 | ||
|
|
db1d547b19 | ||
|
|
c31efd780b | ||
|
|
7cdb7987e3 | ||
|
|
ec07037e88 | ||
|
|
9bb7b0571e | ||
|
|
d98162a118 | ||
|
|
c5ae59bff7 | ||
|
|
efbf2f49a9 | ||
|
|
77c69f661d | ||
|
|
2e09d2adfa | ||
|
|
8620a0e424 | ||
|
|
e2c123456b | ||
|
|
c9be79f062 | ||
|
|
a2a1ef34ab | ||
|
|
761a2b235c | ||
|
|
3ca2fee592 | ||
|
|
a480b2d25f | ||
|
|
d7032b12d7 | ||
|
|
84381ff76c | ||
|
|
007dd43861 | ||
|
|
1bc4e4b702 | ||
|
|
b91f27219d | ||
|
|
dc4560cc9a | ||
|
|
84b471d486 | ||
|
|
c6a3c05a0c | ||
|
|
dd665a758d | ||
|
|
32dc1297ec | ||
|
|
bef244ba77 | ||
|
|
3c231d6215 | ||
|
|
a212cf6ba4 | ||
|
|
bc060f1378 | ||
|
|
22c701fd50 | ||
|
|
4f0e8fce6e | ||
|
|
f642cfe9dd | ||
|
|
24e073680b | ||
|
|
b2f63a1545 | ||
|
|
f506b3e6d4 | ||
|
|
f7412514b3 | ||
|
|
bd6a609b30 | ||
|
|
4e32b69d1d | ||
|
|
1116c7f29e | ||
|
|
d80c671cf2 | ||
|
|
14b12cbea2 | ||
|
|
1f7feafb67 | ||
|
|
cbe689ea7f | ||
|
|
9099ee2e8e | ||
|
|
b134e0c9c9 | ||
|
|
f8c0e80cfc | ||
|
|
1f266ebdad | ||
|
|
b09e1f8825 | ||
|
|
c496278a5b | ||
|
|
cacc65d45e | ||
|
|
2247b354b3 | ||
|
|
43111f0497 | ||
|
|
6c142b4477 | ||
|
|
e8683a24bd | ||
|
|
a4d17d1705 | ||
|
|
a030e13548 | ||
|
|
32b8601c38 | ||
|
|
f5bd28bfdb | ||
|
|
6cbbe23c7c | ||
|
|
d7c2e4e9b3 | ||
|
|
d61ddbb950 | ||
|
|
cf7bb6a391 | ||
|
|
68d7328918 | ||
|
|
ff2985cc80 | ||
|
|
ac69fcbc5f | ||
|
|
609cf42868 | ||
|
|
e43b29fcd1 | ||
|
|
6594b7c50c | ||
|
|
971f2a1cef | ||
|
|
fc88261cc6 | ||
|
|
895b3992c9 | ||
|
|
41951d8ec2 | ||
|
|
ae2121c5c4 | ||
|
|
771fe935f4 | ||
|
|
60f7766cf7 | ||
|
|
63bd983012 | ||
|
|
7ffce1040c | ||
|
|
ac6835fa60 | ||
|
|
3cf5c536c0 | ||
|
|
85cc1a4763 | ||
|
|
180cfcfb72 | ||
|
|
864c1dd901 | ||
|
|
aeaa7ba986 | ||
|
|
b8325c1f89 | ||
|
|
a36936b5ff | ||
|
|
7cf245a0e9 | ||
|
|
9f83cce574 | ||
|
|
30b113d64a | ||
|
|
ccd2023d37 | ||
|
|
3127cd11cf | ||
|
|
6e483cba8d | ||
|
|
46c03c0cb2 | ||
|
|
c2ca0a9467 | ||
|
|
b350e74aca | ||
|
|
de9473f75a | ||
|
|
bfc107fdb3 | ||
|
|
6a2efa54ab | ||
|
|
72f0e1ca14 | ||
|
|
9a1262d557 | ||
|
|
706a8e52dc | ||
|
|
f3c8776878 | ||
|
|
539ea04039 | ||
|
|
8f96b07559 | ||
|
|
0a1e97fe23 | ||
|
|
da637b9c0e | ||
|
|
3618de1227 | ||
|
|
3cb8a63bf9 | ||
|
|
bad99de5b3 | ||
|
|
3d1efbc9bb | ||
|
|
7bdef3146b | ||
|
|
2145bfcabf | ||
|
|
b88832d0a9 | ||
|
|
db08882f20 | ||
|
|
6f52d36b22 | ||
|
|
056a629e87 | ||
|
|
1ffb99f9f9 | ||
|
|
a8e612773d | ||
|
|
67088b2ca2 | ||
|
|
dab49e812f | ||
|
|
0f3f38d8a3 | ||
|
|
7ba433c2e5 | ||
|
|
bb7649e8ad | ||
|
|
832537a5cf | ||
|
|
3234d70270 | ||
|
|
432fe444f8 | ||
|
|
861c25716e | ||
|
|
9ba6653475 | ||
|
|
8107e97304 | ||
|
|
e2663f6d57 | ||
|
|
35479dcbd2 | ||
|
|
3a997277aa | ||
|
|
a7d94f4514 | ||
|
|
a0e3338177 | ||
|
|
d5a353066c | ||
|
|
bc29fc4465 | ||
|
|
987f175fcc | ||
|
|
5b1398b588 | ||
|
|
873b58efc9 | ||
|
|
01a81fceed | ||
|
|
a14b0aa573 | ||
|
|
a1abb7c8fb | ||
|
|
46b166952b | ||
|
|
1375153fda | ||
|
|
1a14157d37 | ||
|
|
40b9fa38b0 | ||
|
|
4d0e9e0428 | ||
|
|
3a53ef5690 | ||
|
|
33a68b5cef | ||
|
|
63bf084a6a | ||
|
|
a882f2e8c5 | ||
|
|
119e0ec408 | ||
|
|
7d782d516a | ||
|
|
d3eac48035 | ||
|
|
60c9ae64b4 | ||
|
|
2d314dce60 | ||
|
|
287b89db66 | ||
|
|
7a160f7335 | ||
|
|
04ddee7d89 | ||
|
|
3a916a8e8e | ||
|
|
f125468ebf | ||
|
|
f0e17cff9a | ||
|
|
93267efe0b | ||
|
|
c663e5a3cf | ||
|
|
676f0015aa | ||
|
|
d44a79ada2 | ||
|
|
d7ed8ffaa6 | ||
|
|
b230d8fbcb | ||
|
|
7d2428fd17 | ||
|
|
4157045a41 | ||
|
|
58987857cf | ||
|
|
fa8858325c | ||
|
|
3357a419ab | ||
|
|
a782adea75 | ||
|
|
e8991e6d52 | ||
|
|
13bd81c2b6 | ||
|
|
21241b98a4 | ||
|
|
bf467e06c5 | ||
|
|
886ed6415a | ||
|
|
97db241109 | ||
|
|
216379918b | ||
|
|
fcab4eb9bd | ||
|
|
19f300cc45 | ||
|
|
26fec098e4 | ||
|
|
7824de655e | ||
|
|
6ef1b1dd59 | ||
|
|
9fae8f5153 | ||
|
|
f660691e78 | ||
|
|
bf60683ba6 | ||
|
|
68215699f7 | ||
|
|
1644ffbf22 | ||
|
|
71f5061848 | ||
|
|
61cadb5cbe | ||
|
|
2034104cfc | ||
|
|
633f725634 | ||
|
|
8bf2e41c95 | ||
|
|
fa9a4a75c6 | ||
|
|
90bef98bc3 | ||
|
|
35c43647aa | ||
|
|
449a583286 | ||
|
|
b1f813782a | ||
|
|
60080cbd19 | ||
|
|
665dba62c7 | ||
|
|
76dea9e89b | ||
|
|
ebc9ed01c5 | ||
|
|
55116763e9 | ||
|
|
e1a6b4e3d6 | ||
|
|
911ca8c37e | ||
|
|
7f5deaf6c8 | ||
|
|
0200756218 | ||
|
|
a63442f843 | ||
|
|
74b3da79aa | ||
|
|
b6a4589da5 | ||
|
|
fbbb0a9931 | ||
|
|
a94d65d522 | ||
|
|
fa2ead002e | ||
|
|
8dfee4867a | ||
|
|
6614ab8e79 | ||
|
|
d32c1dfbb3 | ||
|
|
9f42b958fa | ||
|
|
574a912f7d | ||
|
|
2cfcc3cd45 | ||
|
|
9af625e627 | ||
|
|
1871c94003 | ||
|
|
93220c3874 | ||
|
|
362ed3f1c8 | ||
|
|
62a182b8bb | ||
|
|
6f76d9e24a | ||
|
|
55e83fe05c | ||
|
|
2f0c7d6a3f | ||
|
|
59e1df9efa | ||
|
|
fd56dc68cd | ||
|
|
a0a810edc3 | ||
|
|
c1d3b5984d | ||
|
|
1fb719dd61 | ||
|
|
b87316fb02 | ||
|
|
65f3d78084 | ||
|
|
47bc1526f7 | ||
|
|
3acabec4bb | ||
|
|
36a62d3756 | ||
|
|
2e22473ce3 | ||
|
|
0b0fd64642 | ||
|
|
ca17d8597d | ||
|
|
d16e2716c8 | ||
|
|
89c80cedfd | ||
|
|
021e30ffde | ||
|
|
eb2cd8fec1 | ||
|
|
6356818557 | ||
|
|
8bf724a78b | ||
|
|
65d0a924c1 | ||
|
|
ef7bd00f22 | ||
|
|
3c1fdb5114 | ||
|
|
83411acfbd | ||
|
|
4230812f82 | ||
|
|
7b57ba96dd | ||
|
|
668840921f | ||
|
|
a3bd335f7f | ||
|
|
4e6bca1389 | ||
|
|
c4b15920ff | ||
|
|
2a55e583e7 | ||
|
|
2c4fde06c5 | ||
|
|
eca83d7f47 | ||
|
|
87e818ac39 | ||
|
|
adab7bd7b1 | ||
|
|
750c0e4f4f | ||
|
|
5f8482bdb7 | ||
|
|
cd266477fb | ||
|
|
7860b956ff | ||
|
|
148481a5ca | ||
|
|
93faa276a5 | ||
|
|
53e67a6e81 | ||
|
|
3ad6808d87 | ||
|
|
071ab32ab7 | ||
|
|
ab87588250 | ||
|
|
22db29ee1d | ||
|
|
66bf6c61f0 | ||
|
|
9abf40d21f | ||
|
|
ea4444edbe | ||
|
|
51bb8fd283 | ||
|
|
183e3ac1c7 | ||
|
|
8087b457c6 | ||
|
|
fedb7c23d0 | ||
|
|
e2964f9728 | ||
|
|
e0cea60c87 | ||
|
|
c708c1db07 | ||
|
|
1cb696fb5c | ||
|
|
7cd0213eb9 | ||
|
|
2007c48af3 | ||
|
|
50721786d2 | ||
|
|
6d7eb259cb | ||
|
|
cc5852a9b9 | ||
|
|
0592cee567 | ||
|
|
814eaca524 | ||
|
|
e06916f5ae | ||
|
|
d61e1e9d8c | ||
|
|
25436893bc | ||
|
|
5b9d41fd5c | ||
|
|
b1720d1659 | ||
|
|
4b3a6ab875 | ||
|
|
67dcede2d8 | ||
|
|
0872a6fea3 | ||
|
|
a3aed3219f | ||
|
|
af73e1a2f5 | ||
|
|
55bbf5aa74 | ||
|
|
c1fbb02bfe | ||
|
|
d47575e8e0 | ||
|
|
8fc4f4aa73 | ||
|
|
fe9aa666e9 | ||
|
|
a9e0eeba7e | ||
|
|
0830917afb | ||
|
|
1ed086e90f | ||
|
|
56db631626 | ||
|
|
aa029955ee | ||
|
|
f2236eb7a4 | ||
|
|
df649341cd | ||
|
|
a3c3143868 | ||
|
|
d0201fe7c7 | ||
|
|
664a2a324d | ||
|
|
ea09aca175 | ||
|
|
ca92fddecf | ||
|
|
8259d7c680 | ||
|
|
6653da2973 | ||
|
|
e74998209d | ||
|
|
2bea0f697a | ||
|
|
1d68bef6f2 | ||
|
|
22bacc907b | ||
|
|
1e8ad900c7 | ||
|
|
cd2b535afb | ||
|
|
9dfe7bbcf6 | ||
|
|
120618ffd5 | ||
|
|
e8abcbb06e | ||
|
|
1ec8f0b0b4 | ||
|
|
323694fb12 | ||
|
|
d5e37acd8f | ||
|
|
129a886f2b | ||
|
|
cd15230a92 | ||
|
|
c1b7fdc68e | ||
|
|
9fd67aaad6 | ||
|
|
ce9dc1836e | ||
|
|
2e557d3a36 | ||
|
|
5df05cb37e | ||
|
|
326b62dc57 | ||
|
|
25cc7ba284 | ||
|
|
550f064302 | ||
|
|
11086b93f1 | ||
|
|
df5662136d | ||
|
|
291d65070d | ||
|
|
9b64ee33dc | ||
|
|
5404d26f2f | ||
|
|
26ade1f228 | ||
|
|
b49f1f1b54 | ||
|
|
3bafa2ad22 | ||
|
|
378ec44f3c | ||
|
|
9d0d17b422 | ||
|
|
f7c8145283 | ||
|
|
0de17730ca | ||
|
|
58743ba19b | ||
|
|
f2fefbe287 | ||
|
|
60a24f0e6e | ||
|
|
664f91bfab | ||
|
|
44f92a271f | ||
|
|
6565ca8558 |
No files matched your search
@@ -7,7 +7,7 @@ Before reporting any issues, please make sure that you're using the latest versi
|
||||
- App: https://github.com/lbryio/lbry-desktop/releases
|
||||
- Daemon: https://github.com/lbryio/lbry/releases
|
||||
|
||||
We are also available on Discord at https://chat.lbry.io
|
||||
We are also available on Discord at https://chat.lbry.com
|
||||
-->
|
||||
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
.DS_Store
|
||||
/build
|
||||
/dist
|
||||
/.tox
|
||||
@@ -8,3 +9,5 @@
|
||||
lbrynet.egg-info
|
||||
__pycache__
|
||||
_trial_temp/
|
||||
|
||||
/tests/integration/files
|
||||
+25
-8
@@ -3,6 +3,11 @@ dist: xenial
|
||||
language: python
|
||||
python: "3.7"
|
||||
|
||||
env:
|
||||
global:
|
||||
# must also be updated in wine_build.sh
|
||||
- TORBA=master
|
||||
|
||||
jobs:
|
||||
include:
|
||||
|
||||
@@ -10,7 +15,7 @@ jobs:
|
||||
name: "pylint lbrynet"
|
||||
install:
|
||||
- pip install astroid pylint
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
|
||||
- pip install -e .
|
||||
script: pylint lbrynet
|
||||
|
||||
@@ -18,7 +23,7 @@ jobs:
|
||||
name: "Unit Tests"
|
||||
install:
|
||||
- pip install coverage
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
|
||||
- pip install -e .
|
||||
script:
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m unittest discover -vv tests.unit
|
||||
@@ -29,12 +34,24 @@ jobs:
|
||||
- name: "Integration Tests"
|
||||
install:
|
||||
- pip install tox-travis coverage
|
||||
- pushd .. && git clone https://github.com/lbryio/torba.git && popd
|
||||
- pushd .. && git clone --single-branch --branch ${TORBA} https://github.com/lbryio/torba.git && popd
|
||||
|
||||
script: tox
|
||||
after_success:
|
||||
- coverage combine tests/
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- name: "Run Examples"
|
||||
install:
|
||||
- pip install coverage
|
||||
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
|
||||
- pip install -e .
|
||||
script:
|
||||
- HOME=/tmp coverage run -p --source=lbrynet scripts/generate_json_api.py
|
||||
after_success:
|
||||
- coverage combine
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- stage: build
|
||||
name: "Windows"
|
||||
language: generic
|
||||
@@ -68,8 +85,8 @@ jobs:
|
||||
env: OS=linux
|
||||
install:
|
||||
- pip3 install pyinstaller
|
||||
- pip3 install git+https://github.com/lbryio/torba.git
|
||||
- python scripts/set_build.py
|
||||
- pip3 install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
|
||||
- python3 scripts/set_build.py
|
||||
- pip3 install -e .
|
||||
script:
|
||||
- pyinstaller -F -n lbrynet lbrynet/extras/cli.py
|
||||
@@ -110,15 +127,15 @@ jobs:
|
||||
script:
|
||||
- set -e
|
||||
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
|
||||
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f scripts/Dockerfile.wallet_server .
|
||||
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f scripts/Dockerfile.wallet_server --build-arg TORBA_VERSION=${TORBA} .
|
||||
- docker push lbry/wallet-server:$TRAVIS_TAG
|
||||
- if: tag IS blank AND branch = master
|
||||
- if: tag IS blank AND branch = master AND NOT type IN (pull_request)
|
||||
stage: build
|
||||
name: "Wallet Server Docker Image - Master"
|
||||
script:
|
||||
- set -e
|
||||
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
|
||||
- travis_retry docker build -t lbry/wallet-server:master -f scripts/Dockerfile.wallet_server .
|
||||
- travis_retry docker build -t lbry/wallet-server:master -f scripts/Dockerfile.wallet_server --build-arg TORBA_VERSION=${TORBA} .
|
||||
- docker push lbry/wallet-server:master
|
||||
|
||||
cache:
|
||||
|
||||
+1
-1
@@ -1,3 +1,3 @@
|
||||
## Contributing to LBRY
|
||||
|
||||
https://lbry.io/faq/contributing
|
||||
https://lbry.tech/contribute
|
||||
+16
-13
@@ -9,35 +9,32 @@ Here's a video walkthrough of this setup, which is itself hosted by the LBRY net
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Running `lbrynet` from source requires Python 3.6 or higher (3.7 is preferred). Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/)
|
||||
Running `lbrynet` from source requires Python 3.7 or higher. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/)
|
||||
|
||||
After installing python 3, you'll need to install some additional libraries depending on your operating system.
|
||||
|
||||
### Virtualenv
|
||||
|
||||
Once python 3 is installed run `python3 -m pip install virtualenv` to install virtualenv.
|
||||
|
||||
### Windows
|
||||
|
||||
Windows users will need to install `Visual C++ Build Tools`, which can be installed by [Microsoft Build Tools](Microsoft Build Tools 2015)
|
||||
|
||||
|
||||
### macOS
|
||||
|
||||
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
|
||||
|
||||
These environment variables also need to be set
|
||||
1. PYTHONUNBUFFERED=1
|
||||
2. EVENT_NOKQUEUE=1
|
||||
|
||||
Remaining dependencies can then be installed by running:
|
||||
|
||||
```
|
||||
brew install python3 protobuf
|
||||
brew install python protobuf
|
||||
```
|
||||
|
||||
Assistance installing Python3: https://docs.python-guide.org/starting/install3/osx/
|
||||
|
||||
### Linux
|
||||
|
||||
On Ubuntu (we recommend 18.04), install the following:
|
||||
|
||||
```
|
||||
sudo apt-get install build-essential python3.7 python3.7-dev git python-virtualenv libssl-dev python-protobuf
|
||||
sudo apt-get install build-essential python3.7 python3.7-dev git python3-venv libssl-dev python-protobuf
|
||||
```
|
||||
|
||||
On Raspbian, you will also need to install `python-pyparsing`.
|
||||
@@ -52,8 +49,14 @@ To install:
|
||||
git clone https://github.com/lbryio/lbry.git
|
||||
cd lbry
|
||||
|
||||
virtualenv lbry-venv --python=python3.7
|
||||
Creating venv:
|
||||
python -m venv lbry-venv
|
||||
|
||||
Activating lbry-venv on Linux/Mac:
|
||||
source lbry-venv/bin/activate
|
||||
|
||||
Activating lbry-venv on Windows:
|
||||
lbry-venv\Scripts\activate
|
||||
|
||||
python --version # Python 2 is not supported. Make sure you're on Python 3.7
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ LBRY SDK for Python is currently the most full featured implementation of the LB
|
||||
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
|
||||
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
|
||||
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbrynet.schema](https://github.com/lbryio/lbry/tree/master/lbrynet/schema)).
|
||||
* Wallet implementation for the LBRY blockchain ([lbrynet.extras.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/wallet)).
|
||||
* Wallet implementation for the LBRY blockchain ([lbrynet.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/wallet)).
|
||||
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbrynet.extras.daemon](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/daemon)).
|
||||
|
||||
## Installation
|
||||
@@ -33,7 +33,7 @@ Installing from source is also relatively painless. Full instructions are in [IN
|
||||
|
||||
## Contributing
|
||||
|
||||
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.io/faq/contributing) link.
|
||||
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.tech/contribute) link.
|
||||
|
||||
## License
|
||||
|
||||
@@ -41,11 +41,11 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
|
||||
|
||||
## Security
|
||||
|
||||
We take security seriously. Please contact security@lbry.io regarding any security issues. [Our GPG key is here](https://lbry.io/faq/gpg-key) if you need it.
|
||||
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.com/faq/gpg-key) if you need it.
|
||||
|
||||
## Contact
|
||||
|
||||
The primary contact for this project is [@eukreign](mailto:lex@lbry.io).
|
||||
The primary contact for this project is [@eukreign](mailto:lex@lbry.com).
|
||||
|
||||
## Additional information and links
|
||||
|
||||
|
||||
+2784
-1332
File diff suppressed because it is too large.
Load diff
@@ -4,20 +4,22 @@
|
||||
share_usage_data: True
|
||||
|
||||
lbryum_servers:
|
||||
- lbryumx1.lbry.io:50001
|
||||
- lbryumx2.lbry.io:50001
|
||||
- lbryumx1.lbry.com:50001
|
||||
- lbryumx2.lbry.com:50001
|
||||
- lbryumx4.lbry.com:50001
|
||||
|
||||
blockchain_name: lbrycrd_main
|
||||
|
||||
data_dir: /home/lbry/.lbrynet
|
||||
download_directory: /home/lbry/downloads
|
||||
|
||||
delete_blobs_on_remove: True
|
||||
save_blobs: true
|
||||
save_files: false
|
||||
dht_node_port: 4444
|
||||
peer_port: 3333
|
||||
use_upnp: True
|
||||
use_upnp: true
|
||||
|
||||
#components_to_skip:
|
||||
# - peer_protocol_server
|
||||
# - hash_announcer
|
||||
# - blob_server
|
||||
# - dht
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
__name__ = "lbrynet"
|
||||
__version__ = "0.34.0rc2"
|
||||
__version__ = "0.38.0rc6"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
+270
-132
@@ -4,6 +4,8 @@ import asyncio
|
||||
import binascii
|
||||
import logging
|
||||
import typing
|
||||
import contextlib
|
||||
from io import BytesIO
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, modes
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from cryptography.hazmat.primitives.padding import PKCS7
|
||||
@@ -21,10 +23,6 @@ log = logging.getLogger(__name__)
|
||||
_hexmatch = re.compile("^[a-f,0-9]+$")
|
||||
|
||||
|
||||
def is_valid_hashcharacter(char: str) -> bool:
|
||||
return len(char) == 1 and _hexmatch.match(char)
|
||||
|
||||
|
||||
def is_valid_blobhash(blobhash: str) -> bool:
|
||||
"""Checks whether the blobhash is the correct length and contains only
|
||||
valid characters (0-9, a-f)
|
||||
@@ -46,143 +44,74 @@ def encrypt_blob_bytes(key: bytes, iv: bytes, unencrypted: bytes) -> typing.Tupl
|
||||
return encrypted, digest.hexdigest()
|
||||
|
||||
|
||||
class BlobFile:
|
||||
"""
|
||||
A chunk of data available on the network which is specified by a hashsum
|
||||
def decrypt_blob_bytes(data: bytes, length: int, key: bytes, iv: bytes) -> bytes:
|
||||
if len(data) != length:
|
||||
raise ValueError("unexpected length")
|
||||
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
|
||||
unpadder = PKCS7(AES.block_size).unpadder()
|
||||
decryptor = cipher.decryptor()
|
||||
return unpadder.update(decryptor.update(data) + decryptor.finalize()) + unpadder.finalize()
|
||||
|
||||
This class is used to create blobs on the local filesystem
|
||||
when we already know the blob hash before hand (i.e., when downloading blobs)
|
||||
Also can be used for reading from blobs on the local filesystem
|
||||
"""
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, blob_hash: str,
|
||||
length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['BlobFile'], typing.Awaitable]] = None):
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise InvalidBlobHashError(blob_hash)
|
||||
class AbstractBlob:
|
||||
"""
|
||||
A chunk of data (up to 2MB) available on the network which is specified by a sha384 hash
|
||||
|
||||
This class is non-io specific
|
||||
"""
|
||||
__slots__ = [
|
||||
'loop',
|
||||
'blob_hash',
|
||||
'length',
|
||||
'blob_completed_callback',
|
||||
'blob_directory',
|
||||
'writers',
|
||||
'verified',
|
||||
'writing',
|
||||
'readers'
|
||||
]
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None):
|
||||
self.loop = loop
|
||||
self.blob_hash = blob_hash
|
||||
self.length = length
|
||||
self.blob_dir = blob_dir
|
||||
self.file_path = os.path.join(blob_dir, self.blob_hash)
|
||||
self.writers: typing.List[HashBlobWriter] = []
|
||||
|
||||
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||
self.finished_writing = asyncio.Event(loop=loop)
|
||||
self.blob_write_lock = asyncio.Lock(loop=loop)
|
||||
if self.file_exists:
|
||||
length = int(os.stat(os.path.join(blob_dir, blob_hash)).st_size)
|
||||
self.length = length
|
||||
self.verified.set()
|
||||
self.finished_writing.set()
|
||||
self.saved_verified_blob = False
|
||||
self.blob_completed_callback = blob_completed_callback
|
||||
self.blob_directory = blob_directory
|
||||
self.writers: typing.Dict[typing.Tuple[typing.Optional[str], typing.Optional[int]], HashBlobWriter] = {}
|
||||
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||
self.writing: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||
self.readers: typing.List[typing.BinaryIO] = []
|
||||
|
||||
@property
|
||||
def file_exists(self):
|
||||
return os.path.isfile(self.file_path)
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise InvalidBlobHashError(blob_hash)
|
||||
|
||||
def writer_finished(self, writer: HashBlobWriter):
|
||||
def callback(finished: asyncio.Future):
|
||||
def __del__(self):
|
||||
if self.writers or self.readers:
|
||||
log.warning("%s not closed before being garbage collected", self.blob_hash)
|
||||
self.close()
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
raise NotImplementedError()
|
||||
|
||||
@contextlib.contextmanager
|
||||
def reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
if not self.is_readable():
|
||||
raise OSError(f"{str(type(self))} not readable, {len(self.readers)} readers {len(self.writers)} writers")
|
||||
with self._reader_context() as reader:
|
||||
try:
|
||||
error = finished.exception()
|
||||
except Exception as err:
|
||||
error = err
|
||||
if writer in self.writers: # remove this download attempt
|
||||
self.writers.remove(writer)
|
||||
if not error: # the blob downloaded, cancel all the other download attempts and set the result
|
||||
while self.writers:
|
||||
other = self.writers.pop()
|
||||
other.finished.cancel()
|
||||
t = self.loop.create_task(self.save_verified_blob(writer, finished.result()))
|
||||
t.add_done_callback(lambda *_: self.finished_writing.set())
|
||||
return
|
||||
if isinstance(error, (InvalidBlobHashError, InvalidDataError)):
|
||||
log.debug("writer error downloading %s: %s", self.blob_hash[:8], str(error))
|
||||
elif not isinstance(error, (DownloadCancelledError, asyncio.CancelledError, asyncio.TimeoutError)):
|
||||
log.exception("something else")
|
||||
raise error
|
||||
return callback
|
||||
self.readers.append(reader)
|
||||
yield reader
|
||||
finally:
|
||||
if reader in self.readers:
|
||||
self.readers.remove(reader)
|
||||
|
||||
async def save_verified_blob(self, writer, verified_bytes: bytes):
|
||||
def _save_verified():
|
||||
# log.debug(f"write blob file {self.blob_hash[:8]} from {writer.peer.address}")
|
||||
if not self.saved_verified_blob and not os.path.isfile(self.file_path):
|
||||
if self.get_length() == len(verified_bytes):
|
||||
with open(self.file_path, 'wb') as write_handle:
|
||||
write_handle.write(verified_bytes)
|
||||
self.saved_verified_blob = True
|
||||
else:
|
||||
raise Exception("length mismatch")
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
raise NotImplementedError()
|
||||
|
||||
async with self.blob_write_lock:
|
||||
if self.verified.is_set():
|
||||
return
|
||||
await self.loop.run_in_executor(None, _save_verified)
|
||||
if self.blob_completed_callback:
|
||||
await self.blob_completed_callback(self)
|
||||
self.verified.set()
|
||||
|
||||
def open_for_writing(self) -> HashBlobWriter:
|
||||
if self.file_exists:
|
||||
raise OSError(f"File already exists '{self.file_path}'")
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
|
||||
self.writers.append(writer)
|
||||
fut.add_done_callback(self.writer_finished(writer))
|
||||
return writer
|
||||
|
||||
async def sendfile(self, writer: asyncio.StreamWriter) -> int:
|
||||
"""
|
||||
Read and send the file to the writer and return the number of bytes sent
|
||||
"""
|
||||
|
||||
with open(self.file_path, 'rb') as handle:
|
||||
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
|
||||
|
||||
def close(self):
|
||||
while self.writers:
|
||||
self.writers.pop().finished.cancel()
|
||||
|
||||
def delete(self):
|
||||
self.close()
|
||||
self.saved_verified_blob = False
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
self.verified.clear()
|
||||
self.finished_writing.clear()
|
||||
self.length = None
|
||||
|
||||
def decrypt(self, key: bytes, iv: bytes) -> bytes:
|
||||
"""
|
||||
Decrypt a BlobFile to plaintext bytes
|
||||
"""
|
||||
|
||||
with open(self.file_path, "rb") as f:
|
||||
buff = f.read()
|
||||
if len(buff) != self.length:
|
||||
raise ValueError("unexpected length")
|
||||
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
|
||||
unpadder = PKCS7(AES.block_size).unpadder()
|
||||
decryptor = cipher.decryptor()
|
||||
return unpadder.update(decryptor.update(buff) + decryptor.finalize()) + unpadder.finalize()
|
||||
|
||||
@classmethod
|
||||
async def create_from_unencrypted(cls, loop: asyncio.BaseEventLoop, blob_dir: str, key: bytes,
|
||||
iv: bytes, unencrypted: bytes, blob_num: int) -> BlobInfo:
|
||||
"""
|
||||
Create an encrypted BlobFile from plaintext bytes
|
||||
"""
|
||||
|
||||
blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted)
|
||||
length = len(blob_bytes)
|
||||
blob = cls(loop, blob_dir, blob_hash, length)
|
||||
writer = blob.open_for_writing()
|
||||
writer.write(blob_bytes)
|
||||
await blob.verified.wait()
|
||||
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash)
|
||||
|
||||
def set_length(self, length):
|
||||
def set_length(self, length) -> None:
|
||||
if self.length is not None and length == self.length:
|
||||
return
|
||||
if self.length is None and 0 <= length <= MAX_BLOB_SIZE:
|
||||
@@ -190,8 +119,217 @@ class BlobFile:
|
||||
return
|
||||
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s", self.length, length)
|
||||
|
||||
def get_length(self):
|
||||
def get_length(self) -> typing.Optional[int]:
|
||||
return self.length
|
||||
|
||||
def get_is_verified(self):
|
||||
def get_is_verified(self) -> bool:
|
||||
return self.verified.is_set()
|
||||
|
||||
def is_readable(self) -> bool:
|
||||
return self.verified.is_set()
|
||||
|
||||
def is_writeable(self) -> bool:
|
||||
return not self.writing.is_set()
|
||||
|
||||
def write_blob(self, blob_bytes: bytes):
|
||||
if not self.is_writeable():
|
||||
raise OSError("cannot open blob for writing")
|
||||
try:
|
||||
self.writing.set()
|
||||
self._write_blob(blob_bytes)
|
||||
finally:
|
||||
self.writing.clear()
|
||||
|
||||
def close(self):
|
||||
while self.writers:
|
||||
peer, writer = self.writers.popitem()
|
||||
if writer and writer.finished and not writer.finished.done() and not self.loop.is_closed():
|
||||
writer.finished.cancel()
|
||||
while self.readers:
|
||||
reader = self.readers.pop()
|
||||
if reader:
|
||||
reader.close()
|
||||
|
||||
def delete(self):
|
||||
self.close()
|
||||
self.verified.clear()
|
||||
self.length = None
|
||||
|
||||
async def sendfile(self, writer: asyncio.StreamWriter) -> int:
|
||||
"""
|
||||
Read and send the file to the writer and return the number of bytes sent
|
||||
"""
|
||||
|
||||
if not self.is_readable():
|
||||
raise OSError('blob files cannot be read')
|
||||
with self.reader_context() as handle:
|
||||
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
|
||||
|
||||
def decrypt(self, key: bytes, iv: bytes) -> bytes:
|
||||
"""
|
||||
Decrypt a BlobFile to plaintext bytes
|
||||
"""
|
||||
|
||||
with self.reader_context() as reader:
|
||||
return decrypt_blob_bytes(reader.read(), self.length, key, iv)
|
||||
|
||||
@classmethod
|
||||
async def create_from_unencrypted(
|
||||
cls, loop: asyncio.BaseEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
|
||||
unencrypted: bytes, blob_num: int,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None) -> BlobInfo:
|
||||
"""
|
||||
Create an encrypted BlobFile from plaintext bytes
|
||||
"""
|
||||
|
||||
blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted)
|
||||
length = len(blob_bytes)
|
||||
blob = cls(loop, blob_hash, length, blob_completed_callback, blob_dir)
|
||||
writer = blob.get_blob_writer()
|
||||
writer.write(blob_bytes)
|
||||
await blob.verified.wait()
|
||||
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash)
|
||||
|
||||
def save_verified_blob(self, verified_bytes: bytes):
|
||||
if self.verified.is_set():
|
||||
return
|
||||
if self.is_writeable():
|
||||
self._write_blob(verified_bytes)
|
||||
self.verified.set()
|
||||
if self.blob_completed_callback:
|
||||
self.blob_completed_callback(self)
|
||||
|
||||
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
|
||||
peer_port: typing.Optional[int] = None) -> HashBlobWriter:
|
||||
if (peer_address, peer_port) in self.writers and not self.writers[(peer_address, peer_port)].closed():
|
||||
raise OSError(f"attempted to download blob twice from {peer_address}:{peer_port}")
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
|
||||
self.writers[(peer_address, peer_port)] = writer
|
||||
|
||||
def remove_writer(_):
|
||||
if (peer_address, peer_port) in self.writers:
|
||||
del self.writers[(peer_address, peer_port)]
|
||||
|
||||
fut.add_done_callback(remove_writer)
|
||||
|
||||
def writer_finished_callback(finished: asyncio.Future):
|
||||
try:
|
||||
err = finished.exception()
|
||||
if err:
|
||||
raise err
|
||||
verified_bytes = finished.result()
|
||||
while self.writers:
|
||||
_, other = self.writers.popitem()
|
||||
if other is not writer:
|
||||
other.close_handle()
|
||||
self.save_verified_blob(verified_bytes)
|
||||
except (InvalidBlobHashError, InvalidDataError) as error:
|
||||
log.warning("writer error downloading %s: %s", self.blob_hash[:8], str(error))
|
||||
except (DownloadCancelledError, asyncio.CancelledError, asyncio.TimeoutError):
|
||||
pass
|
||||
|
||||
fut.add_done_callback(writer_finished_callback)
|
||||
return writer
|
||||
|
||||
|
||||
class BlobBuffer(AbstractBlob):
|
||||
"""
|
||||
An in-memory only blob
|
||||
"""
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None):
|
||||
self._verified_bytes: typing.Optional[BytesIO] = None
|
||||
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
if not self.is_readable():
|
||||
raise OSError("cannot open blob for reading")
|
||||
try:
|
||||
yield self._verified_bytes
|
||||
finally:
|
||||
if self._verified_bytes:
|
||||
self._verified_bytes.close()
|
||||
self._verified_bytes = None
|
||||
self.verified.clear()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
if self._verified_bytes:
|
||||
raise OSError("already have bytes for blob")
|
||||
self._verified_bytes = BytesIO(blob_bytes)
|
||||
|
||||
def delete(self):
|
||||
if self._verified_bytes:
|
||||
self._verified_bytes.close()
|
||||
self._verified_bytes = None
|
||||
return super().delete()
|
||||
|
||||
def __del__(self):
|
||||
super().__del__()
|
||||
if self._verified_bytes:
|
||||
self.delete()
|
||||
|
||||
|
||||
class BlobFile(AbstractBlob):
|
||||
"""
|
||||
A blob existing on the local file system
|
||||
"""
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None):
|
||||
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
|
||||
if not blob_directory or not os.path.isdir(blob_directory):
|
||||
raise OSError(f"invalid blob directory '{blob_directory}'")
|
||||
self.file_path = os.path.join(self.blob_directory, self.blob_hash)
|
||||
if self.file_exists:
|
||||
file_size = int(os.stat(self.file_path).st_size)
|
||||
if length and length != file_size:
|
||||
log.warning("expected %s to be %s bytes, file has %s", self.blob_hash, length, file_size)
|
||||
self.delete()
|
||||
else:
|
||||
self.length = file_size
|
||||
self.verified.set()
|
||||
|
||||
@property
|
||||
def file_exists(self):
|
||||
return os.path.isfile(self.file_path)
|
||||
|
||||
def is_writeable(self) -> bool:
|
||||
return super().is_writeable() and not os.path.isfile(self.file_path)
|
||||
|
||||
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
|
||||
peer_port: typing.Optional[str] = None) -> HashBlobWriter:
|
||||
if self.file_exists:
|
||||
raise OSError(f"File already exists '{self.file_path}'")
|
||||
return super().get_blob_writer(peer_address, peer_port)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
handle = open(self.file_path, 'rb')
|
||||
try:
|
||||
yield handle
|
||||
finally:
|
||||
handle.close()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
with open(self.file_path, 'wb') as f:
|
||||
f.write(blob_bytes)
|
||||
|
||||
def delete(self):
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
return super().delete()
|
||||
|
||||
@classmethod
|
||||
async def create_from_unencrypted(
|
||||
cls, loop: asyncio.BaseEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
|
||||
unencrypted: bytes, blob_num: int,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'],
|
||||
asyncio.Task]] = None) -> BlobInfo:
|
||||
if not blob_dir or not os.path.isdir(blob_dir):
|
||||
raise OSError(f"cannot create blob in directory: '{blob_dir}'")
|
||||
return await super().create_from_unencrypted(
|
||||
loop, blob_dir, key, iv, unencrypted, blob_num, blob_completed_callback
|
||||
)
|
||||
@@ -2,6 +2,13 @@ import typing
|
||||
|
||||
|
||||
class BlobInfo:
|
||||
__slots__ = [
|
||||
'blob_hash',
|
||||
'blob_num',
|
||||
'length',
|
||||
'iv',
|
||||
]
|
||||
|
||||
def __init__(self, blob_num: int, length: int, iv: str, blob_hash: typing.Optional[str] = None):
|
||||
self.blob_hash = blob_hash
|
||||
self.blob_num = blob_num
|
||||
|
||||
@@ -2,18 +2,21 @@ import os
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_file import BlobFile, is_valid_blobhash
|
||||
from lbrynet.utils import LRUCache
|
||||
from lbrynet.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.connection_manager import ConnectionManager
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.dht.protocol.data_store import DictDataStore
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobFileManager:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, storage: SQLiteStorage,
|
||||
class BlobManager:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, storage: 'SQLiteStorage', config: 'Config',
|
||||
node_data_store: typing.Optional['DictDataStore'] = None):
|
||||
"""
|
||||
This class stores blobs on the hard disk
|
||||
@@ -27,55 +30,97 @@ class BlobFileManager:
|
||||
self._node_data_store = node_data_store
|
||||
self.completed_blob_hashes: typing.Set[str] = set() if not self._node_data_store\
|
||||
else self._node_data_store.completed_blobs
|
||||
self.blobs: typing.Dict[str, BlobFile] = {}
|
||||
self.blobs: typing.Dict[str, AbstractBlob] = {}
|
||||
self.config = config
|
||||
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCache(
|
||||
self.config.blob_lru_cache_size)
|
||||
self.connection_manager = ConnectionManager(loop)
|
||||
|
||||
def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None):
|
||||
if self.config.save_blobs:
|
||||
return BlobFile(
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
||||
)
|
||||
else:
|
||||
if is_valid_blobhash(blob_hash) and os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
return BlobFile(
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
||||
)
|
||||
return BlobBuffer(
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
||||
)
|
||||
|
||||
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
|
||||
if blob_hash in self.blobs:
|
||||
if self.config.save_blobs and isinstance(self.blobs[blob_hash], BlobBuffer):
|
||||
buffer = self.blobs.pop(blob_hash)
|
||||
if blob_hash in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.remove(blob_hash)
|
||||
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
|
||||
if buffer.is_readable():
|
||||
with buffer.reader_context() as reader:
|
||||
self.blobs[blob_hash].write_blob(reader.read())
|
||||
if length and self.blobs[blob_hash].length is None:
|
||||
self.blobs[blob_hash].set_length(length)
|
||||
else:
|
||||
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
|
||||
return self.blobs[blob_hash]
|
||||
|
||||
def is_blob_verified(self, blob_hash: str, length: typing.Optional[int] = None) -> bool:
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise ValueError(blob_hash)
|
||||
if blob_hash in self.blobs:
|
||||
return self.blobs[blob_hash].get_is_verified()
|
||||
if not os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
return False
|
||||
return self._get_blob(blob_hash, length).get_is_verified()
|
||||
|
||||
async def setup(self) -> bool:
|
||||
def get_files_in_blob_dir() -> typing.Set[str]:
|
||||
if not self.blob_dir:
|
||||
return set()
|
||||
return {
|
||||
item.name for item in os.scandir(self.blob_dir) if is_valid_blobhash(item.name)
|
||||
}
|
||||
|
||||
in_blobfiles_dir = await self.loop.run_in_executor(None, get_files_in_blob_dir)
|
||||
self.completed_blob_hashes.update(await self.storage.sync_missing_blobs(in_blobfiles_dir))
|
||||
to_add = await self.storage.sync_missing_blobs(in_blobfiles_dir)
|
||||
if to_add:
|
||||
self.completed_blob_hashes.update(to_add)
|
||||
self.connection_manager.start()
|
||||
return True
|
||||
|
||||
def stop(self):
|
||||
self.connection_manager.stop()
|
||||
while self.blobs:
|
||||
_, blob = self.blobs.popitem()
|
||||
blob.close()
|
||||
self.completed_blob_hashes.clear()
|
||||
|
||||
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
|
||||
if blob_hash in self.blobs:
|
||||
if length and self.blobs[blob_hash].length is None:
|
||||
self.blobs[blob_hash].set_length(length)
|
||||
else:
|
||||
self.blobs[blob_hash] = BlobFile(self.loop, self.blob_dir, blob_hash, length, self.blob_completed)
|
||||
return self.blobs[blob_hash]
|
||||
|
||||
def get_stream_descriptor(self, sd_hash):
|
||||
return StreamDescriptor.from_stream_descriptor_blob(self.loop, self.blob_dir, self.get_blob(sd_hash))
|
||||
|
||||
async def blob_completed(self, blob: BlobFile):
|
||||
def blob_completed(self, blob: AbstractBlob) -> asyncio.Task:
|
||||
if blob.blob_hash is None:
|
||||
raise Exception("Blob hash is None")
|
||||
if not blob.length:
|
||||
raise Exception("Blob has a length of 0")
|
||||
if blob.blob_hash not in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.add(blob.blob_hash)
|
||||
await self.storage.add_completed_blob(blob.blob_hash, blob.length)
|
||||
if isinstance(blob, BlobFile):
|
||||
if blob.blob_hash not in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.add(blob.blob_hash)
|
||||
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=True))
|
||||
else:
|
||||
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=False))
|
||||
|
||||
def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]:
|
||||
"""Returns of the blobhashes_to_check, which are valid"""
|
||||
blobs = [self.get_blob(b) for b in blob_hashes]
|
||||
return [blob.blob_hash for blob in blobs if blob.get_is_verified()]
|
||||
return [blob_hash for blob_hash in blob_hashes if self.is_blob_verified(blob_hash)]
|
||||
|
||||
def delete_blob(self, blob_hash: str):
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise Exception("invalid blob hash to delete")
|
||||
|
||||
if blob_hash not in self.blobs:
|
||||
if os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
if self.blob_dir and os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
os.remove(os.path.join(self.blob_dir, blob_hash))
|
||||
else:
|
||||
self.blobs.pop(blob_hash).delete()
|
||||
|
||||
@@ -44,16 +44,15 @@ class HashBlobWriter:
|
||||
self._hashsum.update(data)
|
||||
self.len_so_far += len(data)
|
||||
if self.len_so_far > expected_length:
|
||||
self.close_handle()
|
||||
self.finished.set_exception(InvalidDataError(
|
||||
f'Length so far is greater than the expected length. {self.len_so_far} to {expected_length}'
|
||||
))
|
||||
self.close_handle()
|
||||
return
|
||||
self.buffer.write(data)
|
||||
if self.len_so_far == expected_length:
|
||||
blob_hash = self.calculate_blob_hash()
|
||||
if blob_hash != self.expected_blob_hash:
|
||||
self.close_handle()
|
||||
self.finished.set_exception(InvalidBlobHashError(
|
||||
f"blob hash is {blob_hash} vs expected {self.expected_blob_hash}"
|
||||
))
|
||||
@@ -62,6 +61,8 @@ class HashBlobWriter:
|
||||
self.close_handle()
|
||||
|
||||
def close_handle(self):
|
||||
if not self.finished.done():
|
||||
self.finished.cancel()
|
||||
if self.buffer is not None:
|
||||
self.buffer.close()
|
||||
self.buffer = None
|
||||
@@ -4,37 +4,52 @@ import typing
|
||||
import binascii
|
||||
from lbrynet.error import InvalidBlobHashError, InvalidDataError
|
||||
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest
|
||||
from lbrynet.utils import cache_concurrent
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from lbrynet.blob.blob_file import AbstractBlob
|
||||
from lbrynet.blob.writer import HashBlobWriter
|
||||
from lbrynet.connection_manager import ConnectionManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_timeout: typing.Optional[float] = 10):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_timeout: typing.Optional[float] = 10,
|
||||
connection_manager: typing.Optional['ConnectionManager'] = None):
|
||||
self.loop = loop
|
||||
self.peer_port: typing.Optional[int] = None
|
||||
self.peer_address: typing.Optional[str] = None
|
||||
self.transport: typing.Optional[asyncio.Transport] = None
|
||||
self.peer_timeout = peer_timeout
|
||||
self.transport: asyncio.Transport = None
|
||||
|
||||
self.writer: 'HashBlobWriter' = None
|
||||
self.blob: 'BlobFile' = None
|
||||
self.connection_manager = connection_manager
|
||||
self.writer: typing.Optional['HashBlobWriter'] = None
|
||||
self.blob: typing.Optional['AbstractBlob'] = None
|
||||
|
||||
self._blob_bytes_received = 0
|
||||
self._response_fut: asyncio.Future = None
|
||||
self._response_fut: typing.Optional[asyncio.Future] = None
|
||||
self.buf = b''
|
||||
|
||||
# this is here to handle the race when the downloader is closed right as response_fut gets a result
|
||||
self.closed = asyncio.Event(loop=self.loop)
|
||||
|
||||
def data_received(self, data: bytes):
|
||||
log.debug("%s:%d -- got %s bytes -- %s bytes on buffer -- %s blob bytes received",
|
||||
self.peer_address, self.peer_port, len(data), len(self.buf), self._blob_bytes_received)
|
||||
if self.connection_manager:
|
||||
if not self.peer_address:
|
||||
addr_info = self.transport.get_extra_info('peername')
|
||||
self.peer_address, self.peer_port = addr_info
|
||||
# assert self.peer_address is not None
|
||||
self.connection_manager.received_data(f"{self.peer_address}:{self.peer_port}", len(data))
|
||||
#log.debug("%s:%d -- got %s bytes -- %s bytes on buffer -- %s blob bytes received",
|
||||
# self.peer_address, self.peer_port, len(data), len(self.buf), self._blob_bytes_received)
|
||||
if not self.transport or self.transport.is_closing():
|
||||
log.warning("transport closing, but got more bytes from %s:%i\n%s", self.peer_address, self.peer_port,
|
||||
binascii.hexlify(data))
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return
|
||||
if not self._response_fut:
|
||||
log.warning("Protocol received data before expected, probable race on keep alive. Closing transport.")
|
||||
return self.close()
|
||||
if self._blob_bytes_received and not self.writer.closed():
|
||||
return self._write(data)
|
||||
|
||||
@@ -63,8 +78,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
# write blob bytes if we're writing a blob and have blob bytes to write
|
||||
self._write(response.blob_data)
|
||||
|
||||
|
||||
def _write(self, data):
|
||||
def _write(self, data: bytes):
|
||||
if len(data) > (self.blob.get_length() - self._blob_bytes_received):
|
||||
data = data[:(self.blob.get_length() - self._blob_bytes_received)]
|
||||
log.warning("got more than asked from %s:%d, probable sendfile bug", self.peer_address, self.peer_port)
|
||||
@@ -87,14 +101,24 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
:return: download success (bool), keep connection (bool)
|
||||
"""
|
||||
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
|
||||
blob_hash = self.blob.blob_hash
|
||||
if not self.peer_address:
|
||||
addr_info = self.transport.get_extra_info('peername')
|
||||
self.peer_address, self.peer_port = addr_info
|
||||
try:
|
||||
msg = request.serialize()
|
||||
log.debug("send request to %s:%i -> %s", self.peer_address, self.peer_port, msg.decode())
|
||||
self.transport.write(msg)
|
||||
if self.connection_manager:
|
||||
self.connection_manager.sent_data(f"{self.peer_address}:{self.peer_port}", len(msg))
|
||||
response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop)
|
||||
availability_response = response.get_availability_response()
|
||||
price_response = response.get_price_response()
|
||||
blob_response = response.get_blob_response()
|
||||
if self.closed.is_set():
|
||||
msg = f"cancelled blob request for {blob_hash} immediately after we got a response"
|
||||
log.warning(msg)
|
||||
raise asyncio.CancelledError(msg)
|
||||
if (not blob_response or blob_response.error) and\
|
||||
(not availability_response or not availability_response.available_blobs):
|
||||
log.warning("%s not in availability response from %s:%i", self.blob.blob_hash, self.peer_address,
|
||||
@@ -133,6 +157,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
return self._blob_bytes_received, self.close()
|
||||
|
||||
def close(self):
|
||||
self.closed.set()
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
if self.writer and not self.writer.closed():
|
||||
@@ -145,18 +170,21 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
self.transport = None
|
||||
self.buf = b''
|
||||
|
||||
async def download_blob(self, blob: 'BlobFile') -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
if blob.get_is_verified() or blob.file_exists or blob.blob_write_lock.locked():
|
||||
async def download_blob(self, blob: 'AbstractBlob') -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
self.closed.clear()
|
||||
blob_hash = blob.blob_hash
|
||||
if blob.get_is_verified() or not blob.is_writeable():
|
||||
return 0, self.transport
|
||||
try:
|
||||
self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0
|
||||
self._blob_bytes_received = 0
|
||||
self.blob, self.writer = blob, blob.get_blob_writer(self.peer_address, self.peer_port)
|
||||
self._response_fut = asyncio.Future(loop=self.loop)
|
||||
return await self._download_blob()
|
||||
except OSError as e:
|
||||
log.error("race happened downloading from %s:%i", self.peer_address, self.peer_port)
|
||||
# i'm not sure how to fix this race condition - jack
|
||||
log.exception(e)
|
||||
return self._blob_bytes_received, self.transport
|
||||
log.warning("race happened downloading %s from %s:%i", blob_hash, self.peer_address, self.peer_port)
|
||||
# return self._blob_bytes_received, self.transport
|
||||
raise
|
||||
except asyncio.TimeoutError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
@@ -165,30 +193,43 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
except asyncio.CancelledError:
|
||||
self.close()
|
||||
raise
|
||||
finally:
|
||||
if self.writer and not self.writer.closed():
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
|
||||
def connection_made(self, transport: asyncio.Transport):
|
||||
addr = transport.get_extra_info('peername')
|
||||
self.peer_address, self.peer_port = addr[0], addr[1]
|
||||
self.transport = transport
|
||||
self.peer_address, self.peer_port = self.transport.get_extra_info('peername')
|
||||
if self.connection_manager:
|
||||
self.connection_manager.connection_made(f"{self.peer_address}:{self.peer_port}")
|
||||
log.debug("connection made to %s:%i", self.peer_address, self.peer_port)
|
||||
|
||||
def connection_lost(self, reason):
|
||||
if self.connection_manager:
|
||||
self.connection_manager.outgoing_connection_lost(f"{self.peer_address}:{self.peer_port}")
|
||||
log.debug("connection lost to %s:%i (reason: %s, %s)", self.peer_address, self.peer_port, str(reason),
|
||||
str(type(reason)))
|
||||
self.close()
|
||||
|
||||
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', address: str, tcp_port: int,
|
||||
@cache_concurrent
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'AbstractBlob', address: str, tcp_port: int,
|
||||
peer_connect_timeout: float, blob_download_timeout: float,
|
||||
connected_transport: asyncio.Transport = None)\
|
||||
connected_transport: asyncio.Transport = None, connection_id: int = 0,
|
||||
connection_manager: typing.Optional['ConnectionManager'] = None)\
|
||||
-> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
"""
|
||||
Returns [<downloaded blob>, <keep connection>]
|
||||
"""
|
||||
|
||||
protocol = BlobExchangeClientProtocol(loop, blob_download_timeout)
|
||||
protocol = BlobExchangeClientProtocol(
|
||||
loop, blob_download_timeout, connection_manager
|
||||
)
|
||||
if connected_transport and not connected_transport.is_closing():
|
||||
connected_transport.set_protocol(protocol)
|
||||
protocol.connection_made(connected_transport)
|
||||
protocol.transport = connected_transport
|
||||
log.debug("reusing connection for %s:%d", address, tcp_port)
|
||||
else:
|
||||
connected_transport = None
|
||||
@@ -196,7 +237,7 @@ async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', address: s
|
||||
if not connected_transport:
|
||||
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
|
||||
peer_connect_timeout, loop=loop)
|
||||
if blob.get_is_verified() or blob.file_exists:
|
||||
if blob.get_is_verified() or not blob.is_writeable():
|
||||
# file exists but not verified means someone is writing right now, give it time, come back later
|
||||
return 0, connected_transport
|
||||
return await protocol.download_blob(blob)
|
||||
|
||||
@@ -1,21 +1,22 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet.utils import drain_tasks
|
||||
from lbrynet.utils import cache_concurrent
|
||||
from lbrynet.blob_exchange.client import request_blob
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
from lbrynet.blob.blob_file import AbstractBlob
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobDownloader:
|
||||
BAN_TIME = 10.0 # fixme: when connection manager gets implemented, move it out from here
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager',
|
||||
BAN_FACTOR = 2.0 # fixme: when connection manager gets implemented, move it out from here
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
peer_queue: asyncio.Queue):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
@@ -24,114 +25,99 @@ class BlobDownloader:
|
||||
self.active_connections: typing.Dict['KademliaPeer', asyncio.Task] = {} # active request_blob calls
|
||||
self.ignored: typing.Dict['KademliaPeer', int] = {}
|
||||
self.scores: typing.Dict['KademliaPeer', int] = {}
|
||||
self.failures: typing.Dict['KademliaPeer', int] = {}
|
||||
self.connections: typing.Dict['KademliaPeer', asyncio.Transport] = {}
|
||||
self.time_since_last_blob = loop.time()
|
||||
self.is_running = asyncio.Event(loop=self.loop)
|
||||
|
||||
def should_race_continue(self, blob: 'BlobFile'):
|
||||
def should_race_continue(self, blob: 'AbstractBlob'):
|
||||
if len(self.active_connections) >= self.config.max_connections_per_download:
|
||||
return False
|
||||
# if a peer won 3 or more blob races and is active as a downloader, stop the race so bandwidth improves
|
||||
# the safe net side is that any failure will reset the peer score, triggering the race back
|
||||
# TODO: this is a good idea for low bandwidth, but doesnt play nice on high bandwidth
|
||||
# for peer, task in self.active_connections.items():
|
||||
# if self.scores.get(peer, 0) >= 0 and self.rounds_won.get(peer, 0) >= 3 and not task.done():
|
||||
# return False
|
||||
return not (blob.get_is_verified() or blob.file_exists)
|
||||
return not (blob.get_is_verified() or not blob.is_writeable())
|
||||
|
||||
async def request_blob_from_peer(self, blob: 'BlobFile', peer: 'KademliaPeer'):
|
||||
async def request_blob_from_peer(self, blob: 'AbstractBlob', peer: 'KademliaPeer', connection_id: int = 0):
|
||||
if blob.get_is_verified():
|
||||
return
|
||||
self.scores[peer] = self.scores.get(peer, 0) - 1 # starts losing score, to account for cancelled ones
|
||||
transport = self.connections.get(peer)
|
||||
start = self.loop.time()
|
||||
bytes_received, transport = await request_blob(
|
||||
self.loop, blob, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
|
||||
self.config.blob_download_timeout, connected_transport=transport
|
||||
self.config.blob_download_timeout, connected_transport=transport, connection_id=connection_id,
|
||||
connection_manager=self.blob_manager.connection_manager
|
||||
|
||||
)
|
||||
if bytes_received == blob.get_length():
|
||||
self.time_since_last_blob = self.loop.time()
|
||||
if not transport and peer not in self.ignored:
|
||||
self.ignored[peer] = self.loop.time()
|
||||
log.debug("drop peer %s:%i", peer.address, peer.tcp_port)
|
||||
self.failures[peer] = self.failures.get(peer, 0) + 1
|
||||
if peer in self.connections:
|
||||
del self.connections[peer]
|
||||
elif transport:
|
||||
log.debug("keep peer %s:%i", peer.address, peer.tcp_port)
|
||||
self.failures[peer] = 0
|
||||
self.connections[peer] = transport
|
||||
rough_speed = (bytes_received / (self.loop.time() - start)) if bytes_received else 0
|
||||
self.scores[peer] = rough_speed
|
||||
elapsed = self.loop.time() - start
|
||||
self.scores[peer] = bytes_received / elapsed if bytes_received and elapsed else 1
|
||||
|
||||
async def new_peer_or_finished(self, blob: 'BlobFile'):
|
||||
async def get_and_re_add_peers():
|
||||
try:
|
||||
new_peers = await asyncio.wait_for(self.peer_queue.get(), timeout=1.0)
|
||||
self.peer_queue.put_nowait(new_peers)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
tasks = [self.loop.create_task(get_and_re_add_peers()), self.loop.create_task(blob.verified.wait())]
|
||||
active_tasks = list(self.active_connections.values())
|
||||
try:
|
||||
await asyncio.wait(tasks + active_tasks, loop=self.loop, return_when='FIRST_COMPLETED')
|
||||
finally:
|
||||
drain_tasks(tasks)
|
||||
async def new_peer_or_finished(self):
|
||||
active_tasks = list(self.active_connections.values()) + [asyncio.sleep(1)]
|
||||
await asyncio.wait(active_tasks, loop=self.loop, return_when='FIRST_COMPLETED')
|
||||
|
||||
def cleanup_active(self):
|
||||
if not self.active_connections and not self.connections:
|
||||
self.clearbanned()
|
||||
to_remove = [peer for (peer, task) in self.active_connections.items() if task.done()]
|
||||
for peer in to_remove:
|
||||
del self.active_connections[peer]
|
||||
|
||||
def clearbanned(self):
|
||||
now = self.loop.time()
|
||||
if now - self.time_since_last_blob > 60.0:
|
||||
return
|
||||
forgiven = [banned_peer for banned_peer, when in self.ignored.items() if now - when > self.BAN_TIME]
|
||||
self.peer_queue.put_nowait(forgiven)
|
||||
for banned_peer in forgiven:
|
||||
self.ignored.pop(banned_peer)
|
||||
self.ignored = dict((
|
||||
(peer, when) for (peer, when) in self.ignored.items()
|
||||
if (now - when) < min(30.0, (self.failures.get(peer, 0) ** self.BAN_FACTOR))
|
||||
))
|
||||
|
||||
async def download_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
|
||||
@cache_concurrent
|
||||
async def download_blob(self, blob_hash: str, length: typing.Optional[int] = None,
|
||||
connection_id: int = 0) -> 'AbstractBlob':
|
||||
blob = self.blob_manager.get_blob(blob_hash, length)
|
||||
if blob.get_is_verified():
|
||||
return blob
|
||||
self.is_running.set()
|
||||
try:
|
||||
while not blob.get_is_verified():
|
||||
batch: typing.List['KademliaPeer'] = []
|
||||
while not blob.get_is_verified() and self.is_running.is_set():
|
||||
batch: typing.Set['KademliaPeer'] = set()
|
||||
while not self.peer_queue.empty():
|
||||
batch.extend(self.peer_queue.get_nowait())
|
||||
batch.sort(key=lambda peer: self.scores.get(peer, 0), reverse=True)
|
||||
batch.update(self.peer_queue.get_nowait())
|
||||
if batch:
|
||||
self.peer_queue.put_nowait(list(batch))
|
||||
log.debug(
|
||||
"running, %d peers, %d ignored, %d active",
|
||||
len(batch), len(self.ignored), len(self.active_connections)
|
||||
)
|
||||
for peer in batch:
|
||||
for peer in sorted(batch, key=lambda peer: self.scores.get(peer, 0), reverse=True):
|
||||
if not self.should_race_continue(blob):
|
||||
break
|
||||
if peer not in self.active_connections and peer not in self.ignored:
|
||||
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
|
||||
t = self.loop.create_task(self.request_blob_from_peer(blob, peer))
|
||||
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id))
|
||||
self.active_connections[peer] = t
|
||||
await self.new_peer_or_finished(blob)
|
||||
await self.new_peer_or_finished()
|
||||
self.cleanup_active()
|
||||
if batch:
|
||||
self.peer_queue.put_nowait(set(batch).difference(self.ignored))
|
||||
else:
|
||||
self.clearbanned()
|
||||
blob.close()
|
||||
log.debug("downloaded %s", blob_hash[:8])
|
||||
return blob
|
||||
finally:
|
||||
while self.active_connections:
|
||||
self.active_connections.popitem()[1].cancel()
|
||||
blob.close()
|
||||
|
||||
def close(self):
|
||||
self.scores.clear()
|
||||
self.ignored.clear()
|
||||
self.is_running.clear()
|
||||
for transport in self.connections.values():
|
||||
transport.close()
|
||||
|
||||
|
||||
async def download_blob(loop, config: 'Config', blob_manager: 'BlobFileManager', node: 'Node',
|
||||
blob_hash: str) -> 'BlobFile':
|
||||
async def download_blob(loop, config: 'Config', blob_manager: 'BlobManager', node: 'Node',
|
||||
blob_hash: str) -> 'AbstractBlob':
|
||||
search_queue = asyncio.Queue(loop=loop, maxsize=config.max_connections_per_download)
|
||||
search_queue.put_nowait(blob_hash)
|
||||
peer_queue, accumulate_task = node.accumulate_peers(search_queue)
|
||||
|
||||
@@ -8,13 +8,13 @@ from lbrynet.blob_exchange.serialization import BlobAvailabilityResponse, BlobPr
|
||||
BlobPaymentAddressResponse
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobServerProtocol(asyncio.Protocol):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', lbrycrd_address: str):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str):
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
@@ -22,15 +22,23 @@ class BlobServerProtocol(asyncio.Protocol):
|
||||
self.buf = b''
|
||||
self.transport = None
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
self.peer_address_and_port: typing.Optional[str] = None
|
||||
|
||||
def connection_made(self, transport):
|
||||
self.transport = transport
|
||||
self.peer_address_and_port = "%s:%i" % self.transport.get_extra_info('peername')
|
||||
self.blob_manager.connection_manager.connection_received(self.peer_address_and_port)
|
||||
|
||||
def connection_lost(self, exc: typing.Optional[Exception]) -> None:
|
||||
self.blob_manager.connection_manager.incoming_connection_lost(self.peer_address_and_port)
|
||||
|
||||
def send_response(self, responses: typing.List[blob_response_types]):
|
||||
to_send = []
|
||||
while responses:
|
||||
to_send.append(responses.pop())
|
||||
self.transport.write(BlobResponse(to_send).serialize())
|
||||
serialized = BlobResponse(to_send).serialize()
|
||||
self.transport.write(serialized)
|
||||
self.blob_manager.connection_manager.sent_data(self.peer_address_and_port, len(serialized))
|
||||
|
||||
async def handle_request(self, request: BlobRequest):
|
||||
addr = self.transport.get_extra_info('peername')
|
||||
@@ -72,6 +80,7 @@ class BlobServerProtocol(asyncio.Protocol):
|
||||
def data_received(self, data):
|
||||
request = None
|
||||
if data:
|
||||
self.blob_manager.connection_manager.received_data(self.peer_address_and_port, len(data))
|
||||
message, separator, remainder = data.rpartition(b'}')
|
||||
if not separator:
|
||||
self.buf += data
|
||||
@@ -94,10 +103,10 @@ class BlobServerProtocol(asyncio.Protocol):
|
||||
|
||||
|
||||
class BlobServer:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', lbrycrd_address: str):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str):
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
self.server_task: typing.Optional[asyncio.Task] = None
|
||||
self.started_listening = asyncio.Event(loop=self.loop)
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
self.server_protocol_class = BlobServerProtocol
|
||||
|
||||
@@ -1,2 +1,3 @@
|
||||
# don't touch this. CI server changes this during build/deployment
|
||||
BUILD = "dev"
|
||||
BUILD_COMMIT = "source installation"
|
||||
+51
-25
@@ -21,10 +21,6 @@ CURRENCIES = {
|
||||
'LBC': {'type': 'crypto'},
|
||||
'USD': {'type': 'fiat'},
|
||||
}
|
||||
SLACK_WEBHOOK = (
|
||||
'nUE0pUZ6Yl9bo29epl5moTSwnl5wo20ip2IlqzywMKZiIQSFZR5'
|
||||
'AHx4mY0VmF0WQZ1ESEP9kMHZlp1WzJwWOoKN3ImR1M2yUAaMyqGZ='
|
||||
)
|
||||
HEADERS_FILE_SHA256_CHECKSUM = (
|
||||
366295, 'b0c8197153a33ccbc52fb81a279588b6015b68b7726f73f6a2b81f7e25bfe4b9'
|
||||
)
|
||||
@@ -125,7 +121,7 @@ class Toggle(Setting[bool]):
|
||||
)
|
||||
parser.add_argument(
|
||||
self.no_cli_name,
|
||||
help=f"Opposite of --{self.cli_name}",
|
||||
help=f"Opposite of {self.cli_name}",
|
||||
dest=self.name,
|
||||
action="store_false",
|
||||
default=NOT_SET
|
||||
@@ -196,7 +192,21 @@ class MaxKeyFee(Setting[dict]):
|
||||
)
|
||||
|
||||
|
||||
class Servers(Setting[list]):
|
||||
class ListSetting(Setting[list]):
|
||||
|
||||
def validate(self, val):
|
||||
assert isinstance(val, (tuple, list)), \
|
||||
f"Setting '{self.name}' must be a tuple or list."
|
||||
|
||||
def contribute_to_argparse(self, parser: ArgumentParser):
|
||||
parser.add_argument(
|
||||
self.cli_name,
|
||||
help=self.doc,
|
||||
action='append'
|
||||
)
|
||||
|
||||
|
||||
class Servers(ListSetting):
|
||||
|
||||
def validate(self, val):
|
||||
assert isinstance(val, (tuple, list)), \
|
||||
@@ -229,16 +239,8 @@ class Servers(Setting[list]):
|
||||
return [f"{host}:{port}" for host, port in value]
|
||||
return value
|
||||
|
||||
def contribute_to_argparse(self, parser: ArgumentParser):
|
||||
parser.add_argument(
|
||||
self.cli_name,
|
||||
nargs="*",
|
||||
help=self.doc,
|
||||
default=NOT_SET
|
||||
)
|
||||
|
||||
|
||||
class Strings(Setting[list]):
|
||||
class Strings(ListSetting):
|
||||
|
||||
def validate(self, val):
|
||||
assert isinstance(val, (tuple, list)), \
|
||||
@@ -273,7 +275,7 @@ class ArgumentAccess:
|
||||
def load(self, args):
|
||||
for setting in self.configuration.get_settings():
|
||||
value = getattr(args, setting.name, NOT_SET)
|
||||
if value != NOT_SET:
|
||||
if value != NOT_SET and not (isinstance(setting, ListSetting) and value is None):
|
||||
self.args[setting.name] = setting.deserialize(value)
|
||||
|
||||
def __contains__(self, item: str):
|
||||
@@ -484,10 +486,15 @@ class Config(CLIConfig):
|
||||
# protocol timeouts
|
||||
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
|
||||
blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
|
||||
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 2.0)
|
||||
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0)
|
||||
node_rpc_timeout = Float("Timeout when making a DHT request", constants.rpc_timeout)
|
||||
|
||||
# blob announcement and download
|
||||
save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True)
|
||||
blob_lru_cache_size = Integer(
|
||||
"LRU cache size for decrypted downloaded blobs used to minimize re-downloading the same blobs when "
|
||||
"replying to a range request. Set to 0 to disable.", 32
|
||||
)
|
||||
announce_head_and_sd_only = Toggle(
|
||||
"Announce only the descriptor and first (rather than all) data blob for a stream to the DHT", True,
|
||||
previous_names=['announce_head_blobs_only']
|
||||
@@ -497,7 +504,7 @@ class Config(CLIConfig):
|
||||
previous_names=['concurrent_announcers']
|
||||
)
|
||||
max_connections_per_download = Integer(
|
||||
"Maximum number of peers to connect to while downloading a blob", 8,
|
||||
"Maximum number of peers to connect to while downloading a blob", 4,
|
||||
previous_names=['max_connections_per_stream']
|
||||
)
|
||||
fixed_peer_delay = Float(
|
||||
@@ -519,31 +526,50 @@ class Config(CLIConfig):
|
||||
|
||||
# servers
|
||||
reflector_servers = Servers("Reflector re-hosting servers", [
|
||||
('reflector.lbry.io', 5566)
|
||||
('reflector.lbry.com', 5566)
|
||||
])
|
||||
lbryum_servers = Servers("SPV wallet servers", [
|
||||
('lbryumx1.lbry.io', 50001),
|
||||
('lbryumx2.lbry.io', 50001)
|
||||
('lbryumx1.lbry.com', 50001), # US EAST
|
||||
('lbryumx2.lbry.com', 50001), # US EAST
|
||||
('lbryumx4.lbry.com', 50001) #EU
|
||||
])
|
||||
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
|
||||
('lbrynet1.lbry.io', 4444), # US EAST
|
||||
('lbrynet2.lbry.io', 4444), # US WEST
|
||||
('lbrynet3.lbry.io', 4444), # EU
|
||||
('lbrynet4.lbry.io', 4444) # ASIA
|
||||
('lbrynet1.lbry.com', 4444), # US EAST
|
||||
('lbrynet2.lbry.com', 4444), # US WEST
|
||||
('lbrynet3.lbry.com', 4444), # EU
|
||||
('lbrynet4.lbry.com', 4444) # ASIA
|
||||
])
|
||||
|
||||
comment_server = String("Comment server API URL", "https://comments.lbry.com/api")
|
||||
|
||||
# blockchain
|
||||
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
|
||||
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
|
||||
cache_time = Integer("Time to cache resolved claims", 150) # TODO: use this
|
||||
|
||||
# daemon
|
||||
save_files = Toggle("Save downloaded files when calling `get` by default", True)
|
||||
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
|
||||
share_usage_data = Toggle(
|
||||
"Whether to share usage stats and diagnostic info with LBRY.", True,
|
||||
previous_names=['upload_log', 'upload_log', 'share_debug_info']
|
||||
)
|
||||
|
||||
# media server
|
||||
|
||||
streaming_server = String('Host name and port to serve streaming media over range requests',
|
||||
'localhost:5280', metavar='HOST:PORT')
|
||||
streaming_get = Toggle("Enable the /get endpoint for the streaming media server. "
|
||||
"Disable to prevent new streams from being added.", True)
|
||||
|
||||
@property
|
||||
def streaming_host(self):
|
||||
return self.streaming_server.split(':')[0]
|
||||
|
||||
@property
|
||||
def streaming_port(self):
|
||||
return int(self.streaming_server.split(':')[1])
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.set_default_paths()
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import collections
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
CONNECTED_EVENT = "connected"
|
||||
DISCONNECTED_EVENT = "disconnected"
|
||||
TRANSFERRED_EVENT = "transferred"
|
||||
|
||||
|
||||
class ConnectionManager:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop):
|
||||
self.loop = loop
|
||||
self.incoming_connected: typing.Set[str] = set()
|
||||
self.incoming: typing.DefaultDict[str, int] = collections.defaultdict(int)
|
||||
self.outgoing_connected: typing.Set[str] = set()
|
||||
self.outgoing: typing.DefaultDict[str, int] = collections.defaultdict(int)
|
||||
self._status = {}
|
||||
|
||||
self._task: typing.Optional[asyncio.Task] = None
|
||||
|
||||
@property
|
||||
def status(self):
|
||||
return self._status
|
||||
|
||||
def sent_data(self, host_and_port: str, size: int):
|
||||
self.outgoing[host_and_port] += size
|
||||
|
||||
def received_data(self, host_and_port: str, size: int):
|
||||
self.incoming[host_and_port] += size
|
||||
|
||||
def connection_made(self, host_and_port: str):
|
||||
self.outgoing_connected.add(host_and_port)
|
||||
|
||||
def connection_received(self, host_and_port: str):
|
||||
# self.incoming_connected.add(host_and_port)
|
||||
pass
|
||||
|
||||
def outgoing_connection_lost(self, host_and_port: str):
|
||||
if host_and_port in self.outgoing_connected:
|
||||
self.outgoing_connected.remove(host_and_port)
|
||||
|
||||
def incoming_connection_lost(self, host_and_port: str):
|
||||
if host_and_port in self.incoming_connected:
|
||||
self.incoming_connected.remove(host_and_port)
|
||||
|
||||
async def _update(self):
|
||||
|
||||
self._status = {
|
||||
'incoming_bps': {},
|
||||
'outgoing_bps': {},
|
||||
'total_incoming_mbs': 0.0,
|
||||
'total_outgoing_mbs': 0.0,
|
||||
'time': self.loop.time()
|
||||
}
|
||||
|
||||
while True:
|
||||
last = self.loop.time()
|
||||
await asyncio.sleep(1, loop=self.loop)
|
||||
self._status['incoming_bps'].clear()
|
||||
self._status['outgoing_bps'].clear()
|
||||
while self.outgoing:
|
||||
k, v = self.outgoing.popitem()
|
||||
self._status['outgoing_bps'][k] = v
|
||||
while self.incoming:
|
||||
k, v = self.incoming.popitem()
|
||||
self._status['incoming_bps'][k] = v
|
||||
now = self.loop.time()
|
||||
self._status['total_outgoing_mbs'] = int(sum(list(self._status['outgoing_bps'].values())
|
||||
) / (now - last)) / 1000000.0
|
||||
self._status['total_incoming_mbs'] = int(sum(list(self._status['incoming_bps'].values())
|
||||
) / (now - last)) / 1000000.0
|
||||
self._status['time'] = now
|
||||
|
||||
def stop(self):
|
||||
if self._task:
|
||||
self._task.cancel()
|
||||
self._task = None
|
||||
self.outgoing.clear()
|
||||
self.outgoing_connected.clear()
|
||||
self.incoming.clear()
|
||||
self.incoming_connected.clear()
|
||||
self._status.clear()
|
||||
|
||||
def start(self):
|
||||
self.stop()
|
||||
self._task = self.loop.create_task(self._update())
|
||||
@@ -13,55 +13,47 @@ class BlobAnnouncer:
|
||||
self.loop = loop
|
||||
self.node = node
|
||||
self.storage = storage
|
||||
self.pending_call: asyncio.Handle = None
|
||||
self.announce_task: asyncio.Task = None
|
||||
self.running = False
|
||||
self.announce_queue: typing.List[str] = []
|
||||
|
||||
async def _announce(self, batch_size: typing.Optional[int] = 10):
|
||||
if not batch_size:
|
||||
return
|
||||
if not self.node.joined.is_set():
|
||||
await self.node.joined.wait()
|
||||
blob_hashes = await self.storage.get_blobs_to_announce()
|
||||
if blob_hashes:
|
||||
self.announce_queue.extend(blob_hashes)
|
||||
log.info("%i blobs to announce", len(blob_hashes))
|
||||
batch = []
|
||||
while len(self.announce_queue):
|
||||
cnt = 0
|
||||
announced = []
|
||||
while self.announce_queue and cnt < batch_size:
|
||||
blob_hash = self.announce_queue.pop()
|
||||
announced.append(blob_hash)
|
||||
batch.append(self.node.announce_blob(blob_hash))
|
||||
cnt += 1
|
||||
to_await = []
|
||||
while batch:
|
||||
to_await.append(batch.pop())
|
||||
if to_await:
|
||||
await asyncio.gather(*tuple(to_await), loop=self.loop)
|
||||
await self.storage.update_last_announced_blobs(announced)
|
||||
log.info("announced %i blobs", len(announced))
|
||||
if self.running:
|
||||
self.pending_call = self.loop.call_later(60, self.announce, batch_size)
|
||||
async def _submit_announcement(self, blob_hash):
|
||||
try:
|
||||
peers = len(await self.node.announce_blob(blob_hash))
|
||||
if peers > 4:
|
||||
return blob_hash
|
||||
else:
|
||||
log.warning("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError):
|
||||
raise err
|
||||
log.warning("error announcing %s: %s", blob_hash[:8], str(err))
|
||||
|
||||
def announce(self, batch_size: typing.Optional[int] = 10):
|
||||
self.announce_task = self.loop.create_task(self._announce(batch_size))
|
||||
|
||||
async def _announce(self, batch_size: typing.Optional[int] = 10):
|
||||
while batch_size:
|
||||
if not self.node.joined.is_set():
|
||||
await self.node.joined.wait()
|
||||
await asyncio.sleep(60, loop=self.loop)
|
||||
if not self.node.protocol.routing_table.get_peers():
|
||||
log.warning("No peers in DHT, announce round skipped")
|
||||
continue
|
||||
self.announce_queue.extend(await self.storage.get_blobs_to_announce())
|
||||
log.debug("announcer task wake up, %d blobs to announce", len(self.announce_queue))
|
||||
while len(self.announce_queue):
|
||||
log.info("%i blobs to announce", len(self.announce_queue))
|
||||
announced = await asyncio.gather(*[
|
||||
self._submit_announcement(
|
||||
self.announce_queue.pop()) for _ in range(batch_size) if self.announce_queue
|
||||
], loop=self.loop)
|
||||
announced = list(filter(None, announced))
|
||||
if announced:
|
||||
await self.storage.update_last_announced_blobs(announced)
|
||||
log.info("announced %i blobs", len(announced))
|
||||
|
||||
def start(self, batch_size: typing.Optional[int] = 10):
|
||||
if self.running:
|
||||
raise Exception("already running")
|
||||
self.running = True
|
||||
self.announce(batch_size)
|
||||
assert not self.announce_task or self.announce_task.done(), "already running"
|
||||
self.announce_task = self.loop.create_task(self._announce(batch_size))
|
||||
|
||||
def stop(self):
|
||||
self.running = False
|
||||
if self.pending_call:
|
||||
if not self.pending_call.cancelled():
|
||||
self.pending_call.cancel()
|
||||
self.pending_call = None
|
||||
if self.announce_task:
|
||||
if not (self.announce_task.done() or self.announce_task.cancelled()):
|
||||
self.announce_task.cancel()
|
||||
self.announce_task = None
|
||||
if self.announce_task and not self.announce_task.done():
|
||||
self.announce_task.cancel()
|
||||
+69
-98
@@ -2,11 +2,8 @@ import logging
|
||||
import asyncio
|
||||
import typing
|
||||
import binascii
|
||||
import contextlib
|
||||
from lbrynet.utils import resolve_host
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.error import RemoteException
|
||||
from lbrynet.dht.protocol.async_generator_junction import AsyncGeneratorJunction
|
||||
from lbrynet.dht.protocol.distance import Distance
|
||||
from lbrynet.dht.protocol.iterative_find import IterativeNodeFinder, IterativeValueFinder
|
||||
from lbrynet.dht.protocol.protocol import KademliaProtocol
|
||||
@@ -31,7 +28,7 @@ class Node:
|
||||
self._join_task: asyncio.Task = None
|
||||
self._refresh_task: asyncio.Task = None
|
||||
|
||||
async def refresh_node(self):
|
||||
async def refresh_node(self, force_once=False):
|
||||
while True:
|
||||
# remove peers with expired blob announcements from the datastore
|
||||
self.protocol.data_store.removed_expired_peers()
|
||||
@@ -58,6 +55,8 @@ class Node:
|
||||
peers = await self.peer_search(node_ids.pop())
|
||||
total_peers.extend(peers)
|
||||
else:
|
||||
if force_once:
|
||||
break
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
self.loop.call_later(constants.refresh_interval // 4, fut.set_result, None)
|
||||
await fut
|
||||
@@ -67,30 +66,33 @@ class Node:
|
||||
to_ping = [peer for peer in set(total_peers) if self.protocol.peer_manager.peer_is_good(peer) is not True]
|
||||
if to_ping:
|
||||
self.protocol.ping_queue.enqueue_maybe_ping(*to_ping, delay=0)
|
||||
if force_once:
|
||||
break
|
||||
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
self.loop.call_later(constants.refresh_interval, fut.set_result, None)
|
||||
await fut
|
||||
|
||||
async def announce_blob(self, blob_hash: str) -> typing.List[bytes]:
|
||||
announced_to_node_ids = []
|
||||
while not announced_to_node_ids:
|
||||
hash_value = binascii.unhexlify(blob_hash.encode())
|
||||
assert len(hash_value) == constants.hash_length
|
||||
peers = await self.peer_search(hash_value)
|
||||
hash_value = binascii.unhexlify(blob_hash.encode())
|
||||
assert len(hash_value) == constants.hash_length
|
||||
peers = await self.peer_search(hash_value)
|
||||
|
||||
if not self.protocol.external_ip:
|
||||
raise Exception("Cannot determine external IP")
|
||||
log.debug("Store to %i peers", len(peers))
|
||||
for peer in peers:
|
||||
log.debug("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
|
||||
stored_to_tup = await asyncio.gather(
|
||||
*(self.protocol.store_to_peer(hash_value, peer) for peer in peers), loop=self.loop
|
||||
)
|
||||
announced_to_node_ids.extend([node_id for node_id, contacted in stored_to_tup if contacted])
|
||||
if not self.protocol.external_ip:
|
||||
raise Exception("Cannot determine external IP")
|
||||
log.debug("Store to %i peers", len(peers))
|
||||
for peer in peers:
|
||||
log.debug("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
|
||||
stored_to_tup = await asyncio.gather(
|
||||
*(self.protocol.store_to_peer(hash_value, peer) for peer in peers), loop=self.loop
|
||||
)
|
||||
stored_to = [node_id for node_id, contacted in stored_to_tup if contacted]
|
||||
if stored_to:
|
||||
log.info("Stored %s to %i of %i attempted peers", binascii.hexlify(hash_value).decode()[:8],
|
||||
len(announced_to_node_ids), len(peers))
|
||||
return announced_to_node_ids
|
||||
len(stored_to), len(peers))
|
||||
else:
|
||||
log.warning("Failed announcing %s, stored to 0 peers", blob_hash[:8])
|
||||
return stored_to
|
||||
|
||||
def stop(self) -> None:
|
||||
if self.joined.is_set():
|
||||
@@ -101,6 +103,7 @@ class Node:
|
||||
self._refresh_task.cancel()
|
||||
if self.protocol and self.protocol.ping_queue.running:
|
||||
self.protocol.ping_queue.stop()
|
||||
self.protocol.stop()
|
||||
if self.listening_port is not None:
|
||||
self.listening_port.close()
|
||||
self._join_task = None
|
||||
@@ -113,6 +116,7 @@ class Node:
|
||||
lambda: self.protocol, (interface, self.internal_udp_port)
|
||||
)
|
||||
log.info("DHT node listening on UDP %s:%i", interface, self.internal_udp_port)
|
||||
self.protocol.start()
|
||||
else:
|
||||
log.warning("Already bound to port %s", self.listening_port)
|
||||
|
||||
@@ -130,33 +134,31 @@ class Node:
|
||||
if known_node_urls:
|
||||
for host, port in known_node_urls:
|
||||
address = await resolve_host(host, port, proto='udp')
|
||||
if (address, port) not in known_node_addresses and address != self.protocol.external_ip:
|
||||
if (address, port) not in known_node_addresses and\
|
||||
(address, port) != (self.protocol.external_ip, self.protocol.udp_port):
|
||||
known_node_addresses.append((address, port))
|
||||
url_to_addr[address] = host
|
||||
|
||||
if known_node_addresses:
|
||||
while not self.protocol.routing_table.get_peers():
|
||||
success = False
|
||||
# ping the seed nodes, this will set their node ids (since we don't know them ahead of time)
|
||||
for address, port in known_node_addresses:
|
||||
peer = self.protocol.get_rpc_peer(KademliaPeer(self.loop, address, udp_port=port))
|
||||
try:
|
||||
await peer.ping()
|
||||
success = True
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("seed node (%s:%i) timed out in %s", url_to_addr.get(address, address), port,
|
||||
round(self.protocol.rpc_timeout, 2))
|
||||
if success:
|
||||
break
|
||||
# now that we have the seed nodes in routing, to an iterative lookup of our own id to populate the buckets
|
||||
# in the routing table with good peers who are near us
|
||||
async with self.peer_search_junction(self.protocol.node_id, max_results=16) as junction:
|
||||
async for peers in junction:
|
||||
for peer in peers:
|
||||
try:
|
||||
await self.protocol.get_rpc_peer(peer).ping()
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
pass
|
||||
peers = [
|
||||
KademliaPeer(self.loop, address, udp_port=port)
|
||||
for (address, port) in known_node_addresses
|
||||
]
|
||||
while True:
|
||||
if not self.protocol.routing_table.get_peers():
|
||||
if self.joined.is_set():
|
||||
self.joined.clear()
|
||||
self.protocol.peer_manager.reset()
|
||||
self.protocol.ping_queue.enqueue_maybe_ping(*peers, delay=0.0)
|
||||
peers.extend(await self.peer_search(self.protocol.node_id, shortlist=peers, count=32))
|
||||
if self.protocol.routing_table.get_peers():
|
||||
self.joined.set()
|
||||
log.info(
|
||||
"Joined DHT, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
|
||||
self.protocol.routing_table.buckets_with_contacts())
|
||||
else:
|
||||
continue
|
||||
await asyncio.sleep(1, loop=self.loop)
|
||||
|
||||
log.info("Joined DHT, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
|
||||
self.protocol.routing_table.buckets_with_contacts())
|
||||
@@ -169,79 +171,48 @@ class Node:
|
||||
)
|
||||
)
|
||||
|
||||
def get_iterative_node_finder(self, key: bytes, shortlist: typing.Optional[typing.List] = None,
|
||||
def get_iterative_node_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
|
||||
bottom_out_limit: int = constants.bottom_out_limit,
|
||||
max_results: int = constants.k) -> IterativeNodeFinder:
|
||||
|
||||
return IterativeNodeFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
|
||||
key, bottom_out_limit, max_results, None, shortlist)
|
||||
|
||||
def get_iterative_value_finder(self, key: bytes, shortlist: typing.Optional[typing.List] = None,
|
||||
def get_iterative_value_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
|
||||
bottom_out_limit: int = 40,
|
||||
max_results: int = -1) -> IterativeValueFinder:
|
||||
|
||||
return IterativeValueFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
|
||||
key, bottom_out_limit, max_results, None, shortlist)
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def stream_peer_search_junction(self, hash_queue: asyncio.Queue, bottom_out_limit=20,
|
||||
max_results=-1) -> AsyncGeneratorJunction:
|
||||
peer_generator = AsyncGeneratorJunction(self.loop)
|
||||
|
||||
async def _add_hashes_from_queue():
|
||||
while True:
|
||||
blob_hash = await hash_queue.get()
|
||||
peer_generator.add_generator(
|
||||
self.get_iterative_value_finder(
|
||||
binascii.unhexlify(blob_hash.encode()), bottom_out_limit=bottom_out_limit,
|
||||
max_results=max_results
|
||||
)
|
||||
)
|
||||
add_hashes_task = self.loop.create_task(_add_hashes_from_queue())
|
||||
try:
|
||||
async with peer_generator as junction:
|
||||
yield junction
|
||||
await peer_generator.finished.wait()
|
||||
finally:
|
||||
if add_hashes_task and not (add_hashes_task.done() or add_hashes_task.cancelled()):
|
||||
add_hashes_task.cancel()
|
||||
|
||||
def peer_search_junction(self, node_id: bytes, max_results=constants.k*2,
|
||||
bottom_out_limit=20) -> AsyncGeneratorJunction:
|
||||
peer_generator = AsyncGeneratorJunction(self.loop)
|
||||
peer_generator.add_generator(
|
||||
self.get_iterative_node_finder(
|
||||
node_id, bottom_out_limit=bottom_out_limit, max_results=max_results
|
||||
)
|
||||
)
|
||||
return peer_generator
|
||||
|
||||
async def peer_search(self, node_id: bytes, count=constants.k, max_results=constants.k*2,
|
||||
bottom_out_limit=20) -> typing.List['KademliaPeer']:
|
||||
accumulated: typing.List['KademliaPeer'] = []
|
||||
async with self.peer_search_junction(node_id, max_results=max_results,
|
||||
bottom_out_limit=bottom_out_limit) as junction:
|
||||
async for peers in junction:
|
||||
accumulated.extend(peers)
|
||||
bottom_out_limit=20, shortlist: typing.Optional[typing.List['KademliaPeer']] = None
|
||||
) -> typing.List['KademliaPeer']:
|
||||
peers = []
|
||||
async for iteration_peers in self.get_iterative_node_finder(
|
||||
node_id, shortlist=shortlist, bottom_out_limit=bottom_out_limit, max_results=max_results):
|
||||
peers.extend(iteration_peers)
|
||||
distance = Distance(node_id)
|
||||
accumulated.sort(key=lambda peer: distance(peer.node_id))
|
||||
return accumulated[:count]
|
||||
peers.sort(key=lambda peer: distance(peer.node_id))
|
||||
return peers[:count]
|
||||
|
||||
async def _accumulate_search_junction(self, search_queue: asyncio.Queue,
|
||||
result_queue: asyncio.Queue):
|
||||
async with self.stream_peer_search_junction(search_queue) as search_junction: # pylint: disable=E1701
|
||||
async for peers in search_junction:
|
||||
if peers:
|
||||
result_queue.put_nowait([
|
||||
peer for peer in peers
|
||||
if not (
|
||||
peer.address == self.protocol.external_ip
|
||||
and peer.tcp_port == self.protocol.peer_port
|
||||
)
|
||||
])
|
||||
tasks = []
|
||||
try:
|
||||
while True:
|
||||
blob_hash = await search_queue.get()
|
||||
tasks.append(self.loop.create_task(self._value_producer(blob_hash, result_queue)))
|
||||
finally:
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
|
||||
async def _value_producer(self, blob_hash: str, result_queue: asyncio.Queue):
|
||||
async for results in self.get_iterative_value_finder(binascii.unhexlify(blob_hash.encode())):
|
||||
result_queue.put_nowait(results)
|
||||
|
||||
def accumulate_peers(self, search_queue: asyncio.Queue,
|
||||
peer_queue: typing.Optional[asyncio.Queue] = None) -> typing.Tuple[
|
||||
asyncio.Queue, asyncio.Task]:
|
||||
q = peer_queue or asyncio.Queue()
|
||||
return q, asyncio.create_task(self._accumulate_search_junction(search_queue, q))
|
||||
q = peer_queue or asyncio.Queue(loop=self.loop)
|
||||
return q, self.loop.create_task(self._accumulate_search_junction(search_queue, q))
|
||||
+19
-6
@@ -31,7 +31,10 @@ class PeerManager:
|
||||
self._node_id_mapping: typing.Dict[typing.Tuple[str, int], bytes] = {}
|
||||
self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = {}
|
||||
self._node_tokens: typing.Dict[bytes, (float, bytes)] = {}
|
||||
self._kademlia_peers: typing.Dict[typing.Tuple[bytes, str, int], 'KademliaPeer']
|
||||
|
||||
def reset(self):
|
||||
for statistic in (self._rpc_failures, self._last_replied, self._last_sent, self._last_requested):
|
||||
statistic.clear()
|
||||
|
||||
def report_failure(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
@@ -104,11 +107,12 @@ class PeerManager:
|
||||
|
||||
delay = self._loop.time() - constants.check_refresh_interval
|
||||
|
||||
if node_id not in self._node_id_reverse_mapping or (address, udp_port) not in self._node_id_mapping:
|
||||
return
|
||||
addr_tup = (address, udp_port)
|
||||
if self._node_id_reverse_mapping[node_id] != addr_tup or self._node_id_mapping[addr_tup] != node_id:
|
||||
return
|
||||
# fixme: find a way to re-enable that without breaking other parts
|
||||
#if node_id not in self._node_id_reverse_mapping or (address, udp_port) not in self._node_id_mapping:
|
||||
# return
|
||||
#addr_tup = (address, udp_port)
|
||||
#if self._node_id_reverse_mapping[node_id] != addr_tup or self._node_id_mapping[addr_tup] != node_id:
|
||||
# return
|
||||
previous_failure, most_recent_failure = self._rpc_failures.get((address, udp_port), (None, None))
|
||||
last_requested = self._last_requested.get((address, udp_port))
|
||||
last_replied = self._last_replied.get((address, udp_port))
|
||||
@@ -135,6 +139,15 @@ class PeerManager:
|
||||
|
||||
|
||||
class KademliaPeer:
|
||||
__slots__ = [
|
||||
'loop',
|
||||
'_node_id',
|
||||
'address',
|
||||
'udp_port',
|
||||
'tcp_port',
|
||||
'protocol_version',
|
||||
]
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, address: str, node_id: typing.Optional[bytes] = None,
|
||||
udp_port: typing.Optional[int] = None, tcp_port: typing.Optional[int] = None):
|
||||
if node_id is not None:
|
||||
|
||||
@@ -1,105 +0,0 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
import traceback
|
||||
if typing.TYPE_CHECKING:
|
||||
from types import AsyncGeneratorType
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def cancel_task(task: typing.Optional[asyncio.Task]):
|
||||
if task and not (task.done() or task.cancelled()):
|
||||
task.cancel()
|
||||
|
||||
|
||||
def cancel_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
|
||||
for task in tasks:
|
||||
cancel_task(task)
|
||||
|
||||
|
||||
def drain_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
|
||||
while tasks:
|
||||
cancel_task(tasks.pop())
|
||||
|
||||
|
||||
class AsyncGeneratorJunction:
|
||||
"""
|
||||
A helper to interleave the results from multiple async generators into one
|
||||
async generator.
|
||||
"""
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, queue: typing.Optional[asyncio.Queue] = None):
|
||||
self.loop = loop
|
||||
self.result_queue = queue or asyncio.Queue(loop=loop)
|
||||
self.tasks: typing.List[asyncio.Task] = []
|
||||
self.running_iterators: typing.Dict[typing.AsyncGenerator, bool] = {}
|
||||
self.generator_queue: asyncio.Queue = asyncio.Queue(loop=self.loop)
|
||||
self.can_iterate = asyncio.Event(loop=self.loop)
|
||||
self.finished = asyncio.Event(loop=self.loop)
|
||||
|
||||
@property
|
||||
def running(self):
|
||||
return any(self.running_iterators.values())
|
||||
|
||||
async def wait_for_generators(self):
|
||||
async def iterate(iterator: typing.AsyncGenerator):
|
||||
try:
|
||||
async for item in iterator:
|
||||
self.result_queue.put_nowait(item)
|
||||
finally:
|
||||
self.running_iterators[iterator] = False
|
||||
|
||||
while True:
|
||||
async_gen: typing.Union[typing.AsyncGenerator, 'AsyncGeneratorType'] = await self.generator_queue.get()
|
||||
self.running_iterators[async_gen] = True
|
||||
self.tasks.append(self.loop.create_task(iterate(async_gen)))
|
||||
if not self.can_iterate.is_set():
|
||||
self.can_iterate.set()
|
||||
|
||||
def add_generator(self, async_gen: typing.Union[typing.AsyncGenerator, 'AsyncGeneratorType']):
|
||||
"""
|
||||
Add an async generator. This can be called during an iteration of the generator junction.
|
||||
"""
|
||||
self.generator_queue.put_nowait(async_gen)
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
async def __anext__(self):
|
||||
if not self.can_iterate.is_set():
|
||||
await self.can_iterate.wait()
|
||||
if not self.running:
|
||||
raise StopAsyncIteration()
|
||||
try:
|
||||
return await self.result_queue.get()
|
||||
finally:
|
||||
self.awaiting = None
|
||||
|
||||
def aclose(self):
|
||||
async def _aclose():
|
||||
for iterator in list(self.running_iterators.keys()):
|
||||
result = iterator.aclose()
|
||||
if asyncio.iscoroutine(result):
|
||||
await result
|
||||
self.running_iterators[iterator] = False
|
||||
drain_tasks(self.tasks)
|
||||
raise StopAsyncIteration()
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return self.loop.create_task(_aclose())
|
||||
|
||||
async def __aenter__(self):
|
||||
self.tasks.append(self.loop.create_task(self.wait_for_generators()))
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
try:
|
||||
await self.aclose()
|
||||
except StopAsyncIteration:
|
||||
pass
|
||||
finally:
|
||||
if exc_type:
|
||||
if exc_type not in (asyncio.CancelledError, asyncio.TimeoutError, StopAsyncIteration, GeneratorExit):
|
||||
err = traceback.format_exception(exc_type, exc, tb)
|
||||
log.error(err)
|
||||
@@ -9,68 +9,62 @@ if typing.TYPE_CHECKING:
|
||||
class DictDataStore:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager'):
|
||||
# Dictionary format:
|
||||
# { <key>: [<contact>, <value>, <lastPublished>, <originallyPublished> <original_publisher_id>] }
|
||||
self._data_store: typing.Dict[bytes,
|
||||
typing.List[typing.Tuple['KademliaPeer', bytes, float, float, bytes]]] = {}
|
||||
self._get_time = loop.time
|
||||
# { <key>: [(<contact>, <age>), ...] }
|
||||
self._data_store: typing.Dict[bytes, typing.List[typing.Tuple['KademliaPeer', float]]] = {}
|
||||
|
||||
self.loop = loop
|
||||
self._peer_manager = peer_manager
|
||||
self.completed_blobs: typing.Set[str] = set()
|
||||
|
||||
def filter_bad_and_expired_peers(self, key: bytes) -> typing.List['KademliaPeer']:
|
||||
def removed_expired_peers(self):
|
||||
now = self.loop.time()
|
||||
keys = list(self._data_store.keys())
|
||||
for key in keys:
|
||||
to_remove = []
|
||||
for (peer, ts) in self._data_store[key]:
|
||||
if ts + constants.data_expiration < now or self._peer_manager.peer_is_good(peer) is False:
|
||||
to_remove.append((peer, ts))
|
||||
for item in to_remove:
|
||||
self._data_store[key].remove(item)
|
||||
if not self._data_store[key]:
|
||||
del self._data_store[key]
|
||||
|
||||
def filter_bad_and_expired_peers(self, key: bytes) -> typing.Iterator['KademliaPeer']:
|
||||
"""
|
||||
Returns only non-expired and unknown/good peers
|
||||
"""
|
||||
peers = []
|
||||
for peer in map(lambda p: p[0],
|
||||
filter(lambda peer: self._get_time() - peer[3] < constants.data_expiration,
|
||||
self._data_store[key])):
|
||||
for peer in self.filter_expired_peers(key):
|
||||
if self._peer_manager.peer_is_good(peer) is not False:
|
||||
peers.append(peer)
|
||||
return peers
|
||||
yield peer
|
||||
|
||||
def filter_expired_peers(self, key: bytes) -> typing.List['KademliaPeer']:
|
||||
def filter_expired_peers(self, key: bytes) -> typing.Iterator['KademliaPeer']:
|
||||
"""
|
||||
Returns only non-expired peers
|
||||
"""
|
||||
return list(
|
||||
map(
|
||||
lambda p: p[0],
|
||||
filter(lambda peer: self._get_time() - peer[3] < constants.data_expiration, self._data_store[key])
|
||||
)
|
||||
)
|
||||
|
||||
def removed_expired_peers(self):
|
||||
expired_keys = []
|
||||
for key in self._data_store.keys():
|
||||
unexpired_peers = self.filter_expired_peers(key)
|
||||
if not unexpired_peers:
|
||||
expired_keys.append(key)
|
||||
else:
|
||||
self._data_store[key] = [x for x in self._data_store[key] if x[0] in unexpired_peers]
|
||||
for key in expired_keys:
|
||||
del self._data_store[key]
|
||||
now = self.loop.time()
|
||||
for (peer, ts) in self._data_store.get(key, []):
|
||||
if ts + constants.data_expiration > now:
|
||||
yield peer
|
||||
|
||||
def has_peers_for_blob(self, key: bytes) -> bool:
|
||||
return key in self._data_store and len(self.filter_bad_and_expired_peers(key)) > 0
|
||||
return key in self._data_store
|
||||
|
||||
def add_peer_to_blob(self, contact: 'KademliaPeer', key: bytes, compact_address: bytes, last_published: int,
|
||||
originally_published: int, original_publisher_id: bytes) -> None:
|
||||
def add_peer_to_blob(self, contact: 'KademliaPeer', key: bytes) -> None:
|
||||
now = self.loop.time()
|
||||
if key in self._data_store:
|
||||
if compact_address not in map(lambda store_tuple: store_tuple[1], self._data_store[key]):
|
||||
self._data_store[key].append(
|
||||
(contact, compact_address, last_published, originally_published, original_publisher_id)
|
||||
)
|
||||
current = list(filter(lambda x: x[0] == contact, self._data_store[key]))
|
||||
if len(current):
|
||||
self._data_store[key][self._data_store[key].index(current[0])] = contact, now
|
||||
else:
|
||||
self._data_store[key].append((contact, now))
|
||||
else:
|
||||
self._data_store[key] = [(contact, compact_address, last_published, originally_published,
|
||||
original_publisher_id)]
|
||||
self._data_store[key] = [(contact, now)]
|
||||
|
||||
def get_peers_for_blob(self, key: bytes) -> typing.List['KademliaPeer']:
|
||||
return [] if key not in self._data_store else [peer for peer in self.filter_bad_and_expired_peers(key)]
|
||||
return list(self.filter_bad_and_expired_peers(key))
|
||||
|
||||
def get_storing_contacts(self) -> typing.List['KademliaPeer']:
|
||||
peers = set()
|
||||
for key in self._data_store:
|
||||
for values in self._data_store[key]:
|
||||
if values[0] not in peers:
|
||||
peers.add(values[0])
|
||||
for key, stored in self._data_store.items():
|
||||
peers.update(set(map(lambda tup: tup[0], stored)))
|
||||
return list(peers)
|
||||
@@ -10,11 +10,13 @@ class Distance:
|
||||
|
||||
def __init__(self, key: bytes):
|
||||
if len(key) != constants.hash_length:
|
||||
raise ValueError("invalid key length: %i" % len(key))
|
||||
raise ValueError(f"invalid key length: {len(key)}")
|
||||
self.key = key
|
||||
self.val_key_one = int.from_bytes(key, 'big')
|
||||
|
||||
def __call__(self, key_two: bytes) -> int:
|
||||
if len(key_two) != constants.hash_length:
|
||||
raise ValueError(f"invalid length of key to compare: {len(key_two)}")
|
||||
val_key_two = int.from_bytes(key_two, 'big')
|
||||
return self.val_key_one ^ val_key_two
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ from itertools import chain
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.error import RemoteException
|
||||
from lbrynet.dht.error import RemoteException, TransportNotConnected
|
||||
from lbrynet.dht.protocol.distance import Distance
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -66,11 +66,7 @@ def get_shortlist(routing_table: 'TreeRoutingTable', key: bytes,
|
||||
"""
|
||||
if len(key) != constants.hash_length:
|
||||
raise ValueError("invalid key length: %i" % len(key))
|
||||
if not shortlist:
|
||||
shortlist = routing_table.find_close_peers(key)
|
||||
distance = Distance(key)
|
||||
shortlist.sort(key=lambda peer: distance(peer.node_id), reverse=True)
|
||||
return shortlist
|
||||
return shortlist or routing_table.find_close_peers(key)
|
||||
|
||||
|
||||
class IterativeFinder:
|
||||
@@ -91,12 +87,11 @@ class IterativeFinder:
|
||||
self.max_results = max_results
|
||||
self.exclude = exclude or []
|
||||
|
||||
self.shortlist: typing.List['KademliaPeer'] = get_shortlist(routing_table, key, shortlist)
|
||||
self.active: typing.Set['KademliaPeer'] = set()
|
||||
self.contacted: typing.Set[typing.Tuple[str, int]] = set()
|
||||
self.contacted: typing.Set['KademliaPeer'] = set()
|
||||
self.distance = Distance(key)
|
||||
|
||||
self.closest_peer: typing.Optional['KademliaPeer'] = None if not self.shortlist else self.shortlist[0]
|
||||
self.closest_peer: typing.Optional['KademliaPeer'] = None
|
||||
self.prev_closest_peer: typing.Optional['KademliaPeer'] = None
|
||||
|
||||
self.iteration_queue = asyncio.Queue(loop=self.loop)
|
||||
@@ -107,6 +102,12 @@ class IterativeFinder:
|
||||
self.running = False
|
||||
self.tasks: typing.List[asyncio.Task] = []
|
||||
self.delayed_calls: typing.List[asyncio.Handle] = []
|
||||
for peer in get_shortlist(routing_table, key, shortlist):
|
||||
if peer.node_id:
|
||||
self._add_active(peer)
|
||||
else:
|
||||
# seed nodes
|
||||
self._schedule_probe(peer)
|
||||
|
||||
async def send_probe(self, peer: 'KademliaPeer') -> FindResponse:
|
||||
"""
|
||||
@@ -138,25 +139,18 @@ class IterativeFinder:
|
||||
def _is_closer(self, peer: 'KademliaPeer') -> bool:
|
||||
return not self.closest_peer or self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
|
||||
|
||||
def _update_closest(self):
|
||||
self.shortlist.sort(key=lambda peer: self.distance(peer.node_id), reverse=True)
|
||||
if self.closest_peer and self.closest_peer is not self.shortlist[-1]:
|
||||
if self._is_closer(self.shortlist[-1]):
|
||||
def _add_active(self, peer):
|
||||
if peer not in self.active and peer.node_id and peer.node_id != self.protocol.node_id:
|
||||
self.active.add(peer)
|
||||
if self._is_closer(peer):
|
||||
self.prev_closest_peer = self.closest_peer
|
||||
self.closest_peer = self.shortlist[-1]
|
||||
self.closest_peer = peer
|
||||
|
||||
async def _handle_probe_result(self, peer: 'KademliaPeer', response: FindResponse):
|
||||
if peer not in self.shortlist:
|
||||
self.shortlist.append(peer)
|
||||
if peer not in self.active:
|
||||
self.active.add(peer)
|
||||
self._add_active(peer)
|
||||
for contact_triple in response.get_close_triples():
|
||||
node_id, address, udp_port = contact_triple
|
||||
if (address, udp_port) not in self.contacted: # and not self.peer_manager.is_ignored(addr_tuple)
|
||||
found_peer = self.peer_manager.get_kademlia_peer(node_id, address, udp_port)
|
||||
if found_peer not in self.shortlist and self.peer_manager.peer_is_good(peer) is not False:
|
||||
self.shortlist.append(found_peer)
|
||||
self._update_closest()
|
||||
self._add_active(self.peer_manager.get_kademlia_peer(node_id, address, udp_port))
|
||||
self.check_result_ready(response)
|
||||
|
||||
async def _send_probe(self, peer: 'KademliaPeer'):
|
||||
@@ -169,53 +163,59 @@ class IterativeFinder:
|
||||
log.warning(str(err))
|
||||
self.active.discard(peer)
|
||||
return
|
||||
except TransportNotConnected:
|
||||
return self.aclose()
|
||||
except RemoteException:
|
||||
return
|
||||
return await self._handle_probe_result(peer, response)
|
||||
|
||||
async def _search_round(self):
|
||||
"""
|
||||
Send up to constants.alpha (5) probes to the closest peers in the shortlist
|
||||
Send up to constants.alpha (5) probes to closest active peers
|
||||
"""
|
||||
|
||||
added = 0
|
||||
self.shortlist.sort(key=lambda p: self.distance(p.node_id), reverse=True)
|
||||
while self.running and len(self.shortlist) and added < constants.alpha:
|
||||
peer = self.shortlist.pop()
|
||||
to_probe = list(self.active - self.contacted)
|
||||
to_probe.sort(key=lambda peer: self.distance(self.key))
|
||||
for peer in to_probe:
|
||||
if added >= constants.alpha:
|
||||
break
|
||||
origin_address = (peer.address, peer.udp_port)
|
||||
if origin_address in self.exclude or self.peer_manager.peer_is_good(peer) is False:
|
||||
if origin_address in self.exclude:
|
||||
continue
|
||||
if peer.node_id == self.protocol.node_id:
|
||||
continue
|
||||
if (peer.address, peer.udp_port) == (self.protocol.external_ip, self.protocol.udp_port):
|
||||
if origin_address == (self.protocol.external_ip, self.protocol.udp_port):
|
||||
continue
|
||||
if (peer.address, peer.udp_port) not in self.contacted:
|
||||
self.contacted.add((peer.address, peer.udp_port))
|
||||
|
||||
t = self.loop.create_task(self._send_probe(peer))
|
||||
|
||||
def callback(_):
|
||||
self.running_probes.difference_update({
|
||||
probe for probe in self.running_probes if probe.done() or probe == t
|
||||
})
|
||||
if not self.running_probes and self.shortlist:
|
||||
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
|
||||
|
||||
t.add_done_callback(callback)
|
||||
self.running_probes.add(t)
|
||||
added += 1
|
||||
self._schedule_probe(peer)
|
||||
added += 1
|
||||
log.debug("running %d probes", len(self.running_probes))
|
||||
if not added and not self.running_probes:
|
||||
log.debug("search for %s exhausted", hexlify(self.key)[:8])
|
||||
self.search_exhausted()
|
||||
|
||||
def _schedule_probe(self, peer: 'KademliaPeer'):
|
||||
self.contacted.add(peer)
|
||||
|
||||
t = self.loop.create_task(self._send_probe(peer))
|
||||
|
||||
def callback(_):
|
||||
self.running_probes.difference_update({
|
||||
probe for probe in self.running_probes if probe.done() or probe == t
|
||||
})
|
||||
if not self.running_probes:
|
||||
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
|
||||
|
||||
t.add_done_callback(callback)
|
||||
self.running_probes.add(t)
|
||||
|
||||
async def _search_task(self, delay: typing.Optional[float] = constants.iterative_lookup_delay):
|
||||
try:
|
||||
if self.running:
|
||||
await self._search_round()
|
||||
if self.running:
|
||||
self.delayed_calls.append(self.loop.call_later(delay, self._search))
|
||||
except (asyncio.CancelledError, StopAsyncIteration):
|
||||
except (asyncio.CancelledError, StopAsyncIteration, TransportNotConnected):
|
||||
if self.running:
|
||||
self.loop.call_soon(self.aclose)
|
||||
|
||||
@@ -264,6 +264,7 @@ class IterativeNodeFinder(IterativeFinder):
|
||||
self.yielded_peers: typing.Set['KademliaPeer'] = set()
|
||||
|
||||
async def send_probe(self, peer: 'KademliaPeer') -> FindNodeResponse:
|
||||
log.debug("probing %s:%d %s", peer.address, peer.udp_port, hexlify(peer.node_id)[:8] if peer.node_id else '')
|
||||
response = await self.protocol.get_rpc_peer(peer).find_node(self.key)
|
||||
return FindNodeResponse(self.key, response)
|
||||
|
||||
@@ -271,12 +272,16 @@ class IterativeNodeFinder(IterativeFinder):
|
||||
self.put_result(self.active, finish=True)
|
||||
|
||||
def put_result(self, from_iter: typing.Iterable['KademliaPeer'], finish=False):
|
||||
not_yet_yielded = [peer for peer in from_iter if peer not in self.yielded_peers]
|
||||
not_yet_yielded = [
|
||||
peer for peer in from_iter
|
||||
if peer not in self.yielded_peers
|
||||
and peer.node_id != self.protocol.node_id
|
||||
and self.peer_manager.peer_is_good(peer) is not False
|
||||
]
|
||||
not_yet_yielded.sort(key=lambda peer: self.distance(peer.node_id))
|
||||
to_yield = not_yet_yielded[:min(constants.k, len(not_yet_yielded))]
|
||||
if to_yield:
|
||||
for peer in to_yield:
|
||||
self.yielded_peers.add(peer)
|
||||
self.yielded_peers.update(to_yield)
|
||||
self.iteration_queue.put_nowait(to_yield)
|
||||
if finish:
|
||||
self.iteration_queue.put_nowait(None)
|
||||
@@ -286,21 +291,17 @@ class IterativeNodeFinder(IterativeFinder):
|
||||
|
||||
if found:
|
||||
log.debug("found")
|
||||
return self.put_result(self.shortlist, finish=True)
|
||||
return self.put_result(self.active, finish=True)
|
||||
if self.prev_closest_peer and self.closest_peer and not self._is_closer(self.prev_closest_peer):
|
||||
# log.info("improving, %i %i %i %i %i", len(self.shortlist), len(self.active), len(self.contacted),
|
||||
# self.bottom_out_count, self.iteration_count)
|
||||
self.bottom_out_count = 0
|
||||
elif self.prev_closest_peer and self.closest_peer:
|
||||
self.bottom_out_count += 1
|
||||
log.info("bottom out %i %i %i %i", len(self.active), len(self.contacted), len(self.shortlist),
|
||||
self.bottom_out_count)
|
||||
log.info("bottom out %i %i %i", len(self.active), len(self.contacted), self.bottom_out_count)
|
||||
if self.bottom_out_count >= self.bottom_out_limit or self.iteration_count >= self.bottom_out_limit:
|
||||
log.info("limit hit")
|
||||
self.put_result(self.active, True)
|
||||
elif self.max_results and len(self.active) - len(self.yielded_peers) >= self.max_results:
|
||||
log.debug("max results")
|
||||
self.put_result(self.active, True)
|
||||
|
||||
|
||||
class IterativeValueFinder(IterativeFinder):
|
||||
|
||||
@@ -45,19 +45,19 @@ class KademliaRPC:
|
||||
def ping():
|
||||
return b'pong'
|
||||
|
||||
def store(self, rpc_contact: 'KademliaPeer', blob_hash: bytes, token: bytes, port: int,
|
||||
original_publisher_id: bytes, age: int) -> bytes:
|
||||
if original_publisher_id is None:
|
||||
original_publisher_id = rpc_contact.node_id
|
||||
def store(self, rpc_contact: 'KademliaPeer', blob_hash: bytes, token: bytes, port: int) -> bytes:
|
||||
if len(blob_hash) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid length of blob hash: {len(blob_hash)}")
|
||||
if not 0 < port < 65535:
|
||||
raise ValueError(f"invalid tcp port: {port}")
|
||||
rpc_contact.update_tcp_port(port)
|
||||
if self.loop.time() - self.protocol.started_listening_time < constants.token_secret_refresh_interval:
|
||||
pass
|
||||
elif not self.verify_token(token, rpc_contact.compact_ip()):
|
||||
raise ValueError("Invalid token")
|
||||
now = int(self.loop.time())
|
||||
originally_published = now - age
|
||||
if not self.verify_token(token, rpc_contact.compact_ip()):
|
||||
if self.loop.time() - self.protocol.started_listening_time < constants.token_secret_refresh_interval:
|
||||
pass
|
||||
else:
|
||||
raise ValueError("Invalid token")
|
||||
self.protocol.data_store.add_peer_to_blob(
|
||||
rpc_contact, blob_hash, rpc_contact.compact_address_tcp(), now, originally_published, original_publisher_id
|
||||
rpc_contact, blob_hash
|
||||
)
|
||||
return b'OK'
|
||||
|
||||
@@ -191,12 +191,14 @@ class PingQueue:
|
||||
self._process_task: asyncio.Task = None
|
||||
self._running = False
|
||||
self._running_pings: typing.Set[asyncio.Task] = set()
|
||||
self._default_delay = constants.maybe_ping_delay
|
||||
|
||||
@property
|
||||
def running(self):
|
||||
return self._running
|
||||
|
||||
def enqueue_maybe_ping(self, *peers: 'KademliaPeer', delay: float = constants.maybe_ping_delay):
|
||||
def enqueue_maybe_ping(self, *peers: 'KademliaPeer', delay: typing.Optional[float] = None):
|
||||
delay = delay if delay is not None else self._default_delay
|
||||
now = self._loop.time()
|
||||
for peer in peers:
|
||||
if peer not in self._pending_contacts or now + delay < self._pending_contacts[peer]:
|
||||
@@ -207,7 +209,7 @@ class PingQueue:
|
||||
try:
|
||||
if self._protocol.peer_manager.peer_is_good(peer):
|
||||
if peer not in self._protocol.routing_table.get_peers():
|
||||
await self._protocol.add_peer(peer)
|
||||
self._protocol.add_peer(peer)
|
||||
return
|
||||
await self._protocol.get_rpc_peer(peer).ping()
|
||||
except asyncio.TimeoutError:
|
||||
@@ -268,11 +270,21 @@ class KademliaProtocol(DatagramProtocol):
|
||||
self.node_rpc = KademliaRPC(self, self.loop, self.peer_port)
|
||||
self.rpc_timeout = rpc_timeout
|
||||
self._split_lock = asyncio.Lock(loop=self.loop)
|
||||
self._to_remove: typing.Set['KademliaPeer'] = set()
|
||||
self._to_add: typing.Set['KademliaPeer'] = set()
|
||||
self._wakeup_routing_task = asyncio.Event(loop=self.loop)
|
||||
self.maintaing_routing_task: typing.Optional[asyncio.Task] = None
|
||||
|
||||
@functools.lru_cache(128)
|
||||
def get_rpc_peer(self, peer: 'KademliaPeer') -> RemoteKademliaRPC:
|
||||
return RemoteKademliaRPC(self.loop, self.peer_manager, self, peer)
|
||||
|
||||
def start(self):
|
||||
self.maintaing_routing_task = self.loop.create_task(self.routing_table_task())
|
||||
|
||||
def stop(self):
|
||||
if self.maintaing_routing_task:
|
||||
self.maintaing_routing_task.cancel()
|
||||
if self.transport:
|
||||
self.disconnect()
|
||||
|
||||
@@ -363,13 +375,30 @@ class KademliaProtocol(DatagramProtocol):
|
||||
self.routing_table.buckets[bucket_index].remove_peer(to_replace)
|
||||
return await self._add_peer(peer)
|
||||
|
||||
async def add_peer(self, peer: 'KademliaPeer') -> bool:
|
||||
def add_peer(self, peer: 'KademliaPeer'):
|
||||
if peer.node_id == self.node_id:
|
||||
return False
|
||||
async with self._split_lock:
|
||||
return await self._add_peer(peer)
|
||||
self._to_add.add(peer)
|
||||
self._wakeup_routing_task.set()
|
||||
|
||||
async def _handle_rpc(self, sender_contact: 'KademliaPeer', message: RequestDatagram):
|
||||
def remove_peer(self, peer: 'KademliaPeer'):
|
||||
self._to_remove.add(peer)
|
||||
self._wakeup_routing_task.set()
|
||||
|
||||
async def routing_table_task(self):
|
||||
while True:
|
||||
while self._to_remove:
|
||||
async with self._split_lock:
|
||||
peer = self._to_remove.pop()
|
||||
self.routing_table.remove_peer(peer)
|
||||
self.routing_table.join_buckets()
|
||||
while self._to_add:
|
||||
async with self._split_lock:
|
||||
await self._add_peer(self._to_add.pop())
|
||||
await asyncio.gather(self._wakeup_routing_task.wait(), asyncio.sleep(.1, loop=self.loop), loop=self.loop)
|
||||
self._wakeup_routing_task.clear()
|
||||
|
||||
def _handle_rpc(self, sender_contact: 'KademliaPeer', message: RequestDatagram):
|
||||
assert sender_contact.node_id != self.node_id, (binascii.hexlify(sender_contact.node_id)[:8].decode(),
|
||||
binascii.hexlify(self.node_id)[:8].decode())
|
||||
method = message.method
|
||||
@@ -387,7 +416,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
result = self.node_rpc.ping()
|
||||
elif method == b'store':
|
||||
blob_hash, token, port, original_publisher_id, age = a
|
||||
result = self.node_rpc.store(sender_contact, blob_hash, token, port, original_publisher_id, age)
|
||||
result = self.node_rpc.store(sender_contact, blob_hash, token, port)
|
||||
elif method == b'findNode':
|
||||
key, = a
|
||||
result = self.node_rpc.find_node(sender_contact, key)
|
||||
@@ -396,11 +425,11 @@ class KademliaProtocol(DatagramProtocol):
|
||||
key, = a
|
||||
result = self.node_rpc.find_value(sender_contact, key)
|
||||
|
||||
await self.send_response(
|
||||
self.send_response(
|
||||
sender_contact, ResponseDatagram(RESPONSE_TYPE, message.rpc_id, self.node_id, result),
|
||||
)
|
||||
|
||||
async def handle_request_datagram(self, address: typing.Tuple[str, int], request_datagram: RequestDatagram):
|
||||
def handle_request_datagram(self, address: typing.Tuple[str, int], request_datagram: RequestDatagram):
|
||||
# This is an RPC method request
|
||||
self.peer_manager.report_last_requested(address[0], address[1])
|
||||
try:
|
||||
@@ -408,7 +437,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
except IndexError:
|
||||
peer = self.peer_manager.get_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
try:
|
||||
await self._handle_rpc(peer, request_datagram)
|
||||
self._handle_rpc(peer, request_datagram)
|
||||
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
|
||||
# will be added to our routing table if successful
|
||||
is_good = self.peer_manager.peer_is_good(peer)
|
||||
@@ -416,12 +445,12 @@ class KademliaProtocol(DatagramProtocol):
|
||||
self.ping_queue.enqueue_maybe_ping(peer)
|
||||
# only add a requesting contact to the routing table if it has replied to one of our requests
|
||||
elif is_good is True:
|
||||
await self.add_peer(peer)
|
||||
self.add_peer(peer)
|
||||
except ValueError as err:
|
||||
log.debug("error raised handling %s request from %s:%i - %s(%s)",
|
||||
request_datagram.method, peer.address, peer.udp_port, str(type(err)),
|
||||
str(err))
|
||||
await self.send_error(
|
||||
self.send_error(
|
||||
peer,
|
||||
ErrorDatagram(ERROR_TYPE, request_datagram.rpc_id, self.node_id, str(type(err)).encode(),
|
||||
str(err).encode())
|
||||
@@ -430,13 +459,13 @@ class KademliaProtocol(DatagramProtocol):
|
||||
log.warning("error raised handling %s request from %s:%i - %s(%s)",
|
||||
request_datagram.method, peer.address, peer.udp_port, str(type(err)),
|
||||
str(err))
|
||||
await self.send_error(
|
||||
self.send_error(
|
||||
peer,
|
||||
ErrorDatagram(ERROR_TYPE, request_datagram.rpc_id, self.node_id, str(type(err)).encode(),
|
||||
str(err).encode())
|
||||
)
|
||||
|
||||
async def handle_response_datagram(self, address: typing.Tuple[str, int], response_datagram: ResponseDatagram):
|
||||
def handle_response_datagram(self, address: typing.Tuple[str, int], response_datagram: ResponseDatagram):
|
||||
# Find the message that triggered this response
|
||||
if response_datagram.rpc_id in self.sent_messages:
|
||||
peer, df, request = self.sent_messages[response_datagram.rpc_id]
|
||||
@@ -459,7 +488,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
|
||||
if not df.cancelled():
|
||||
df.set_result(response_datagram)
|
||||
await self.add_peer(peer)
|
||||
self.add_peer(peer)
|
||||
else:
|
||||
log.warning("%s:%i replied, but after we cancelled the request attempt",
|
||||
peer.address, peer.udp_port)
|
||||
@@ -510,37 +539,39 @@ class KademliaProtocol(DatagramProtocol):
|
||||
return
|
||||
|
||||
if isinstance(message, RequestDatagram):
|
||||
self.loop.create_task(self.handle_request_datagram(address, message))
|
||||
self.handle_request_datagram(address, message)
|
||||
elif isinstance(message, ErrorDatagram):
|
||||
self.handle_error_datagram(address, message)
|
||||
else:
|
||||
assert isinstance(message, ResponseDatagram), "sanity"
|
||||
self.loop.create_task(self.handle_response_datagram(address, message))
|
||||
self.handle_response_datagram(address, message)
|
||||
|
||||
async def send_request(self, peer: 'KademliaPeer', request: RequestDatagram) -> ResponseDatagram:
|
||||
await self._send(peer, request)
|
||||
self._send(peer, request)
|
||||
response_fut = self.sent_messages[request.rpc_id][1]
|
||||
try:
|
||||
response = await asyncio.wait_for(response_fut, self.rpc_timeout)
|
||||
self.peer_manager.report_last_replied(peer.address, peer.udp_port)
|
||||
return response
|
||||
except asyncio.CancelledError:
|
||||
if not response_fut.done():
|
||||
response_fut.cancel()
|
||||
raise
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
if self.peer_manager.peer_is_good(peer) is False:
|
||||
async with self._split_lock:
|
||||
self.routing_table.remove_peer(peer)
|
||||
self.routing_table.join_buckets()
|
||||
self.remove_peer(peer)
|
||||
raise
|
||||
|
||||
async def send_response(self, peer: 'KademliaPeer', response: ResponseDatagram):
|
||||
await self._send(peer, response)
|
||||
def send_response(self, peer: 'KademliaPeer', response: ResponseDatagram):
|
||||
self._send(peer, response)
|
||||
|
||||
async def send_error(self, peer: 'KademliaPeer', error: ErrorDatagram):
|
||||
await self._send(peer, error)
|
||||
def send_error(self, peer: 'KademliaPeer', error: ErrorDatagram):
|
||||
self._send(peer, error)
|
||||
|
||||
async def _send(self, peer: 'KademliaPeer', message: typing.Union[RequestDatagram, ResponseDatagram,
|
||||
def _send(self, peer: 'KademliaPeer', message: typing.Union[RequestDatagram, ResponseDatagram,
|
||||
ErrorDatagram]):
|
||||
if not self.transport:
|
||||
if not self.transport or self.transport.is_closing():
|
||||
raise TransportNotConnected()
|
||||
|
||||
data = message.bencode()
|
||||
@@ -598,12 +629,15 @@ class KademliaProtocol(DatagramProtocol):
|
||||
return True
|
||||
|
||||
async def store_to_peer(self, hash_value: bytes, peer: 'KademliaPeer') -> typing.Tuple[bytes, bool]:
|
||||
try:
|
||||
async def __store():
|
||||
res = await self.get_rpc_peer(peer).store(hash_value)
|
||||
if res != b"OK":
|
||||
raise ValueError(res)
|
||||
log.debug("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
return peer.node_id, True
|
||||
|
||||
try:
|
||||
return await __store()
|
||||
except asyncio.TimeoutError:
|
||||
log.debug("Timeout while storing blob_hash %s at %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
except ValueError as err:
|
||||
@@ -611,6 +645,10 @@ class KademliaProtocol(DatagramProtocol):
|
||||
except Exception as err:
|
||||
if 'Invalid token' in str(err):
|
||||
self.peer_manager.clear_token(peer.node_id)
|
||||
try:
|
||||
return await __store()
|
||||
except:
|
||||
return peer.node_id, False
|
||||
else:
|
||||
log.exception("Unexpected error while storing blob_hash")
|
||||
return peer.node_id, False
|
||||
|
||||
@@ -65,45 +65,36 @@ class RequestDatagram(KademliaDatagramBase):
|
||||
|
||||
@classmethod
|
||||
def make_ping(cls, from_node_id: bytes, rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
|
||||
if rpc_id and len(rpc_id) != constants.rpc_id_length:
|
||||
raise ValueError("invalid rpc id length")
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(from_node_id) != constants.hash_bits // 8:
|
||||
raise ValueError("invalid node id")
|
||||
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'ping')
|
||||
|
||||
@classmethod
|
||||
def make_store(cls, from_node_id: bytes, blob_hash: bytes, token: bytes, port: int,
|
||||
rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
|
||||
if rpc_id and len(rpc_id) != constants.rpc_id_length:
|
||||
raise ValueError("invalid rpc id length")
|
||||
if not rpc_id:
|
||||
rpc_id = constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(from_node_id) != constants.hash_bits // 8:
|
||||
raise ValueError("invalid node id")
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(blob_hash) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid blob hash length: {len(blob_hash)}")
|
||||
if not 0 < port < 65536:
|
||||
raise ValueError(f"invalid port: {port}")
|
||||
if len(token) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid token length: {len(token)}")
|
||||
store_args = [blob_hash, token, port, from_node_id, 0]
|
||||
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'store', store_args)
|
||||
|
||||
@classmethod
|
||||
def make_find_node(cls, from_node_id: bytes, key: bytes,
|
||||
rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
|
||||
if rpc_id and len(rpc_id) != constants.rpc_id_length:
|
||||
raise ValueError("invalid rpc id length")
|
||||
if not rpc_id:
|
||||
rpc_id = constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(from_node_id) != constants.hash_bits // 8:
|
||||
raise ValueError("invalid node id")
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(key) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid key length: {len(key)}")
|
||||
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'findNode', [key])
|
||||
|
||||
@classmethod
|
||||
def make_find_value(cls, from_node_id: bytes, key: bytes,
|
||||
rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
|
||||
if rpc_id and len(rpc_id) != constants.rpc_id_length:
|
||||
raise ValueError("invalid rpc id length")
|
||||
if not rpc_id:
|
||||
rpc_id = constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(from_node_id) != constants.hash_bits // 8:
|
||||
raise ValueError("invalid node id")
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(key) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid key length: {len(key)}")
|
||||
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'findValue', [key])
|
||||
|
||||
|
||||
@@ -147,8 +138,6 @@ def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDa
|
||||
}
|
||||
|
||||
primitive: typing.Dict = bdecode(datagram)
|
||||
if not isinstance(primitive, dict):
|
||||
raise ValueError("invalid datagram type")
|
||||
if primitive[0] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
|
||||
datagram_type = primitive[0] # pylint: disable=unsubscriptable-object
|
||||
else:
|
||||
@@ -162,14 +151,19 @@ def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDa
|
||||
)
|
||||
|
||||
|
||||
def make_compact_ip(address: str):
|
||||
return reduce(lambda buff, x: buff + bytearray([int(x)]), address.split('.'), bytearray())
|
||||
def make_compact_ip(address: str) -> bytearray:
|
||||
compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), address.split('.'), bytearray())
|
||||
if len(compact_ip) != 4:
|
||||
raise ValueError(f"invalid IPv4 length")
|
||||
return compact_ip
|
||||
|
||||
|
||||
def make_compact_address(node_id: bytes, address: str, port: int) -> bytearray:
|
||||
compact_ip = make_compact_ip(address)
|
||||
if not 0 <= port <= 65536:
|
||||
if not 0 < port < 65536:
|
||||
raise ValueError(f'Invalid port: {port}')
|
||||
if len(node_id) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid node node_id length")
|
||||
return compact_ip + port.to_bytes(2, 'big') + node_id
|
||||
|
||||
|
||||
@@ -177,4 +171,8 @@ def decode_compact_address(compact_address: bytes) -> typing.Tuple[bytes, str, i
|
||||
address = "{}.{}.{}.{}".format(*compact_address[:4])
|
||||
port = int.from_bytes(compact_address[4:6], 'big')
|
||||
node_id = compact_address[6:]
|
||||
if not 0 < port < 65536:
|
||||
raise ValueError(f'Invalid port: {port}')
|
||||
if len(node_id) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid node node_id length")
|
||||
return node_id, address, port
|
||||
@@ -32,6 +32,12 @@ class DownloadDataTimeout(Exception):
|
||||
self.download = download
|
||||
|
||||
|
||||
class ResolveTimeout(Exception):
|
||||
def __init__(self, uri):
|
||||
super().__init__(f'Failed to resolve "{uri}" within the timeout')
|
||||
self.uri = uri
|
||||
|
||||
|
||||
class RequestCanceledError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ def display(data):
|
||||
print(json.dumps(data, indent=2))
|
||||
|
||||
|
||||
async def execute_command(conf, method, params):
|
||||
async def execute_command(conf, method, params, callback=display):
|
||||
async with aiohttp.ClientSession() as session:
|
||||
try:
|
||||
message = {'method': method, 'params': params}
|
||||
@@ -33,12 +33,9 @@ async def execute_command(conf, method, params):
|
||||
try:
|
||||
data = await resp.json()
|
||||
if 'result' in data:
|
||||
display(data['result'])
|
||||
return callback(data['result'])
|
||||
elif 'error' in data:
|
||||
if 'message' in data['error']:
|
||||
display(data['error']['message'])
|
||||
else:
|
||||
display(data['error'])
|
||||
return callback(data['error'])
|
||||
except Exception as e:
|
||||
log.exception('Could not process response from server:', exc_info=e)
|
||||
except aiohttp.ClientConnectionError:
|
||||
@@ -172,7 +169,7 @@ def add_command_parser(parent, command):
|
||||
|
||||
def get_argument_parser():
|
||||
main = ArgumentParser(
|
||||
'lbrynet', description='An interface to the LBRY Network.'
|
||||
'lbrynet', description='An interface to the LBRY Network.', allow_abbrev=False,
|
||||
)
|
||||
main.add_argument(
|
||||
'-v', '--version', dest='cli_version', action="store_true",
|
||||
@@ -195,8 +192,8 @@ def get_argument_parser():
|
||||
help=('Enable debug output. Optionally specify loggers for which debug output '
|
||||
'should selectively be applied.')
|
||||
)
|
||||
start.set_defaults(command='start', start_parser=start)
|
||||
Config.contribute_to_argparse(start)
|
||||
start.set_defaults(command='start', start_parser=start, doc=start.format_help())
|
||||
|
||||
api = Daemon.get_api_definitions()
|
||||
groups = {}
|
||||
@@ -242,6 +239,7 @@ def main(argv=None):
|
||||
if args.help:
|
||||
args.start_parser.print_help()
|
||||
return 0
|
||||
|
||||
default_formatter = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s")
|
||||
file_handler = logging.handlers.RotatingFileHandler(
|
||||
conf.log_file_path, maxBytes=2097152, backupCount=5
|
||||
@@ -267,6 +265,7 @@ def main(argv=None):
|
||||
loop.set_debug(True)
|
||||
else:
|
||||
log.setLevel(logging.INFO)
|
||||
|
||||
if conf.share_usage_data:
|
||||
loggly_handler = get_loggly_handler()
|
||||
loggly_handler.setLevel(logging.ERROR)
|
||||
|
||||
@@ -12,19 +12,18 @@ from aioupnp import __version__ as aioupnp_version
|
||||
from aioupnp.upnp import UPnP
|
||||
from aioupnp.fault import UPnPError
|
||||
|
||||
import lbrynet.schema
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import HEADERS_FILE_SHA256_CHECKSUM
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.blob_announcer import BlobAnnouncer
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
from lbrynet.blob_exchange.server import BlobServer
|
||||
from lbrynet.stream.stream_manager import StreamManager
|
||||
from lbrynet.extras.daemon.Component import Component
|
||||
from lbrynet.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.extras.wallet import LbryWalletManager
|
||||
from lbrynet.extras.wallet import Network
|
||||
from lbrynet.wallet import LbryWalletManager
|
||||
from lbrynet.wallet import Network
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -64,7 +63,7 @@ class DatabaseComponent(Component):
|
||||
|
||||
@staticmethod
|
||||
def get_current_db_revision():
|
||||
return 9
|
||||
return 11
|
||||
|
||||
@property
|
||||
def revision_filename(self):
|
||||
@@ -252,23 +251,20 @@ class WalletComponent(Component):
|
||||
|
||||
async def get_status(self):
|
||||
if self.wallet_manager and self.running:
|
||||
local_height = self.wallet_manager.network.get_local_height()
|
||||
remote_height = self.wallet_manager.network.get_server_height()
|
||||
local_height = self.wallet_manager.ledger.headers.height
|
||||
remote_height = await self.wallet_manager.ledger.network.get_server_height()
|
||||
best_hash = self.wallet_manager.get_best_blockhash()
|
||||
return {
|
||||
'blocks': max(local_height, 0),
|
||||
'blocks_behind': max(remote_height - local_height, 0),
|
||||
'best_blockhash': best_hash,
|
||||
'is_encrypted': self.wallet_manager.wallet.use_encryption,
|
||||
'is_encrypted': self.wallet_manager.use_encryption,
|
||||
'is_locked': not self.wallet_manager.is_wallet_unlocked,
|
||||
}
|
||||
|
||||
async def start(self):
|
||||
log.info("Starting torba wallet")
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
lbrynet.schema.BLOCKCHAIN_NAME = self.conf.blockchain_name
|
||||
self.wallet_manager = await LbryWalletManager.from_lbrynet_config(self.conf, storage)
|
||||
self.wallet_manager.old_db = storage
|
||||
self.wallet_manager = await LbryWalletManager.from_lbrynet_config(self.conf)
|
||||
await self.wallet_manager.start()
|
||||
|
||||
async def stop(self):
|
||||
@@ -282,10 +278,10 @@ class BlobComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.blob_manager: BlobFileManager = None
|
||||
self.blob_manager: typing.Optional[BlobManager] = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[BlobFileManager]:
|
||||
def component(self) -> typing.Optional[BlobManager]:
|
||||
return self.blob_manager
|
||||
|
||||
async def start(self):
|
||||
@@ -295,8 +291,10 @@ class BlobComponent(Component):
|
||||
dht_node: Node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
if dht_node:
|
||||
data_store = dht_node.protocol.data_store
|
||||
self.blob_manager = BlobFileManager(asyncio.get_event_loop(), os.path.join(self.conf.data_dir, "blobfiles"),
|
||||
storage, data_store)
|
||||
blob_dir = os.path.join(self.conf.data_dir, 'blobfiles')
|
||||
if not os.path.isdir(blob_dir):
|
||||
os.mkdir(blob_dir)
|
||||
self.blob_manager = BlobManager(self.component_manager.loop, blob_dir, storage, self.conf, data_store)
|
||||
return await self.blob_manager.setup()
|
||||
|
||||
async def stop(self):
|
||||
@@ -306,7 +304,10 @@ class BlobComponent(Component):
|
||||
count = 0
|
||||
if self.blob_manager:
|
||||
count = len(self.blob_manager.completed_blob_hashes)
|
||||
return {'finished_blobs': count}
|
||||
return {
|
||||
'finished_blobs': count,
|
||||
'connections': {} if not self.blob_manager else self.blob_manager.connection_manager.status
|
||||
}
|
||||
|
||||
|
||||
class DHTComponent(Component):
|
||||
@@ -407,7 +408,7 @@ class StreamManagerComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.stream_manager: StreamManager = None
|
||||
self.stream_manager: typing.Optional[StreamManager] = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[StreamManager]:
|
||||
@@ -417,7 +418,7 @@ class StreamManagerComponent(Component):
|
||||
if not self.stream_manager:
|
||||
return
|
||||
return {
|
||||
'managed_files': len(self.stream_manager.streams)
|
||||
'managed_files': len(self.stream_manager.streams),
|
||||
}
|
||||
|
||||
async def start(self):
|
||||
@@ -446,7 +447,7 @@ class PeerProtocolServerComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.blob_server: BlobServer = None
|
||||
self.blob_server: typing.Optional[BlobServer] = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[BlobServer]:
|
||||
@@ -455,7 +456,7 @@ class PeerProtocolServerComponent(Component):
|
||||
async def start(self):
|
||||
log.info("start blob server")
|
||||
upnp = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
blob_manager: BlobFileManager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
blob_manager: BlobManager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
wallet: LbryWalletManager = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
peer_port = self.conf.tcp_port
|
||||
address = await wallet.get_unused_address()
|
||||
@@ -489,7 +490,7 @@ class UPnPComponent(Component):
|
||||
while True:
|
||||
if now:
|
||||
await self._maintain_redirects()
|
||||
await asyncio.sleep(360)
|
||||
await asyncio.sleep(360, loop=self.component_manager.loop)
|
||||
|
||||
async def _maintain_redirects(self):
|
||||
# setup the gateway if necessary
|
||||
@@ -512,7 +513,7 @@ class UPnPComponent(Component):
|
||||
pass
|
||||
|
||||
if external_ip == "0.0.0.0" or not external_ip:
|
||||
log.warning("unable to get external ip from UPnP, checking lbry.io fallback")
|
||||
log.warning("unable to get external ip from UPnP, checking lbry.com fallback")
|
||||
external_ip = await utils.get_external_ip()
|
||||
if self.external_ip and self.external_ip != external_ip:
|
||||
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
|
||||
@@ -532,7 +533,7 @@ class UPnPComponent(Component):
|
||||
self.upnp_redirects.update(upnp_redirects)
|
||||
except (asyncio.TimeoutError, UPnPError):
|
||||
self.upnp = None
|
||||
return self._maintain_redirects()
|
||||
return
|
||||
elif self.upnp: # check existing redirects are still active
|
||||
found = set()
|
||||
mappings = await self.upnp.get_redirects()
|
||||
|
||||
+1737
-1136
File diff suppressed because it is too large.
Load diff
@@ -1,27 +1,6 @@
|
||||
import aiohttp
|
||||
import logging
|
||||
from lbrynet.conf import Config
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
USER_AGENT = "AuthServiceProxy/0.1"
|
||||
TWISTED_SECURE_SESSION = "TWISTED_SECURE_SESSION"
|
||||
TWISTED_SESSION = "TWISTED_SESSION"
|
||||
LBRY_SECRET = "LBRY_SECRET"
|
||||
HTTP_TIMEOUT = 30
|
||||
from lbrynet.extras.cli import execute_command
|
||||
|
||||
|
||||
class JSONRPCException(Exception):
|
||||
def __init__(self, rpc_error):
|
||||
super().__init__()
|
||||
self.error = rpc_error
|
||||
|
||||
|
||||
async def daemon_rpc(conf: Config, method: str, *args, **kwargs):
|
||||
async with aiohttp.ClientSession() as session:
|
||||
message = {'method': method, 'params': [args, kwargs]}
|
||||
async with session.get(conf.api_connection_url, json=message) as resp:
|
||||
data = await resp.json()
|
||||
if 'result' in data:
|
||||
return data['result']
|
||||
elif 'error' in data:
|
||||
raise JSONRPCException(data['error'])
|
||||
def daemon_rpc(conf: Config, method: str, **kwargs):
|
||||
return execute_command(conf, method, kwargs, callback=lambda data: data)
|
||||
@@ -0,0 +1,51 @@
|
||||
import logging
|
||||
|
||||
import aiohttp
|
||||
import hashlib
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
from cryptography.hazmat.primitives import hashes
|
||||
from cryptography.hazmat.primitives.asymmetric import padding
|
||||
from cryptography.hazmat.primitives.asymmetric import utils
|
||||
from cryptography.hazmat.primitives.asymmetric.rsa import generate_private_key
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def sign_comment(**kwargs):
|
||||
private_key = generate_private_key(
|
||||
public_exponent=65537,
|
||||
key_size=4096,
|
||||
backend=default_backend()
|
||||
)
|
||||
chosen_hash = hashes.SHA256()
|
||||
hasher = hashes.Hash(chosen_hash, default_backend())
|
||||
value_to_hash = b':'.join(bytes(v, 'utf-8') for v in kwargs.values() if type(v) is str)
|
||||
hasher.update(value_to_hash)
|
||||
digest = hasher.finalize()
|
||||
signature = private_key.sign(
|
||||
digest,
|
||||
padding.PSS(
|
||||
mgf=padding.MGF1(hashes.SHA256()),
|
||||
salt_length=padding.PSS.MAX_LENGTH
|
||||
),
|
||||
utils.Prehashed(chosen_hash)
|
||||
)
|
||||
m = hashlib.sha3_256()
|
||||
m.update(signature)
|
||||
return m.hexdigest()
|
||||
|
||||
|
||||
def rpc_body(method: str, rpc_id: any, **params) -> dict:
|
||||
return {'jsonrpc': '2.0', 'id': rpc_id, 'method': method, 'params': {**params}}
|
||||
|
||||
|
||||
async def jsonrpc_post(url: str, method: str, **params) -> any:
|
||||
json_body = {'jsonrpc': '2.0', 'id': None, 'method': method, 'params': params}
|
||||
headers = {'Content-Type': 'application/json'}
|
||||
async with aiohttp.request('POST', url, json=json_body, headers=headers) as response:
|
||||
try:
|
||||
result = await response.json()
|
||||
return result['result'] if 'result' in result else result
|
||||
except aiohttp.client.ContentTypeError as cte:
|
||||
log.exception('Unable to decode respose from server: %s', cte)
|
||||
return await response.text()
|
||||
@@ -2,6 +2,7 @@ import asyncio
|
||||
import time
|
||||
import logging
|
||||
import json
|
||||
from decimal import Decimal
|
||||
|
||||
from lbrynet.error import InvalidExchangeRateResponse, CurrencyConversionError
|
||||
from lbrynet.utils import aiohttp_request
|
||||
@@ -122,8 +123,8 @@ class LBRYioFeed(MarketFeed):
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
"BTCLBC",
|
||||
"lbry.io",
|
||||
"https://api.lbry.io/lbc/exchange_rate",
|
||||
"lbry.com",
|
||||
"https://api.lbry.com/lbc/exchange_rate",
|
||||
{},
|
||||
0.0,
|
||||
)
|
||||
@@ -139,8 +140,8 @@ class LBRYioBTCFeed(MarketFeed):
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
"USDBTC",
|
||||
"lbry.io",
|
||||
"https://api.lbry.io/lbc/exchange_rate",
|
||||
"lbry.com",
|
||||
"https://api.lbry.com/lbc/exchange_rate",
|
||||
{},
|
||||
0.0,
|
||||
)
|
||||
@@ -226,12 +227,12 @@ class ExchangeRateManager:
|
||||
for market in self.market_feeds:
|
||||
if (market.rate_is_initialized() and market.is_online() and
|
||||
market.rate.currency_pair == (from_currency, to_currency)):
|
||||
return amount * market.rate.spot
|
||||
return amount * Decimal(market.rate.spot)
|
||||
for market in self.market_feeds:
|
||||
if (market.rate_is_initialized() and market.is_online() and
|
||||
market.rate.currency_pair[0] == from_currency):
|
||||
return self.convert_currency(
|
||||
market.rate.currency_pair[1], to_currency, amount * market.rate.spot)
|
||||
market.rate.currency_pair[1], to_currency, amount * Decimal(market.rate.spot))
|
||||
raise CurrencyConversionError(
|
||||
f'Unable to convert {amount} from {from_currency} to {to_currency}')
|
||||
|
||||
|
||||
@@ -1,28 +1,130 @@
|
||||
import os
|
||||
import logging
|
||||
from decimal import Decimal
|
||||
from binascii import hexlify
|
||||
from binascii import hexlify, unhexlify
|
||||
from datetime import datetime
|
||||
from json import JSONEncoder
|
||||
from ecdsa import BadSignatureError
|
||||
from lbrynet.extras.wallet import MainNetLedger
|
||||
from lbrynet.extras.wallet.transaction import Transaction, Output
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc
|
||||
|
||||
from google.protobuf.message import DecodeError
|
||||
|
||||
from lbrynet.schema.claim import Claim
|
||||
from lbrynet.wallet.ledger import MainNetLedger, Account
|
||||
from lbrynet.wallet.transaction import Transaction, Output
|
||||
from lbrynet.wallet.dewies import dewies_to_lbc
|
||||
from lbrynet.stream.managed_stream import ManagedStream
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def encode_txo_doc():
|
||||
return {
|
||||
'txid': "hash of transaction in hex",
|
||||
'nout': "position in the transaction",
|
||||
'height': "block where transaction was recorded",
|
||||
'amount': "value of the txo as a decimal",
|
||||
'address': "address of who can spend the txo",
|
||||
'confirmations': "number of confirmed blocks",
|
||||
'is_change': "payment to change address, only available when it can be determined",
|
||||
'is_mine': "payment to one of your accounts, only available when it can be determined",
|
||||
'type': "one of 'claim', 'support' or 'payment'",
|
||||
'name': "when type is 'claim' or 'support', this is the claim name",
|
||||
'claim_id': "when type is 'claim' or 'support', this is the claim id",
|
||||
'claim_op': "when type is 'claim', this determines if it is 'create' or 'update'",
|
||||
'value': "when type is 'claim' or 'support' with payload, this is the decoded protobuf payload",
|
||||
'value_type': "determines the type of the 'value' field: 'channel', 'stream', etc",
|
||||
'protobuf': "hex encoded raw protobuf version of 'value' field",
|
||||
'permanent_url': "when type is 'claim' or 'support', this is the long permanent claim URL",
|
||||
'signing_channel': "for signed claims only, metadata of signing channel",
|
||||
'is_channel_signature_valid': "for signed claims only, whether signature is valid",
|
||||
}
|
||||
|
||||
|
||||
def encode_tx_doc():
|
||||
return {
|
||||
'txid': "hash of transaction in hex",
|
||||
'height': "block where transaction was recorded",
|
||||
'inputs': [encode_txo_doc()],
|
||||
'outputs': [encode_txo_doc()],
|
||||
'total_input': "sum of inputs as a decimal",
|
||||
'total_output': "sum of outputs, sans fee, as a decimal",
|
||||
'total_fee': "fee amount",
|
||||
'hex': "entire transaction encoded in hex",
|
||||
}
|
||||
|
||||
|
||||
def encode_account_doc():
|
||||
return {
|
||||
'id': 'account_id',
|
||||
'is_default': 'this account is used by default',
|
||||
'ledger': 'name of crypto currency and network',
|
||||
'name': 'optional account name',
|
||||
'seed': 'human friendly words from which account can be recreated',
|
||||
'encrypted': 'if account is encrypted',
|
||||
'private_key': 'extended private key',
|
||||
'public_key': 'extended public key',
|
||||
'address_generator': 'settings for generating addresses',
|
||||
'modified_on': 'date of last modification to account settings'
|
||||
}
|
||||
|
||||
|
||||
def encode_file_doc():
|
||||
return {
|
||||
'streaming_url': '(str) url to stream the file using range requests',
|
||||
'completed': '(bool) true if download is completed',
|
||||
'file_name': '(str) name of file',
|
||||
'download_directory': '(str) download directory',
|
||||
'points_paid': '(float) credit paid to download file',
|
||||
'stopped': '(bool) true if download is stopped',
|
||||
'stream_hash': '(str) stream hash of file',
|
||||
'stream_name': '(str) stream name',
|
||||
'suggested_file_name': '(str) suggested file name',
|
||||
'sd_hash': '(str) sd hash of file',
|
||||
'download_path': '(str) download path of file',
|
||||
'mime_type': '(str) mime type of file',
|
||||
'key': '(str) key attached to file',
|
||||
'total_bytes_lower_bound': '(int) lower bound file size in bytes',
|
||||
'total_bytes': '(int) file upper bound size in bytes',
|
||||
'written_bytes': '(int) written size in bytes',
|
||||
'blobs_completed': '(int) number of fully downloaded blobs',
|
||||
'blobs_in_stream': '(int) total blobs on stream',
|
||||
'blobs_remaining': '(int) total blobs remaining to download',
|
||||
'status': '(str) downloader status',
|
||||
'claim_id': '(str) None if claim is not found else the claim id',
|
||||
'txid': '(str) None if claim is not found else the transaction id',
|
||||
'nout': '(int) None if claim is not found else the transaction output index',
|
||||
'outpoint': '(str) None if claim is not found else the tx and output',
|
||||
'claim': '(Claim) None if claim is not found',
|
||||
'metadata': '(dict) None if claim is not found else the claim metadata',
|
||||
'protobuf': '(str) hex encoded Stream protobuf',
|
||||
'channel_claim_id': '(str) None if claim is not found or not signed',
|
||||
'channel_name': '(str) None if claim is not found or not signed',
|
||||
'claim_name': '(str) None if claim is not found else the claim name',
|
||||
'content_fee': '(Transaction) None if content claim did not have a fee',
|
||||
'height': '(int) height of the claim',
|
||||
'confirmations': '(int) number of blocks since the claim was confirmed',
|
||||
'timestamp': '(int) timestamp of the block containing the claim'
|
||||
}
|
||||
|
||||
|
||||
class JSONResponseEncoder(JSONEncoder):
|
||||
|
||||
def __init__(self, *args, ledger: MainNetLedger, **kwargs):
|
||||
def __init__(self, *args, ledger: MainNetLedger, include_protobuf=False, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.ledger = ledger
|
||||
self.include_protobuf = include_protobuf
|
||||
|
||||
def default(self, obj): # pylint: disable=method-hidden
|
||||
if isinstance(obj, Account):
|
||||
return self.encode_account(obj)
|
||||
if isinstance(obj, ManagedStream):
|
||||
return self.encode_file(obj)
|
||||
if isinstance(obj, Transaction):
|
||||
return self.encode_transaction(obj)
|
||||
if isinstance(obj, Output):
|
||||
return self.encode_output(obj)
|
||||
if isinstance(obj, Claim):
|
||||
return self.encode_claim(obj)
|
||||
if isinstance(obj, datetime):
|
||||
return obj.strftime("%Y%m%dT%H:%M:%S")
|
||||
if isinstance(obj, Decimal):
|
||||
@@ -43,62 +145,154 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
'hex': hexlify(tx.raw).decode(),
|
||||
}
|
||||
|
||||
def encode_output(self, txo):
|
||||
def encode_output(self, txo, check_signature=True):
|
||||
tx_height = txo.tx_ref.height
|
||||
best_height = self.ledger.headers.height
|
||||
output = {
|
||||
'txid': txo.tx_ref.id,
|
||||
'nout': txo.position,
|
||||
'height': tx_height,
|
||||
'amount': dewies_to_lbc(txo.amount),
|
||||
'address': txo.get_address(self.ledger),
|
||||
'height': tx_height,
|
||||
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height
|
||||
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
|
||||
'timestamp': self.ledger.headers[tx_height]['timestamp'] if tx_height > 0 else None
|
||||
}
|
||||
if txo.is_change is not None:
|
||||
output['is_change'] = txo.is_change
|
||||
if txo.is_my_account is not None:
|
||||
output['is_mine'] = txo.is_my_account
|
||||
|
||||
if txo.script.is_claim_name:
|
||||
output['type'] = 'claim'
|
||||
output['claim_op'] = 'create'
|
||||
elif txo.script.is_update_claim:
|
||||
output['type'] = 'claim'
|
||||
output['claim_op'] = 'update'
|
||||
elif txo.script.is_support_claim:
|
||||
output['type'] = 'support'
|
||||
else:
|
||||
output['type'] = 'payment'
|
||||
|
||||
if txo.script.is_claim_involved:
|
||||
output.update({
|
||||
'name': txo.claim_name,
|
||||
'normalized_name': txo.normalized_name,
|
||||
'claim_id': txo.claim_id,
|
||||
'permanent_url': txo.permanent_url,
|
||||
'meta': self.encode_claim_meta(txo.meta)
|
||||
})
|
||||
|
||||
if 'short_url' in output['meta']:
|
||||
output['short_url'] = output['meta'].pop('short_url')
|
||||
if 'canonical_url' in output['meta']:
|
||||
output['canonical_url'] = output['meta'].pop('canonical_url')
|
||||
if txo.script.is_claim_name or txo.script.is_update_claim:
|
||||
claim = txo.claim
|
||||
output['value'] = claim.claim_dict
|
||||
if claim.has_signature:
|
||||
output['valid_signature'] = None
|
||||
if txo.channel is not None:
|
||||
output['channel_name'] = txo.channel.claim_name
|
||||
try:
|
||||
output['valid_signature'] = claim.validate_signature(
|
||||
txo.get_address(self.ledger), txo.channel.claim, name=txo.claim_name
|
||||
)
|
||||
except BadSignatureError:
|
||||
output['valid_signature'] = False
|
||||
except ValueError:
|
||||
log.exception(
|
||||
'txo.id: %s, txo.channel.id:%s, output: %s',
|
||||
txo.id, txo.channel.id, output
|
||||
)
|
||||
output['valid_signature'] = False
|
||||
|
||||
if txo.script.is_claim_name:
|
||||
output['type'] = 'claim'
|
||||
elif txo.script.is_update_claim:
|
||||
output['type'] = 'update'
|
||||
elif txo.script.is_support_claim:
|
||||
output['type'] = 'support'
|
||||
else:
|
||||
output['type'] = 'basic'
|
||||
|
||||
try:
|
||||
output['value'] = txo.claim
|
||||
output['value_type'] = txo.claim.claim_type
|
||||
if self.include_protobuf:
|
||||
output['protobuf'] = hexlify(txo.claim.to_bytes())
|
||||
if check_signature and txo.claim.is_signed:
|
||||
if txo.channel is not None:
|
||||
output['signing_channel'] = self.encode_output(txo.channel)
|
||||
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
|
||||
else:
|
||||
output['signing_channel'] = {'channel_id': txo.claim.signing_channel_id}
|
||||
output['is_channel_signature_valid'] = False
|
||||
except DecodeError:
|
||||
pass
|
||||
return output
|
||||
|
||||
def encode_claim_meta(self, meta):
|
||||
for key, value in meta.items():
|
||||
if key.endswith('_amount'):
|
||||
if isinstance(value, int):
|
||||
meta[key] = dewies_to_lbc(value)
|
||||
if meta.get('creation_height', 0) > 0:
|
||||
meta['creation_timestamp'] = self.ledger.headers[meta['creation_height']]['timestamp']
|
||||
return meta
|
||||
|
||||
def encode_input(self, txi):
|
||||
return self.encode_output(txi.txo_ref.txo) if txi.txo_ref.txo is not None else {
|
||||
return self.encode_output(txi.txo_ref.txo, False) if txi.txo_ref.txo is not None else {
|
||||
'txid': txi.txo_ref.tx_ref.id,
|
||||
'nout': txi.txo_ref.position
|
||||
}
|
||||
|
||||
def encode_account(self, account):
|
||||
result = account.to_dict()
|
||||
result['id'] = account.id
|
||||
result.pop('certificates', None)
|
||||
result['is_default'] = self.ledger.accounts[0] == account
|
||||
return result
|
||||
|
||||
def encode_file(self, managed_stream):
|
||||
full_path = managed_stream.full_path
|
||||
file_name = managed_stream.file_name
|
||||
download_directory = managed_stream.download_directory
|
||||
if managed_stream.full_path and managed_stream.output_file_exists:
|
||||
if managed_stream.written_bytes:
|
||||
written_bytes = managed_stream.written_bytes
|
||||
else:
|
||||
written_bytes = os.stat(managed_stream.full_path).st_size
|
||||
else:
|
||||
full_path = None
|
||||
file_name = None
|
||||
download_directory = None
|
||||
written_bytes = None
|
||||
|
||||
if managed_stream.stream_claim_info:
|
||||
tx_height = managed_stream.stream_claim_info.height
|
||||
best_height = self.ledger.headers.height
|
||||
timestamp = self.ledger.headers[tx_height]['timestamp'] if tx_height > 0 else None
|
||||
confirmations = None if not tx_height else ((best_height + 1) - tx_height if tx_height > 0 else tx_height)
|
||||
else:
|
||||
tx_height = None
|
||||
timestamp = None
|
||||
confirmations = None
|
||||
return {
|
||||
'streaming_url': f"http://{managed_stream.config.streaming_host}:{managed_stream.config.streaming_port}"
|
||||
f"/stream/{managed_stream.sd_hash}",
|
||||
'completed': (managed_stream.output_file_exists and (managed_stream.status in ('stopped', 'finished'))
|
||||
or not managed_stream.saving.is_set()) or all(
|
||||
managed_stream.blob_manager.is_blob_verified(b.blob_hash)
|
||||
for b in managed_stream.descriptor.blobs[:-1]),
|
||||
'file_name': file_name,
|
||||
'download_directory': download_directory,
|
||||
'points_paid': 0.0,
|
||||
'stopped': not managed_stream.running,
|
||||
'stream_hash': managed_stream.stream_hash,
|
||||
'stream_name': managed_stream.descriptor.stream_name,
|
||||
'suggested_file_name': managed_stream.descriptor.suggested_file_name,
|
||||
'sd_hash': managed_stream.descriptor.sd_hash,
|
||||
'download_path': full_path,
|
||||
'mime_type': managed_stream.mime_type,
|
||||
'key': managed_stream.descriptor.key,
|
||||
'total_bytes_lower_bound': managed_stream.descriptor.lower_bound_decrypted_length(),
|
||||
'total_bytes': managed_stream.descriptor.upper_bound_decrypted_length(),
|
||||
'written_bytes': written_bytes,
|
||||
'blobs_completed': managed_stream.blobs_completed,
|
||||
'blobs_in_stream': managed_stream.blobs_in_stream,
|
||||
'blobs_remaining': managed_stream.blobs_remaining,
|
||||
'status': managed_stream.status,
|
||||
'claim_id': managed_stream.claim_id,
|
||||
'txid': managed_stream.txid,
|
||||
'nout': managed_stream.nout,
|
||||
'outpoint': managed_stream.outpoint,
|
||||
'claim': managed_stream.claim,
|
||||
'metadata': managed_stream.metadata,
|
||||
'protobuf': managed_stream.metadata_protobuf,
|
||||
'channel_claim_id': managed_stream.channel_claim_id,
|
||||
'channel_name': managed_stream.channel_name,
|
||||
'claim_name': managed_stream.claim_name,
|
||||
'content_fee': managed_stream.content_fee,
|
||||
'height': tx_height,
|
||||
'confirmations': confirmations,
|
||||
'timestamp': timestamp
|
||||
}
|
||||
|
||||
def encode_claim(self, claim):
|
||||
encoded = getattr(claim, claim.claim_type).to_dict()
|
||||
if 'public_key' in encoded:
|
||||
encoded['public_key_id'] = self.ledger.public_key_to_address(
|
||||
unhexlify(encoded['public_key'])
|
||||
)
|
||||
return encoded
|
||||
@@ -22,6 +22,10 @@ def migrate_db(conf, start, end):
|
||||
from .migrate7to8 import do_migration
|
||||
elif current == 8:
|
||||
from .migrate8to9 import do_migration
|
||||
elif current == 9:
|
||||
from .migrate9to10 import do_migration
|
||||
elif current == 10:
|
||||
from .migrate10to11 import do_migration
|
||||
else:
|
||||
raise Exception("DB migration of version {} to {} is not available".format(current,
|
||||
current+1))
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
import sqlite3
|
||||
import os
|
||||
import binascii
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
current_columns = []
|
||||
for col_info in cursor.execute("pragma table_info('file');").fetchall():
|
||||
current_columns.append(col_info[1])
|
||||
if 'content_fee' in current_columns or 'saved_file' in current_columns:
|
||||
connection.close()
|
||||
print("already migrated")
|
||||
return
|
||||
|
||||
cursor.execute(
|
||||
"pragma foreign_keys=off;"
|
||||
)
|
||||
|
||||
cursor.execute("""
|
||||
create table if not exists new_file (
|
||||
stream_hash text primary key not null references stream,
|
||||
file_name text,
|
||||
download_directory text,
|
||||
blob_data_rate real not null,
|
||||
status text not null,
|
||||
saved_file integer not null,
|
||||
content_fee text
|
||||
);
|
||||
""")
|
||||
for (stream_hash, file_name, download_dir, data_rate, status) in cursor.execute("select * from file").fetchall():
|
||||
saved_file = 0
|
||||
if download_dir != '{stream}' and file_name != '{stream}':
|
||||
try:
|
||||
if os.path.isfile(os.path.join(binascii.unhexlify(download_dir).decode(),
|
||||
binascii.unhexlify(file_name).decode())):
|
||||
saved_file = 1
|
||||
else:
|
||||
download_dir, file_name = None, None
|
||||
except Exception:
|
||||
download_dir, file_name = None, None
|
||||
else:
|
||||
download_dir, file_name = None, None
|
||||
cursor.execute(
|
||||
"insert into new_file values (?, ?, ?, ?, ?, ?, NULL)",
|
||||
(stream_hash, file_name, download_dir, data_rate, status, saved_file)
|
||||
)
|
||||
cursor.execute("drop table file")
|
||||
cursor.execute("alter table new_file rename to file")
|
||||
connection.commit()
|
||||
connection.close()
|
||||
@@ -3,7 +3,7 @@ import os
|
||||
import json
|
||||
import logging
|
||||
from binascii import hexlify
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
from lbrynet.schema.claim import Claim
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -260,7 +260,7 @@ def do_migration(conf):
|
||||
(
|
||||
"%s:%i" % (claim_arg_tup[0], claim_arg_tup[1]), claim_arg_tup[2], claim_arg_tup[3],
|
||||
claim_arg_tup[7], claim_arg_tup[6], claim_arg_tup[8],
|
||||
smart_decode(claim_arg_tup[8]).certificate_id, claim_arg_tup[5], claim_arg_tup[4]
|
||||
Claim.from_bytes(claim_arg_tup[8]).signing_channel_id, claim_arg_tup[5], claim_arg_tup[4]
|
||||
)
|
||||
for sd_hash, claim_arg_tup in claim_queries.items() if claim_arg_tup
|
||||
] # sd_hash, (txid, nout, claim_id, name, sequence, address, height, amount, serialized)
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
query = "select stream_hash, sd_hash from main.stream"
|
||||
for stream_hash, sd_hash in cursor.execute(query).fetchall():
|
||||
head_blob_hash = cursor.execute(
|
||||
"select blob_hash from stream_blob where position = 0 and stream_hash = ?",
|
||||
(stream_hash,)
|
||||
).fetchone()
|
||||
if not head_blob_hash:
|
||||
continue
|
||||
cursor.execute("update blob set should_announce=1 where blob_hash in (?, ?)", (sd_hash, head_blob_hash[0],))
|
||||
connection.commit()
|
||||
connection.close()
|
||||
@@ -1,160 +0,0 @@
|
||||
import os
|
||||
|
||||
|
||||
types_map = {
|
||||
# http://www.iana.org/assignments/media-types
|
||||
'.a': 'application/octet-stream',
|
||||
'.ai': 'application/postscript',
|
||||
'.aif': 'audio/x-aiff',
|
||||
'.aifc': 'audio/x-aiff',
|
||||
'.aiff': 'audio/x-aiff',
|
||||
'.au': 'audio/basic',
|
||||
'.avi': 'video/x-msvideo',
|
||||
'.bat': 'text/plain',
|
||||
'.bcpio': 'application/x-bcpio',
|
||||
'.bin': 'application/octet-stream',
|
||||
'.bmp': 'image/bmp',
|
||||
'.c': 'text/plain',
|
||||
'.cdf': 'application/x-netcdf',
|
||||
'.cpio': 'application/x-cpio',
|
||||
'.csh': 'application/x-csh',
|
||||
'.css': 'text/css',
|
||||
'.csv': 'text/csv',
|
||||
'.dll': 'application/octet-stream',
|
||||
'.doc': 'application/msword',
|
||||
'.dot': 'application/msword',
|
||||
'.dvi': 'application/x-dvi',
|
||||
'.eml': 'message/rfc822',
|
||||
'.eps': 'application/postscript',
|
||||
'.epub': 'application/epub+zip',
|
||||
'.etx': 'text/x-setext',
|
||||
'.exe': 'application/octet-stream',
|
||||
'.gif': 'image/gif',
|
||||
'.gtar': 'application/x-gtar',
|
||||
'.h': 'text/plain',
|
||||
'.hdf': 'application/x-hdf',
|
||||
'.htm': 'text/html',
|
||||
'.html': 'text/html',
|
||||
'.ico': 'image/vnd.microsoft.icon',
|
||||
'.ief': 'image/ief',
|
||||
'.iges': 'model/iges',
|
||||
'.jpe': 'image/jpeg',
|
||||
'.jpeg': 'image/jpeg',
|
||||
'.jpg': 'image/jpeg',
|
||||
'.js': 'application/javascript',
|
||||
'.json': 'application/json',
|
||||
'.ksh': 'text/plain',
|
||||
'.latex': 'application/x-latex',
|
||||
'.m1v': 'video/mpeg',
|
||||
'.m3u': 'application/vnd.apple.mpegurl',
|
||||
'.m3u8': 'application/vnd.apple.mpegurl',
|
||||
'.man': 'application/x-troff-man',
|
||||
'.markdown': 'text/markdown',
|
||||
'.md': 'text/markdown',
|
||||
'.me': 'application/x-troff-me',
|
||||
'.mht': 'message/rfc822',
|
||||
'.mhtml': 'message/rfc822',
|
||||
'.mif': 'application/x-mif',
|
||||
'.mov': 'video/quicktime',
|
||||
'.movie': 'video/x-sgi-movie',
|
||||
'.mp2': 'audio/mpeg',
|
||||
'.mp3': 'audio/mpeg',
|
||||
'.mp4': 'video/mp4',
|
||||
'.mpa': 'video/mpeg',
|
||||
'.mpe': 'video/mpeg',
|
||||
'.mpeg': 'video/mpeg',
|
||||
'.mpg': 'video/mpeg',
|
||||
'.ms': 'application/x-troff-ms',
|
||||
'.nc': 'application/x-netcdf',
|
||||
'.nws': 'message/rfc822',
|
||||
'.o': 'application/octet-stream',
|
||||
'.obj': 'application/octet-stream',
|
||||
'.oda': 'application/oda',
|
||||
'.p12': 'application/x-pkcs12',
|
||||
'.p7c': 'application/pkcs7-mime',
|
||||
'.pbm': 'image/x-portable-bitmap',
|
||||
'.pdf': 'application/pdf',
|
||||
'.pfx': 'application/x-pkcs12',
|
||||
'.pgm': 'image/x-portable-graymap',
|
||||
'.pl': 'text/plain',
|
||||
'.png': 'image/png',
|
||||
'.pnm': 'image/x-portable-anymap',
|
||||
'.pot': 'application/vnd.ms-powerpoint',
|
||||
'.ppa': 'application/vnd.ms-powerpoint',
|
||||
'.ppm': 'image/x-portable-pixmap',
|
||||
'.pps': 'application/vnd.ms-powerpoint',
|
||||
'.ppt': 'application/vnd.ms-powerpoint',
|
||||
'.ps': 'application/postscript',
|
||||
'.pwz': 'application/vnd.ms-powerpoint',
|
||||
'.py': 'text/x-python',
|
||||
'.pyc': 'application/x-python-code',
|
||||
'.pyo': 'application/x-python-code',
|
||||
'.qt': 'video/quicktime',
|
||||
'.ra': 'audio/x-pn-realaudio',
|
||||
'.ram': 'application/x-pn-realaudio',
|
||||
'.ras': 'image/x-cmu-raster',
|
||||
'.rdf': 'application/xml',
|
||||
'.rgb': 'image/x-rgb',
|
||||
'.roff': 'application/x-troff',
|
||||
'.rtx': 'text/richtext',
|
||||
'.sgm': 'text/x-sgml',
|
||||
'.sgml': 'text/x-sgml',
|
||||
'.sh': 'application/x-sh',
|
||||
'.shar': 'application/x-shar',
|
||||
'.snd': 'audio/basic',
|
||||
'.so': 'application/octet-stream',
|
||||
'.src': 'application/x-wais-source',
|
||||
'.stl': 'model/stl',
|
||||
'.sv4cpio': 'application/x-sv4cpio',
|
||||
'.sv4crc': 'application/x-sv4crc',
|
||||
'.svg': 'image/svg+xml',
|
||||
'.swf': 'application/x-shockwave-flash',
|
||||
'.t': 'application/x-troff',
|
||||
'.tar': 'application/x-tar',
|
||||
'.tcl': 'application/x-tcl',
|
||||
'.tex': 'application/x-tex',
|
||||
'.texi': 'application/x-texinfo',
|
||||
'.texinfo': 'application/x-texinfo',
|
||||
'.tif': 'image/tiff',
|
||||
'.tiff': 'image/tiff',
|
||||
'.tr': 'application/x-troff',
|
||||
'.tsv': 'text/tab-separated-values',
|
||||
'.txt': 'text/plain',
|
||||
'.ustar': 'application/x-ustar',
|
||||
'.vcf': 'text/x-vcard',
|
||||
'.wav': 'audio/x-wav',
|
||||
'.webm': 'video/webm',
|
||||
'.wiz': 'application/msword',
|
||||
'.wsdl': 'application/xml',
|
||||
'.xbm': 'image/x-xbitmap',
|
||||
'.xlb': 'application/vnd.ms-excel',
|
||||
'.xls': 'application/vnd.ms-excel',
|
||||
'.xml': 'text/xml',
|
||||
'.xpdl': 'application/xml',
|
||||
'.xpm': 'image/x-xpixmap',
|
||||
'.xsl': 'application/xml',
|
||||
'.xwd': 'image/x-xwindowdump',
|
||||
'.zip': 'application/zip',
|
||||
|
||||
# These are non-standard types, commonly found in the wild.
|
||||
'.mid': 'audio/midi',
|
||||
'.midi': 'audio/midi',
|
||||
'.pct': 'image/pict',
|
||||
'.pic': 'image/pict',
|
||||
'.pict': 'image/pict',
|
||||
'.rtf': 'application/rtf',
|
||||
'.xul': 'text/xul',
|
||||
'.m4v': 'video/m4v',
|
||||
|
||||
# microsoft is special and has its own 'standard'
|
||||
# https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
|
||||
'.wmv': 'video/x-ms-wmv'
|
||||
}
|
||||
|
||||
|
||||
def guess_media_type(path):
|
||||
_, ext = os.path.splitext(path)
|
||||
return types_map.get(
|
||||
ext.strip().lower(),
|
||||
'application/octet-stream' if not (ext and ext[1:].strip()) else f'application/x-ext-{ext[1:].strip().lower()}'
|
||||
)
|
||||
@@ -7,9 +7,9 @@ import binascii
|
||||
import time
|
||||
from torba.client.basedatabase import SQLiteMixin
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
from lbrynet.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
||||
from lbrynet.wallet.transaction import Transaction
|
||||
from lbrynet.schema.claim import Claim
|
||||
from lbrynet.dht.constants import data_expiration
|
||||
from lbrynet.blob.blob_info import BlobInfo
|
||||
|
||||
@@ -39,7 +39,9 @@ class StoredStreamClaim:
|
||||
self.claim_name = name
|
||||
self.amount = amount
|
||||
self.height = height
|
||||
self.claim: typing.Optional[ClaimDict] = None if not serialized else smart_decode(serialized)
|
||||
self.claim: typing.Optional[Claim] = None if not serialized else Claim.from_bytes(
|
||||
binascii.unhexlify(serialized)
|
||||
)
|
||||
self.claim_address = address
|
||||
self.claim_sequence = claim_sequence
|
||||
self.channel_claim_id = channel_claim_id
|
||||
@@ -53,10 +55,6 @@ class StoredStreamClaim:
|
||||
def nout(self) -> typing.Optional[int]:
|
||||
return None if not self.outpoint else int(self.outpoint.split(":")[1])
|
||||
|
||||
@property
|
||||
def metadata(self) -> typing.Optional[typing.Dict]:
|
||||
return None if not self.claim else self.claim.claim_dict['stream']['metadata']
|
||||
|
||||
def as_dict(self) -> typing.Dict:
|
||||
return {
|
||||
"name": self.claim_name,
|
||||
@@ -117,8 +115,8 @@ def _batched_select(transaction, query, parameters, batch_size=900):
|
||||
def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
|
||||
files = []
|
||||
signed_claims = {}
|
||||
for (rowid, stream_hash, file_name, download_dir, data_rate, status, _, sd_hash, stream_key,
|
||||
stream_name, suggested_file_name, *claim_args) in _batched_select(
|
||||
for (rowid, stream_hash, file_name, download_dir, data_rate, status, saved_file, raw_content_fee, _,
|
||||
sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in _batched_select(
|
||||
transaction, "select file.rowid, file.*, stream.*, c.* "
|
||||
"from file inner join stream on file.stream_hash=stream.stream_hash "
|
||||
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
|
||||
@@ -144,7 +142,11 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
|
||||
"key": stream_key,
|
||||
"stream_name": stream_name, # hex
|
||||
"suggested_file_name": suggested_file_name, # hex
|
||||
"claim": claim
|
||||
"claim": claim,
|
||||
"saved_file": bool(saved_file),
|
||||
"content_fee": None if not raw_content_fee else Transaction(
|
||||
binascii.unhexlify(raw_content_fee)
|
||||
)
|
||||
}
|
||||
)
|
||||
for claim_name, claim_id in _batched_select(
|
||||
@@ -156,21 +158,12 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
|
||||
|
||||
|
||||
def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'):
|
||||
# add the head blob and set it to be announced
|
||||
transaction.execute(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
sd_blob.blob_hash, sd_blob.length, 0, 1, "pending", 0, 0,
|
||||
descriptor.blobs[0].blob_hash, descriptor.blobs[0].length, 0, 1, "pending", 0, 0
|
||||
)
|
||||
# add all blobs, except the last one, which is empty
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
[(blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
|
||||
for blob in (descriptor.blobs[:-1] if len(descriptor.blobs) > 1 else descriptor.blobs) + [sd_blob]]
|
||||
)
|
||||
# add the rest of the blobs with announcement off
|
||||
if len(descriptor.blobs) > 2:
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
[(blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
|
||||
for blob in descriptor.blobs[1:-1]]
|
||||
)
|
||||
# associate the blobs to the stream
|
||||
transaction.execute("insert or ignore into stream values (?, ?, ?, ?, ?)",
|
||||
(descriptor.stream_hash, sd_blob.blob_hash, descriptor.key,
|
||||
@@ -182,6 +175,11 @@ def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descripto
|
||||
[(descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
|
||||
for blob in descriptor.blobs]
|
||||
)
|
||||
# ensure should_announce is set regardless if insert was ignored
|
||||
transaction.execute(
|
||||
"update blob set should_announce=1 where blob_hash in (?, ?)",
|
||||
(sd_blob.blob_hash, descriptor.blobs[0].blob_hash,)
|
||||
)
|
||||
|
||||
|
||||
def delete_stream(transaction: sqlite3.Connection, descriptor: 'StreamDescriptor'):
|
||||
@@ -194,13 +192,21 @@ def delete_stream(transaction: sqlite3.Connection, descriptor: 'StreamDescriptor
|
||||
transaction.executemany("delete from blob where blob_hash=?", blob_hashes)
|
||||
|
||||
|
||||
def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: str, download_directory: str,
|
||||
data_payment_rate: float, status: str) -> int:
|
||||
def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: typing.Optional[str],
|
||||
download_directory: typing.Optional[str], data_payment_rate: float, status: str,
|
||||
content_fee: typing.Optional[Transaction]) -> int:
|
||||
if not file_name and not download_directory:
|
||||
encoded_file_name, encoded_download_dir = None, None
|
||||
else:
|
||||
encoded_file_name = binascii.hexlify(file_name.encode()).decode()
|
||||
encoded_download_dir = binascii.hexlify(download_directory.encode()).decode()
|
||||
transaction.execute(
|
||||
"insert or replace into file values (?, ?, ?, ?, ?)",
|
||||
(stream_hash, binascii.hexlify(file_name.encode()).decode(),
|
||||
binascii.hexlify(download_directory.encode()).decode(), data_payment_rate, status)
|
||||
"insert or replace into file values (?, ?, ?, ?, ?, ?, ?)",
|
||||
(stream_hash, encoded_file_name, encoded_download_dir, data_payment_rate, status,
|
||||
1 if (file_name and download_directory and os.path.isfile(os.path.join(download_directory, file_name))) else 0,
|
||||
None if not content_fee else binascii.hexlify(content_fee.raw).decode())
|
||||
)
|
||||
|
||||
return transaction.execute("select rowid from file where stream_hash=?", (stream_hash, )).fetchone()[0]
|
||||
|
||||
|
||||
@@ -249,10 +255,12 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
create table if not exists file (
|
||||
stream_hash text primary key not null references stream,
|
||||
file_name text not null,
|
||||
download_directory text not null,
|
||||
file_name text,
|
||||
download_directory text,
|
||||
blob_data_rate real not null,
|
||||
status text not null
|
||||
status text not null,
|
||||
saved_file integer not null,
|
||||
content_fee text
|
||||
);
|
||||
|
||||
create table if not exists content_claim (
|
||||
@@ -295,27 +303,28 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
# # # # # # # # # blob functions # # # # # # # # #
|
||||
|
||||
def add_completed_blob(self, blob_hash: str, length: int):
|
||||
def _add_blob(transaction: sqlite3.Connection):
|
||||
transaction.execute(
|
||||
async def add_blobs(self, *blob_hashes_and_lengths: typing.Tuple[str, int], finished=False):
|
||||
def _add_blobs(transaction: sqlite3.Connection):
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
(blob_hash, length, 0, 0, "pending", 0, 0)
|
||||
[
|
||||
(blob_hash, length, 0, 0, "pending" if not finished else "finished", 0, 0)
|
||||
for blob_hash, length in blob_hashes_and_lengths
|
||||
]
|
||||
)
|
||||
transaction.execute(
|
||||
"update blob set status='finished' where blob.blob_hash=?", (blob_hash, )
|
||||
)
|
||||
return self.db.run(_add_blob)
|
||||
if finished:
|
||||
transaction.executemany(
|
||||
"update blob set status='finished' where blob.blob_hash=?", [
|
||||
(blob_hash, ) for blob_hash, _ in blob_hashes_and_lengths
|
||||
]
|
||||
)
|
||||
return await self.db.run(_add_blobs)
|
||||
|
||||
def get_blob_status(self, blob_hash: str):
|
||||
return self.run_and_return_one_or_none(
|
||||
"select status from blob where blob_hash=?", blob_hash
|
||||
)
|
||||
|
||||
def add_known_blob(self, blob_hash: str, length: int):
|
||||
return self.db.execute(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)", (blob_hash, length, 0, 0, "pending", 0, 0)
|
||||
)
|
||||
|
||||
def should_announce(self, blob_hash: str):
|
||||
return self.run_and_return_one_or_none(
|
||||
"select should_announce from blob where blob_hash=?", blob_hash
|
||||
@@ -418,6 +427,26 @@ class SQLiteStorage(SQLiteMixin):
|
||||
}
|
||||
return self.db.run(_sync_blobs)
|
||||
|
||||
def sync_files_to_blobs(self):
|
||||
def _sync_blobs(transaction: sqlite3.Connection):
|
||||
transaction.executemany(
|
||||
"update file set status='stopped' where stream_hash=?",
|
||||
transaction.execute(
|
||||
"select distinct sb.stream_hash from stream_blob sb "
|
||||
"inner join blob b on b.blob_hash=sb.blob_hash and b.status=='pending'"
|
||||
).fetchall()
|
||||
)
|
||||
return self.db.run(_sync_blobs)
|
||||
|
||||
def set_files_as_streaming(self, stream_hashes: typing.List[str]):
|
||||
def _set_streaming(transaction: sqlite3.Connection):
|
||||
transaction.executemany(
|
||||
"update file set file_name=null, download_directory=null where stream_hash=?",
|
||||
[(stream_hash, ) for stream_hash in stream_hashes]
|
||||
)
|
||||
|
||||
return self.db.run(_set_streaming)
|
||||
|
||||
# # # # # # # # # stream functions # # # # # # # # #
|
||||
|
||||
async def stream_exists(self, sd_hash: str) -> bool:
|
||||
@@ -480,20 +509,53 @@ class SQLiteStorage(SQLiteMixin):
|
||||
"select stream_hash from stream where sd_hash = ?", sd_blob_hash
|
||||
)
|
||||
|
||||
def get_stream_info_for_sd_hash(self, sd_blob_hash):
|
||||
return self.run_and_return_one_or_none(
|
||||
"select stream_hash, stream_name, suggested_filename, stream_key from stream where sd_hash = ?",
|
||||
sd_blob_hash
|
||||
)
|
||||
|
||||
def delete_stream(self, descriptor: 'StreamDescriptor'):
|
||||
return self.db.run_with_foreign_keys_disabled(delete_stream, descriptor)
|
||||
|
||||
# # # # # # # # # file stuff # # # # # # # # #
|
||||
|
||||
def save_downloaded_file(self, stream_hash, file_name, download_directory,
|
||||
data_payment_rate) -> typing.Awaitable[int]:
|
||||
def save_downloaded_file(self, stream_hash: str, file_name: typing.Optional[str],
|
||||
download_directory: typing.Optional[str], data_payment_rate: float,
|
||||
content_fee: typing.Optional[Transaction] = None) -> typing.Awaitable[int]:
|
||||
return self.save_published_file(
|
||||
stream_hash, file_name, download_directory, data_payment_rate, status="running"
|
||||
stream_hash, file_name, download_directory, data_payment_rate, status="running",
|
||||
content_fee=content_fee
|
||||
)
|
||||
|
||||
def save_published_file(self, stream_hash: str, file_name: str, download_directory: str, data_payment_rate: float,
|
||||
status="finished") -> typing.Awaitable[int]:
|
||||
return self.db.run(store_file, stream_hash, file_name, download_directory, data_payment_rate, status)
|
||||
def save_published_file(self, stream_hash: str, file_name: typing.Optional[str],
|
||||
download_directory: typing.Optional[str], data_payment_rate: float,
|
||||
status: str = "finished",
|
||||
content_fee: typing.Optional[Transaction] = None) -> typing.Awaitable[int]:
|
||||
return self.db.run(store_file, stream_hash, file_name, download_directory, data_payment_rate, status,
|
||||
content_fee)
|
||||
|
||||
async def update_manually_removed_files_since_last_run(self):
|
||||
"""
|
||||
Update files that have been removed from the downloads directory since the last run
|
||||
"""
|
||||
def update_manually_removed_files(transaction: sqlite3.Connection):
|
||||
removed = []
|
||||
for (stream_hash, download_directory, file_name) in transaction.execute(
|
||||
"select stream_hash, download_directory, file_name from file where saved_file=1"
|
||||
).fetchall():
|
||||
if download_directory and file_name and os.path.isfile(
|
||||
os.path.join(binascii.unhexlify(download_directory).decode(),
|
||||
binascii.unhexlify(file_name).decode())):
|
||||
continue
|
||||
else:
|
||||
removed.append((stream_hash,))
|
||||
if removed:
|
||||
transaction.executemany(
|
||||
"update file set file_name=null, download_directory=null, saved_file=0 where stream_hash=?",
|
||||
removed
|
||||
)
|
||||
return await self.db.run(update_manually_removed_files)
|
||||
|
||||
def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]:
|
||||
return self.db.run(get_all_lbry_files)
|
||||
@@ -502,24 +564,45 @@ class SQLiteStorage(SQLiteMixin):
|
||||
log.debug("update file status %s -> %s", stream_hash, new_status)
|
||||
return self.db.execute("update file set status=? where stream_hash=?", (new_status, stream_hash))
|
||||
|
||||
def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: str, file_name: str):
|
||||
return self.db.execute("update file set download_directory=?, file_name=? where stream_hash=?", (
|
||||
binascii.hexlify(download_dir.encode()).decode(), binascii.hexlify(file_name.encode()).decode(),
|
||||
stream_hash
|
||||
async def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: typing.Optional[str],
|
||||
file_name: typing.Optional[str]):
|
||||
if not file_name or not download_dir:
|
||||
encoded_file_name, encoded_download_dir = None, None
|
||||
else:
|
||||
encoded_file_name = binascii.hexlify(file_name.encode()).decode()
|
||||
encoded_download_dir = binascii.hexlify(download_dir.encode()).decode()
|
||||
return await self.db.execute("update file set download_directory=?, file_name=? where stream_hash=?", (
|
||||
encoded_download_dir, encoded_file_name, stream_hash,
|
||||
))
|
||||
|
||||
async def recover_streams(self, descriptors_and_sds: typing.List[typing.Tuple['StreamDescriptor', 'BlobFile']],
|
||||
async def save_content_fee(self, stream_hash: str, content_fee: Transaction):
|
||||
return await self.db.execute("update file set content_fee=? where stream_hash=?", (
|
||||
binascii.hexlify(content_fee.raw), stream_hash,
|
||||
))
|
||||
|
||||
async def set_saved_file(self, stream_hash: str):
|
||||
return await self.db.execute("update file set saved_file=1 where stream_hash=?", (
|
||||
stream_hash,
|
||||
))
|
||||
|
||||
async def clear_saved_file(self, stream_hash: str):
|
||||
return await self.db.execute("update file set saved_file=0 where stream_hash=?", (
|
||||
stream_hash,
|
||||
))
|
||||
|
||||
async def recover_streams(self, descriptors_and_sds: typing.List[typing.Tuple['StreamDescriptor', 'BlobFile',
|
||||
typing.Optional[Transaction]]],
|
||||
download_directory: str):
|
||||
def _recover(transaction: sqlite3.Connection):
|
||||
stream_hashes = [d.stream_hash for d, s in descriptors_and_sds]
|
||||
for descriptor, sd_blob in descriptors_and_sds:
|
||||
stream_hashes = [x[0].stream_hash for x in descriptors_and_sds]
|
||||
for descriptor, sd_blob, content_fee in descriptors_and_sds:
|
||||
content_claim = transaction.execute(
|
||||
"select * from content_claim where stream_hash=?", (descriptor.stream_hash, )
|
||||
).fetchone()
|
||||
delete_stream(transaction, descriptor) # this will also delete the content claim
|
||||
store_stream(transaction, sd_blob, descriptor)
|
||||
store_file(transaction, descriptor.stream_hash, os.path.basename(descriptor.suggested_file_name),
|
||||
download_directory, 0.0, 'stopped')
|
||||
download_directory, 0.0, 'stopped', content_fee=content_fee)
|
||||
if content_claim:
|
||||
transaction.execute("insert or ignore into content_claim values (?, ?)", content_claim)
|
||||
transaction.executemany(
|
||||
@@ -538,16 +621,18 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
# # # # # # # # # support functions # # # # # # # # #
|
||||
|
||||
def save_supports(self, claim_id, supports):
|
||||
def save_supports(self, claim_id_to_supports: dict):
|
||||
# TODO: add 'address' to support items returned for a claim from lbrycrdd and lbryum-server
|
||||
def _save_support(transaction):
|
||||
transaction.execute("delete from support where claim_id=?", (claim_id,))
|
||||
for support in supports:
|
||||
transaction.execute(
|
||||
"insert into support values (?, ?, ?, ?)",
|
||||
("%s:%i" % (support['txid'], support['nout']), claim_id, lbc_to_dewies(support['amount']),
|
||||
support.get('address', ""))
|
||||
)
|
||||
bind = "({})".format(','.join(['?'] * len(claim_id_to_supports)))
|
||||
transaction.execute(f"delete from support where claim_id in {bind}", list(claim_id_to_supports.keys()))
|
||||
for claim_id, supports in claim_id_to_supports.items():
|
||||
for support in supports:
|
||||
transaction.execute(
|
||||
"insert into support values (?, ?, ?, ?)",
|
||||
("%s:%i" % (support['txid'], support['nout']), claim_id, lbc_to_dewies(support['amount']),
|
||||
support.get('address', ""))
|
||||
)
|
||||
return self.db.run(_save_support)
|
||||
|
||||
def get_supports(self, *claim_ids):
|
||||
@@ -575,7 +660,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
# # # # # # # # # claim functions # # # # # # # # #
|
||||
|
||||
async def save_claims(self, claim_infos):
|
||||
support_callbacks = []
|
||||
claim_id_to_supports = {}
|
||||
update_file_callbacks = []
|
||||
|
||||
def _save_claims(transaction):
|
||||
@@ -588,19 +673,12 @@ class SQLiteStorage(SQLiteMixin):
|
||||
height = claim_info['height']
|
||||
address = claim_info['address']
|
||||
sequence = claim_info['claim_sequence']
|
||||
certificate_id = claim_info['value'].signing_channel_id
|
||||
try:
|
||||
certificate_id = claim_info['value'].get('publisherSignature', {}).get('certificateId')
|
||||
except AttributeError:
|
||||
certificate_id = None
|
||||
try:
|
||||
if claim_info['value'].get('stream', {}).get('source', {}).get('sourceType') == "lbry_sd_hash":
|
||||
source_hash = claim_info['value'].get('stream', {}).get('source', {}).get('source')
|
||||
else:
|
||||
source_hash = None
|
||||
except AttributeError:
|
||||
source_hash = claim_info['value'].stream.source.sd_hash
|
||||
except (AttributeError, ValueError):
|
||||
source_hash = None
|
||||
serialized = claim_info.get('hex') or binascii.hexlify(
|
||||
smart_decode(claim_info['value']).serialized).decode()
|
||||
serialized = binascii.hexlify(claim_info['value'].to_bytes())
|
||||
transaction.execute(
|
||||
"insert or replace into claim values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(outpoint, claim_id, name, amount, height, serialized, certificate_id, address, sequence)
|
||||
@@ -608,7 +686,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
# if this response doesn't have support info don't overwrite the existing
|
||||
# support info
|
||||
if 'supports' in claim_info:
|
||||
support_callbacks.append((claim_id, claim_info['supports']))
|
||||
claim_id_to_supports[claim_id] = claim_info['supports']
|
||||
if not source_hash:
|
||||
continue
|
||||
stream_hash = transaction.execute(
|
||||
@@ -620,12 +698,12 @@ class SQLiteStorage(SQLiteMixin):
|
||||
stream_hash = stream_hash[0]
|
||||
known_outpoint = transaction.execute(
|
||||
"select claim_outpoint from content_claim where stream_hash=?", (stream_hash,)
|
||||
)
|
||||
).fetchone()
|
||||
known_claim_id = transaction.execute(
|
||||
"select claim_id from claim "
|
||||
"inner join content_claim c3 ON claim.claim_outpoint=c3.claim_outpoint "
|
||||
"where c3.stream_hash=?", (stream_hash,)
|
||||
)
|
||||
).fetchone()
|
||||
if not known_claim_id:
|
||||
content_claims_to_update.append((stream_hash, outpoint))
|
||||
elif known_outpoint != outpoint:
|
||||
@@ -638,23 +716,20 @@ class SQLiteStorage(SQLiteMixin):
|
||||
await self.db.run(_save_claims)
|
||||
if update_file_callbacks:
|
||||
await asyncio.wait(update_file_callbacks)
|
||||
if support_callbacks:
|
||||
await asyncio.wait([
|
||||
self.save_supports(*args) for args in support_callbacks
|
||||
])
|
||||
if claim_id_to_supports:
|
||||
await self.save_supports(claim_id_to_supports)
|
||||
|
||||
def save_claims_for_resolve(self, claim_infos):
|
||||
to_save = []
|
||||
to_save = {}
|
||||
for info in claim_infos:
|
||||
if 'value' in info:
|
||||
if info['value']:
|
||||
to_save.append(info)
|
||||
to_save[info['claim_id']] = info
|
||||
else:
|
||||
if 'certificate' in info and info['certificate']['value']:
|
||||
to_save.append(info['certificate'])
|
||||
if 'claim' in info and info['claim']['value']:
|
||||
to_save.append(info['claim'])
|
||||
return self.save_claims(to_save)
|
||||
for key in ('certificate', 'claim'):
|
||||
if info.get(key, {}).get('value'):
|
||||
to_save[info[key]['claim_id']] = info[key]
|
||||
return self.save_claims(to_save.values())
|
||||
|
||||
@staticmethod
|
||||
def _save_content_claim(transaction, claim_outpoint, stream_hash):
|
||||
@@ -664,7 +739,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
).fetchone()
|
||||
if not claim_info:
|
||||
raise Exception("claim not found")
|
||||
new_claim_id, claim = claim_info[0], ClaimDict.deserialize(binascii.unhexlify(claim_info[1]))
|
||||
new_claim_id, claim = claim_info[0], Claim.from_bytes(binascii.unhexlify(claim_info[1]))
|
||||
|
||||
# certificate claims should not be in the content_claim table
|
||||
if not claim.is_stream:
|
||||
@@ -677,7 +752,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
if not known_sd_hash:
|
||||
raise Exception("stream not found")
|
||||
# check the claim contains the same sd hash
|
||||
if known_sd_hash[0].encode() != claim.source_hash:
|
||||
if known_sd_hash[0] != claim.stream.source.sd_hash:
|
||||
raise Exception("stream mismatch")
|
||||
|
||||
# if there is a current claim associated to the file, check that the new claim is an update to it
|
||||
|
||||
@@ -2,21 +2,24 @@ import platform
|
||||
import os
|
||||
import logging.handlers
|
||||
|
||||
from lbrynet.schema import __version__ as schema_version
|
||||
from lbrynet import build_type, __version__ as lbrynet_version
|
||||
from torba import __version__ as torba_version
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_platform() -> dict:
|
||||
os_system = platform.system()
|
||||
if os.environ and 'ANDROID_ARGUMENT' in os.environ:
|
||||
os_system = 'android'
|
||||
p = {
|
||||
"processor": platform.processor(),
|
||||
"python_version": platform.python_version(),
|
||||
"platform": platform.platform(),
|
||||
"os_release": platform.release(),
|
||||
"os_system": platform.system(),
|
||||
"os_system": os_system,
|
||||
"lbrynet_version": lbrynet_version,
|
||||
"lbryschema_version": schema_version,
|
||||
"torba_version": torba_version,
|
||||
"build": build_type.BUILD, # CI server sets this during build step
|
||||
}
|
||||
if p["os_system"] == "Linux":
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
__node_daemon__ = 'lbrycrdd'
|
||||
__node_cli__ = 'lbrycrd-cli'
|
||||
__node_bin__ = ''
|
||||
__node_url__ = (
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.12.4.0/lbrycrd-linux.zip'
|
||||
)
|
||||
__spvserver__ = 'lbrynet.extras.wallet.server.coin.LBCRegTest'
|
||||
|
||||
from lbrynet.extras.wallet.ledger import MainNetLedger, RegTestLedger
|
||||
from lbrynet.extras.wallet.manager import LbryWalletManager
|
||||
from lbrynet.extras.wallet.network import Network
|
||||
@@ -1,254 +0,0 @@
|
||||
import json
|
||||
import logging
|
||||
import binascii
|
||||
from hashlib import sha256
|
||||
|
||||
from lbrynet.schema.validator import validate_claim_id
|
||||
from torba.client.baseaccount import BaseAccount
|
||||
from torba.client.basetransaction import TXORef
|
||||
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.schema.signer import SECP256k1, get_signer
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def generate_certificate():
|
||||
secp256k1_private_key = get_signer(SECP256k1).generate().private_key.to_pem()
|
||||
return ClaimDict.generate_certificate(secp256k1_private_key, curve=SECP256k1), secp256k1_private_key
|
||||
|
||||
|
||||
class Account(BaseAccount):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.certificates = {}
|
||||
|
||||
@property
|
||||
def hash(self) -> bytes:
|
||||
h = sha256(json.dumps(self.to_dict(False)).encode())
|
||||
for cert in sorted(self.certificates.keys()):
|
||||
h.update(cert.encode())
|
||||
return h.digest()
|
||||
|
||||
def apply(self, d: dict):
|
||||
super().apply(d)
|
||||
self.certificates.update(d.get('certificates', {}))
|
||||
|
||||
def add_certificate_private_key(self, ref: TXORef, private_key):
|
||||
assert ref.id not in self.certificates, 'Trying to add a duplicate certificate.'
|
||||
self.certificates[ref.id] = private_key
|
||||
|
||||
def get_certificate_private_key(self, ref: TXORef):
|
||||
return self.certificates.get(ref.id)
|
||||
|
||||
async def maybe_migrate_certificates(self):
|
||||
if not self.certificates:
|
||||
return
|
||||
|
||||
addresses = {}
|
||||
results = {
|
||||
'total': 0,
|
||||
'not-a-claim-tx': 0,
|
||||
'migrate-success': 0,
|
||||
'migrate-failed': 0,
|
||||
'previous-success': 0,
|
||||
'previous-corrupted': 0
|
||||
}
|
||||
double_hex_encoded_to_pop = []
|
||||
|
||||
for maybe_claim_id in list(self.certificates):
|
||||
if ':' not in maybe_claim_id:
|
||||
try:
|
||||
validate_claim_id(maybe_claim_id)
|
||||
continue
|
||||
except Exception:
|
||||
try:
|
||||
maybe_claim_id_bytes = maybe_claim_id
|
||||
if isinstance(maybe_claim_id_bytes, str):
|
||||
maybe_claim_id_bytes = maybe_claim_id_bytes.encode()
|
||||
decoded_double_hex = binascii.unhexlify(maybe_claim_id_bytes).decode()
|
||||
validate_claim_id(decoded_double_hex)
|
||||
if decoded_double_hex in self.certificates:
|
||||
log.warning("don't know how to migrate certificate %s", decoded_double_hex)
|
||||
else:
|
||||
log.info("claim id was double hex encoded, fixing it")
|
||||
double_hex_encoded_to_pop.append((maybe_claim_id, decoded_double_hex))
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
for double_encoded_claim_id, correct_claim_id in double_hex_encoded_to_pop:
|
||||
self.certificates[correct_claim_id] = self.certificates.pop(double_encoded_claim_id)
|
||||
|
||||
for maybe_claim_id in list(self.certificates):
|
||||
results['total'] += 1
|
||||
if ':' not in maybe_claim_id:
|
||||
try:
|
||||
validate_claim_id(maybe_claim_id)
|
||||
except Exception as e:
|
||||
log.warning(
|
||||
"Failed to migrate claim '%s': %s",
|
||||
maybe_claim_id, str(e)
|
||||
)
|
||||
results['migrate-failed'] += 1
|
||||
continue
|
||||
claims = await self.ledger.network.get_claims_by_ids(maybe_claim_id)
|
||||
if maybe_claim_id not in claims:
|
||||
log.warning(
|
||||
"Failed to migrate claim '%s', server did not return any claim information.",
|
||||
maybe_claim_id
|
||||
)
|
||||
results['migrate-failed'] += 1
|
||||
continue
|
||||
claim = claims[maybe_claim_id]
|
||||
tx = None
|
||||
if claim:
|
||||
tx = await self.ledger.db.get_transaction(txid=claim['txid'])
|
||||
else:
|
||||
log.warning(maybe_claim_id)
|
||||
if tx is not None:
|
||||
txo = tx.outputs[claim['nout']]
|
||||
if not txo.script.is_claim_involved:
|
||||
results['not-a-claim-tx'] += 1
|
||||
raise ValueError(
|
||||
"Certificate with claim_id {} doesn't point to a valid transaction."
|
||||
.format(maybe_claim_id)
|
||||
)
|
||||
tx_nout = '{txid}:{nout}'.format(**claim)
|
||||
self.certificates[tx_nout] = self.certificates[maybe_claim_id]
|
||||
del self.certificates[maybe_claim_id]
|
||||
log.info(
|
||||
"Migrated certificate with claim_id '%s' ('%s') to a new look up key %s.",
|
||||
maybe_claim_id, txo.script.values['claim_name'], tx_nout
|
||||
)
|
||||
results['migrate-success'] += 1
|
||||
else:
|
||||
if claim:
|
||||
addresses.setdefault(claim['address'], 0)
|
||||
addresses[claim['address']] += 1
|
||||
log.warning(
|
||||
"Failed to migrate claim '%s', it's not associated with any of your addresses.",
|
||||
maybe_claim_id
|
||||
)
|
||||
else:
|
||||
log.warning(
|
||||
"Failed to migrate claim '%s', it appears abandoned.",
|
||||
maybe_claim_id
|
||||
)
|
||||
results['migrate-failed'] += 1
|
||||
else:
|
||||
try:
|
||||
txid, nout = maybe_claim_id.split(':')
|
||||
tx = await self.ledger.db.get_transaction(txid=txid)
|
||||
if not tx:
|
||||
log.warning(
|
||||
"Claim migration failed to find a transaction for outpoint %s", maybe_claim_id
|
||||
)
|
||||
results['previous-corrupted'] += 1
|
||||
continue
|
||||
if tx.outputs[int(nout)].script.is_claim_involved:
|
||||
results['previous-success'] += 1
|
||||
else:
|
||||
results['previous-corrupted'] += 1
|
||||
except Exception:
|
||||
log.exception("Couldn't verify certificate with look up key: %s", maybe_claim_id)
|
||||
results['previous-corrupted'] += 1
|
||||
|
||||
self.wallet.save()
|
||||
log.info('verifying and possibly migrating certificates:')
|
||||
log.info(json.dumps(results, indent=2))
|
||||
if addresses:
|
||||
log.warning('failed for addresses:')
|
||||
log.warning(json.dumps(
|
||||
[{'address': a, 'number of certificates': c} for a, c in addresses.items()],
|
||||
indent=2
|
||||
))
|
||||
|
||||
async def save_max_gap(self):
|
||||
gap = await self.get_max_gap()
|
||||
self.receiving.gap = max(20, gap['max_receiving_gap'] + 1)
|
||||
self.change.gap = max(6, gap['max_change_gap'] + 1)
|
||||
self.wallet.save()
|
||||
|
||||
def get_balance(self, confirmations=0, include_claims=False, **constraints):
|
||||
if not include_claims:
|
||||
constraints.update({'is_claim': 0, 'is_update': 0, 'is_support': 0})
|
||||
return super().get_balance(confirmations, **constraints)
|
||||
|
||||
@classmethod
|
||||
def get_private_key_from_seed(cls, ledger: 'baseledger.BaseLedger', seed: str, password: str):
|
||||
return super().get_private_key_from_seed(
|
||||
ledger, seed, password or 'lbryum'
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, ledger, wallet, d: dict) -> 'Account':
|
||||
account = super().from_dict(ledger, wallet, d)
|
||||
account.certificates = d.get('certificates', {})
|
||||
return account
|
||||
|
||||
def to_dict(self, include_certificates=True):
|
||||
d = super().to_dict()
|
||||
if include_certificates:
|
||||
d['certificates'] = self.certificates
|
||||
return d
|
||||
|
||||
async def get_details(self, **kwargs):
|
||||
details = await super().get_details(**kwargs)
|
||||
details['certificates'] = len(self.certificates)
|
||||
return details
|
||||
|
||||
def get_claim(self, claim_id=None, txid=None, nout=None):
|
||||
if claim_id is not None:
|
||||
return self.ledger.db.get_claims(account=self, claim_id=claim_id)
|
||||
elif txid is not None and nout is not None:
|
||||
return self.ledger.db.get_claims(**{'account': self, 'txo.txid': txid, 'txo.position': nout})
|
||||
|
||||
@staticmethod
|
||||
def constraint_utxos_sans_claims(constraints):
|
||||
constraints.update({'is_claim': 0, 'is_update': 0, 'is_support': 0})
|
||||
|
||||
def get_utxos(self, **constraints):
|
||||
self.constraint_utxos_sans_claims(constraints)
|
||||
return super().get_utxos(**constraints)
|
||||
|
||||
def get_utxo_count(self, **constraints):
|
||||
self.constraint_utxos_sans_claims(constraints)
|
||||
return super().get_utxo_count(**constraints)
|
||||
|
||||
def get_claims(self, **constraints):
|
||||
return self.ledger.db.get_claims(account=self, **constraints)
|
||||
|
||||
def get_claim_count(self, **constraints):
|
||||
return self.ledger.db.get_claim_count(account=self, **constraints)
|
||||
|
||||
def get_channels(self, **constraints):
|
||||
return self.ledger.db.get_channels(account=self, **constraints)
|
||||
|
||||
def get_channel_count(self, **constraints):
|
||||
return self.ledger.db.get_channel_count(account=self, **constraints)
|
||||
|
||||
async def send_to_addresses(self, amount, addresses, broadcast=False):
|
||||
tx_class = self.ledger.transaction_class
|
||||
tx = await tx_class.create(
|
||||
inputs=[],
|
||||
outputs=[
|
||||
tx_class.output_class.pay_pubkey_hash(amount, self.ledger.address_to_hash160(address))
|
||||
for address in addresses
|
||||
],
|
||||
funding_accounts=[self],
|
||||
change_account=self
|
||||
)
|
||||
|
||||
if broadcast:
|
||||
await self.ledger.broadcast(tx)
|
||||
else:
|
||||
await self.ledger.release_outputs(
|
||||
[txi.txo_ref.txo for txi in tx.inputs]
|
||||
)
|
||||
|
||||
return tx
|
||||
|
||||
async def release_all_outputs(self):
|
||||
await self.ledger.db.release_all_outputs(self)
|
||||
@@ -1,123 +0,0 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from binascii import unhexlify
|
||||
|
||||
from lbrynet.schema.validator import validate_claim_id
|
||||
from torba.client.baseledger import BaseLedger
|
||||
from lbrynet.schema.error import URIParseError
|
||||
from lbrynet.schema.uri import parse_lbry_uri
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc
|
||||
from lbrynet.extras.wallet.resolve import Resolver
|
||||
from lbrynet.extras.wallet.account import Account
|
||||
from lbrynet.extras.wallet.network import Network
|
||||
from lbrynet.extras.wallet.database import WalletDatabase
|
||||
from lbrynet.extras.wallet.transaction import Transaction
|
||||
from lbrynet.extras.wallet.header import Headers, UnvalidatedHeaders
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MainNetLedger(BaseLedger):
|
||||
name = 'LBRY Credits'
|
||||
symbol = 'LBC'
|
||||
network_name = 'mainnet'
|
||||
|
||||
headers: Headers
|
||||
|
||||
account_class = Account
|
||||
database_class = WalletDatabase
|
||||
headers_class = Headers
|
||||
network_class = Network
|
||||
transaction_class = Transaction
|
||||
|
||||
secret_prefix = bytes((0x1c,))
|
||||
pubkey_address_prefix = bytes((0x55,))
|
||||
script_address_prefix = bytes((0x7a,))
|
||||
extended_public_key_prefix = unhexlify('0488b21e')
|
||||
extended_private_key_prefix = unhexlify('0488ade4')
|
||||
|
||||
max_target = 0x0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
|
||||
genesis_hash = '9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463'
|
||||
genesis_bits = 0x1f00ffff
|
||||
target_timespan = 150
|
||||
|
||||
default_fee_per_byte = 50
|
||||
default_fee_per_name_char = 200000
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.fee_per_name_char = self.config.get('fee_per_name_char', self.default_fee_per_name_char)
|
||||
|
||||
@property
|
||||
def resolver(self):
|
||||
return Resolver(self.headers.claim_trie_root, self.headers.height, self.transaction_class,
|
||||
hash160_to_address=self.hash160_to_address, network=self.network)
|
||||
|
||||
async def resolve(self, page, page_size, *uris):
|
||||
for uri in uris:
|
||||
try:
|
||||
parsed_uri = parse_lbry_uri(uri)
|
||||
if parsed_uri.claim_id:
|
||||
validate_claim_id(parsed_uri.claim_id)
|
||||
except URIParseError as err:
|
||||
return {'error': err.args[0]}
|
||||
except Exception as e:
|
||||
return {'error': str(e)}
|
||||
try:
|
||||
resolver = self.resolver
|
||||
resolutions = await self.network.get_values_for_uris(self.headers.hash().decode(), *uris)
|
||||
return await resolver._handle_resolutions(resolutions, uris, page, page_size)
|
||||
except Exception as e:
|
||||
return {'error': str(e)}
|
||||
|
||||
async def get_claim_by_claim_id(self, claim_id):
|
||||
result = (await self.network.get_claims_by_ids(claim_id)).pop(claim_id, {})
|
||||
return await self.resolver.get_certificate_and_validate_result(result)
|
||||
|
||||
async def get_claim_by_outpoint(self, txid, nout):
|
||||
claims = (await self.network.get_claims_in_tx(txid)) or []
|
||||
for claim in claims:
|
||||
if claim['nout'] == nout:
|
||||
return await self.resolver.get_certificate_and_validate_result(claim)
|
||||
return 'claim not found'
|
||||
|
||||
async def start(self):
|
||||
await super().start()
|
||||
await asyncio.gather(*(a.maybe_migrate_certificates() for a in self.accounts))
|
||||
await asyncio.gather(*(a.save_max_gap() for a in self.accounts))
|
||||
await self._report_state()
|
||||
|
||||
async def _report_state(self):
|
||||
for account in self.accounts:
|
||||
total_receiving = len((await account.receiving.get_addresses()))
|
||||
total_change = len((await account.change.get_addresses()))
|
||||
channel_count = await account.get_channel_count()
|
||||
claim_count = await account.get_claim_count()
|
||||
balance = dewies_to_lbc(await account.get_balance())
|
||||
log.info("Loaded account %s with %s LBC, %d receiving addresses (gap: %d), "
|
||||
"%d change addresses (gap: %d), %d channels, %d certificates and %d claims. ",
|
||||
account.id, balance, total_receiving, account.receiving.gap, total_change, account.change.gap,
|
||||
channel_count, len(account.certificates), claim_count)
|
||||
|
||||
|
||||
class TestNetLedger(MainNetLedger):
|
||||
network_name = 'testnet'
|
||||
pubkey_address_prefix = bytes((111,))
|
||||
script_address_prefix = bytes((196,))
|
||||
extended_public_key_prefix = unhexlify('043587cf')
|
||||
extended_private_key_prefix = unhexlify('04358394')
|
||||
|
||||
|
||||
class RegTestLedger(MainNetLedger):
|
||||
network_name = 'regtest'
|
||||
headers_class = UnvalidatedHeaders
|
||||
pubkey_address_prefix = bytes((111,))
|
||||
script_address_prefix = bytes((196,))
|
||||
extended_public_key_prefix = unhexlify('043587cf')
|
||||
extended_private_key_prefix = unhexlify('04358394')
|
||||
|
||||
max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
|
||||
genesis_hash = '6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556'
|
||||
genesis_bits = 0x207fffff
|
||||
target_timespan = 1
|
||||
@@ -1,26 +0,0 @@
|
||||
from torba.client.basenetwork import BaseNetwork
|
||||
|
||||
|
||||
class Network(BaseNetwork):
|
||||
|
||||
def get_block(self, block_hash):
|
||||
return self.rpc('blockchain.block.get_block', block_hash)
|
||||
|
||||
def get_server_height(self):
|
||||
return self.rpc('blockchain.block.get_server_height')
|
||||
|
||||
def get_values_for_uris(self, block_hash, *uris):
|
||||
return self.rpc('blockchain.claimtrie.getvaluesforuris', block_hash, *uris)
|
||||
|
||||
def get_claims_by_ids(self, *claim_ids):
|
||||
return self.rpc('blockchain.claimtrie.getclaimsbyids', *claim_ids)
|
||||
|
||||
def get_claims_in_tx(self, txid):
|
||||
return self.rpc('blockchain.claimtrie.getclaimsintx', txid)
|
||||
|
||||
def get_claims_for_name(self, name):
|
||||
return self.rpc('blockchain.claimtrie.getclaimsforname', name)
|
||||
|
||||
def get_transaction_height(self, txid):
|
||||
# 1.0 protocol specific workaround. Newer protocol should use get_transaction with verbose True
|
||||
return self.rpc('blockchain.transaction.get_height', txid)
|
||||
@@ -1,499 +0,0 @@
|
||||
import logging
|
||||
|
||||
from ecdsa import BadSignatureError
|
||||
from binascii import unhexlify, hexlify
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc
|
||||
from lbrynet.error import UnknownNameError, UnknownClaimID, UnknownURI, UnknownOutpoint
|
||||
from lbrynet.schema.address import is_address
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
from lbrynet.schema.error import DecodeError
|
||||
from lbrynet.schema.uri import parse_lbry_uri
|
||||
from lbrynet.extras.wallet.claim_proofs import verify_proof, InvalidProofError
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Resolver:
|
||||
|
||||
def __init__(self, claim_trie_root, height, transaction_class, hash160_to_address, network):
|
||||
self.claim_trie_root = claim_trie_root
|
||||
self.height = height
|
||||
self.transaction_class = transaction_class
|
||||
self.hash160_to_address = hash160_to_address
|
||||
self.network = network
|
||||
|
||||
async def _handle_resolutions(self, resolutions, requested_uris, page, page_size):
|
||||
results = {}
|
||||
for uri in requested_uris:
|
||||
resolution = (resolutions or {}).get(uri, {})
|
||||
if resolution:
|
||||
try:
|
||||
results[uri] = _handle_claim_result(
|
||||
await self._handle_resolve_uri_response(uri, resolution, page, page_size)
|
||||
)
|
||||
except (UnknownNameError, UnknownClaimID, UnknownURI) as err:
|
||||
results[uri] = {'error': str(err)}
|
||||
else:
|
||||
results[uri] = {'error': "URI lbry://{} cannot be resolved".format(uri.replace("lbry://", ""))}
|
||||
return results
|
||||
|
||||
async def _handle_resolve_uri_response(self, uri, resolution, page=0, page_size=10, raw=False):
|
||||
result = {}
|
||||
claim_trie_root = self.claim_trie_root
|
||||
parsed_uri = parse_lbry_uri(uri)
|
||||
certificate = None
|
||||
# parse an included certificate
|
||||
if 'certificate' in resolution:
|
||||
certificate_response = resolution['certificate']['result']
|
||||
certificate_resolution_type = resolution['certificate']['resolution_type']
|
||||
if certificate_resolution_type == "winning" and certificate_response:
|
||||
if 'height' in certificate_response:
|
||||
height = certificate_response['height']
|
||||
depth = self.height - height
|
||||
certificate_result = _verify_proof(parsed_uri.name,
|
||||
claim_trie_root,
|
||||
certificate_response,
|
||||
height, depth,
|
||||
transaction_class=self.transaction_class,
|
||||
hash160_to_address=self.hash160_to_address)
|
||||
result['certificate'] = await self.parse_and_validate_claim_result(certificate_result,
|
||||
raw=raw)
|
||||
elif certificate_resolution_type == "claim_id":
|
||||
result['certificate'] = await self.parse_and_validate_claim_result(certificate_response,
|
||||
raw=raw)
|
||||
elif certificate_resolution_type == "sequence":
|
||||
result['certificate'] = await self.parse_and_validate_claim_result(certificate_response,
|
||||
raw=raw)
|
||||
else:
|
||||
log.error("unknown response type: %s", certificate_resolution_type)
|
||||
|
||||
if 'certificate' in result:
|
||||
certificate = result['certificate']
|
||||
if 'unverified_claims_in_channel' in resolution:
|
||||
max_results = len(resolution['unverified_claims_in_channel'])
|
||||
result['claims_in_channel'] = max_results
|
||||
else:
|
||||
result['claims_in_channel'] = 0
|
||||
else:
|
||||
result['error'] = "claim not found"
|
||||
result['success'] = False
|
||||
result['uri'] = str(parsed_uri)
|
||||
|
||||
else:
|
||||
certificate = None
|
||||
|
||||
# if this was a resolution for a name, parse the result
|
||||
if 'claim' in resolution:
|
||||
claim_response = resolution['claim']['result']
|
||||
claim_resolution_type = resolution['claim']['resolution_type']
|
||||
if claim_resolution_type == "winning" and claim_response:
|
||||
if 'height' in claim_response:
|
||||
height = claim_response['height']
|
||||
depth = self.height - height
|
||||
claim_result = _verify_proof(parsed_uri.name,
|
||||
claim_trie_root,
|
||||
claim_response,
|
||||
height, depth,
|
||||
transaction_class=self.transaction_class,
|
||||
hash160_to_address=self.hash160_to_address)
|
||||
result['claim'] = await self.parse_and_validate_claim_result(claim_result,
|
||||
certificate,
|
||||
raw)
|
||||
elif claim_resolution_type == "claim_id":
|
||||
result['claim'] = await self.parse_and_validate_claim_result(claim_response,
|
||||
certificate,
|
||||
raw)
|
||||
elif claim_resolution_type == "sequence":
|
||||
result['claim'] = await self.parse_and_validate_claim_result(claim_response,
|
||||
certificate,
|
||||
raw)
|
||||
else:
|
||||
log.error("unknown response type: %s", claim_resolution_type)
|
||||
|
||||
# if this was a resolution for a name in a channel make sure there is only one valid
|
||||
# match
|
||||
elif 'unverified_claims_for_name' in resolution and 'certificate' in result:
|
||||
unverified_claims_for_name = resolution['unverified_claims_for_name']
|
||||
|
||||
channel_info = await self.get_channel_claims_page(unverified_claims_for_name,
|
||||
result['certificate'], page=1)
|
||||
claims_in_channel, upper_bound = channel_info
|
||||
|
||||
if not claims_in_channel:
|
||||
log.error("No valid claims for this name for this channel")
|
||||
elif len(claims_in_channel) > 1:
|
||||
log.warning("Multiple signed claims for the same name.")
|
||||
winner = pick_winner_from_channel_path_collision(claims_in_channel)
|
||||
if winner:
|
||||
result['claim'] = winner
|
||||
else:
|
||||
log.error("No valid claims for this name for this channel")
|
||||
else:
|
||||
result['claim'] = claims_in_channel[0]
|
||||
|
||||
# parse and validate claims in a channel iteratively into pages of results
|
||||
elif 'unverified_claims_in_channel' in resolution and 'certificate' in result:
|
||||
ids_to_check = resolution['unverified_claims_in_channel']
|
||||
channel_info = await self.get_channel_claims_page(ids_to_check, result['certificate'],
|
||||
page=page, page_size=page_size)
|
||||
claims_in_channel, upper_bound = channel_info
|
||||
|
||||
if claims_in_channel:
|
||||
result['claims_in_channel'] = claims_in_channel
|
||||
elif 'error' not in result:
|
||||
return {'error': 'claim not found', 'success': False, 'uri': str(parsed_uri)}
|
||||
|
||||
# invalid signatures can only return outside a channel
|
||||
if result.get('claim', {}).get('has_signature', False):
|
||||
if parsed_uri.path and not result['claim']['signature_is_valid']:
|
||||
return {'error': 'claim not found', 'success': False, 'uri': str(parsed_uri)}
|
||||
return result
|
||||
|
||||
async def get_certificate_and_validate_result(self, claim_result):
|
||||
if not claim_result or 'value' not in claim_result:
|
||||
return claim_result
|
||||
certificate = None
|
||||
certificate_id = smart_decode(claim_result['value']).certificate_id
|
||||
if certificate_id:
|
||||
certificate = await self.network.get_claims_by_ids(certificate_id)
|
||||
certificate = certificate.pop(certificate_id) if certificate else None
|
||||
return await self.parse_and_validate_claim_result(claim_result, certificate=certificate)
|
||||
|
||||
async def parse_and_validate_claim_result(self, claim_result, certificate=None, raw=False):
|
||||
if not claim_result or 'value' not in claim_result:
|
||||
return claim_result
|
||||
|
||||
claim_result['decoded_claim'] = False
|
||||
decoded = None
|
||||
|
||||
if not raw:
|
||||
claim_value = claim_result['value']
|
||||
try:
|
||||
decoded = smart_decode(claim_value)
|
||||
claim_result['value'] = decoded.claim_dict
|
||||
claim_result['decoded_claim'] = True
|
||||
except DecodeError:
|
||||
pass
|
||||
|
||||
if decoded:
|
||||
claim_result['has_signature'] = False
|
||||
if decoded.has_signature:
|
||||
if certificate is None:
|
||||
log.info("fetching certificate to check claim signature")
|
||||
certificate = await self.network.get_claims_by_ids(decoded.certificate_id)
|
||||
if not certificate:
|
||||
log.warning('Certificate %s not found', decoded.certificate_id)
|
||||
claim_result['has_signature'] = True
|
||||
claim_result['signature_is_valid'] = False
|
||||
validated, channel_name = validate_claim_signature_and_get_channel_name(
|
||||
decoded, certificate, claim_result['address'], claim_result['name'])
|
||||
claim_result['channel_name'] = channel_name
|
||||
if validated:
|
||||
claim_result['signature_is_valid'] = True
|
||||
|
||||
if 'height' in claim_result and claim_result['height'] is None:
|
||||
claim_result['height'] = -1
|
||||
|
||||
if 'amount' in claim_result:
|
||||
claim_result = format_amount_value(claim_result)
|
||||
|
||||
claim_result['permanent_url'] = _get_permanent_url(claim_result, decoded.certificate_id if decoded else None)
|
||||
|
||||
return claim_result
|
||||
|
||||
@staticmethod
|
||||
def prepare_claim_queries(start_position, query_size, channel_claim_infos):
|
||||
queries = [tuple()]
|
||||
names = {}
|
||||
# a table of index counts for the sorted claim ids, including ignored claims
|
||||
absolute_position_index = {}
|
||||
|
||||
block_sorted_infos = sorted(channel_claim_infos.items(), key=lambda x: int(x[1][1]))
|
||||
per_block_infos = {}
|
||||
for claim_id, (name, height) in block_sorted_infos:
|
||||
claims = per_block_infos.get(height, [])
|
||||
claims.append((claim_id, name))
|
||||
per_block_infos[height] = sorted(claims, key=lambda x: int(x[0], 16))
|
||||
|
||||
abs_position = 0
|
||||
|
||||
for height in sorted(per_block_infos.keys(), reverse=True):
|
||||
for claim_id, name in per_block_infos[height]:
|
||||
names[claim_id] = name
|
||||
absolute_position_index[claim_id] = abs_position
|
||||
if abs_position >= start_position:
|
||||
if len(queries[-1]) >= query_size:
|
||||
queries.append(tuple())
|
||||
queries[-1] += (claim_id,)
|
||||
abs_position += 1
|
||||
return queries, names, absolute_position_index
|
||||
|
||||
async def iter_channel_claims_pages(self, queries, claim_positions, claim_names, certificate,
|
||||
page_size=10):
|
||||
# lbryum server returns a dict of {claim_id: (name, claim_height)}
|
||||
# first, sort the claims by block height (and by claim id int value within a block).
|
||||
|
||||
# map the sorted claims into getclaimsbyids queries of query_size claim ids each
|
||||
|
||||
# send the batched queries to lbryum server and iteratively validate and parse
|
||||
# the results, yield a page of results at a time.
|
||||
|
||||
# these results can include those where `signature_is_valid` is False. if they are skipped,
|
||||
# page indexing becomes tricky, as the number of results isn't known until after having
|
||||
# processed them.
|
||||
# TODO: fix ^ in lbrynet.schema
|
||||
|
||||
async def iter_validate_channel_claims():
|
||||
formatted_claims = []
|
||||
for claim_ids in queries:
|
||||
batch_result = await self.network.get_claims_by_ids(*claim_ids)
|
||||
for claim_id in claim_ids:
|
||||
claim = batch_result[claim_id]
|
||||
if claim['name'] == claim_names[claim_id]:
|
||||
formatted_claim = await self.parse_and_validate_claim_result(claim, certificate)
|
||||
formatted_claim['absolute_channel_position'] = claim_positions[
|
||||
claim['claim_id']]
|
||||
formatted_claims.append(formatted_claim)
|
||||
else:
|
||||
log.warning("ignoring claim with name mismatch %s %s", claim['name'],
|
||||
claim['claim_id'])
|
||||
return formatted_claims
|
||||
|
||||
results = []
|
||||
|
||||
for claim in await iter_validate_channel_claims():
|
||||
results.append(claim)
|
||||
|
||||
# if there is a full page of results, yield it
|
||||
if len(results) and len(results) % page_size == 0:
|
||||
return results[-page_size:]
|
||||
|
||||
return results
|
||||
|
||||
async def get_channel_claims_page(self, channel_claim_infos, certificate, page, page_size=10):
|
||||
page = page or 0
|
||||
page_size = max(page_size, 1)
|
||||
if page_size > 500:
|
||||
raise Exception("page size above maximum allowed")
|
||||
start_position = (page - 1) * page_size
|
||||
queries, names, claim_positions = self.prepare_claim_queries(start_position, page_size,
|
||||
channel_claim_infos)
|
||||
page_generator = await self.iter_channel_claims_pages(queries, claim_positions, names,
|
||||
certificate, page_size=page_size)
|
||||
upper_bound = len(claim_positions)
|
||||
if not page:
|
||||
return None, upper_bound
|
||||
if start_position > upper_bound:
|
||||
raise IndexError("claim %i greater than max %i" % (start_position, upper_bound))
|
||||
return page_generator, upper_bound
|
||||
|
||||
|
||||
# Format amount to be decimal encoded string
|
||||
# Format value to be hex encoded string
|
||||
# TODO: refactor. Came from lbryum, there could be another part of torba doing it
|
||||
def format_amount_value(obj):
|
||||
if isinstance(obj, dict):
|
||||
for k, v in obj.items():
|
||||
if k in ('amount', 'effective_amount'):
|
||||
if not isinstance(obj[k], float):
|
||||
obj[k] = dewies_to_lbc(obj[k])
|
||||
elif k == 'supports' and isinstance(v, list):
|
||||
obj[k] = [{'txid': txid, 'nout': nout, 'amount': dewies_to_lbc(amount)}
|
||||
for (txid, nout, amount) in v]
|
||||
elif isinstance(v, (list, dict)):
|
||||
obj[k] = format_amount_value(v)
|
||||
elif isinstance(obj, list):
|
||||
obj = [format_amount_value(o) for o in obj]
|
||||
return obj
|
||||
|
||||
|
||||
def _get_permanent_url(claim_result, certificate_id):
|
||||
if certificate_id:
|
||||
return f"{claim_result['channel_name']}#{certificate_id}/{claim_result['name']}"
|
||||
else:
|
||||
return f"{claim_result['name']}#{claim_result['claim_id']}"
|
||||
|
||||
|
||||
def _verify_proof(name, claim_trie_root, result, height, depth, transaction_class, hash160_to_address):
|
||||
"""
|
||||
Verify proof for name claim
|
||||
"""
|
||||
|
||||
def _build_response(name, value, claim_id, txid, n, amount, effective_amount,
|
||||
claim_sequence, claim_address, supports):
|
||||
r = {
|
||||
'name': name,
|
||||
'value': hexlify(value),
|
||||
'claim_id': claim_id,
|
||||
'txid': txid,
|
||||
'nout': n,
|
||||
'amount': amount,
|
||||
'effective_amount': effective_amount,
|
||||
'height': height,
|
||||
'depth': depth,
|
||||
'claim_sequence': claim_sequence,
|
||||
'address': claim_address,
|
||||
'supports': supports
|
||||
}
|
||||
return r
|
||||
|
||||
def _parse_proof_result(name, result):
|
||||
support_amount = sum([amt for (stxid, snout, amt) in result['supports']])
|
||||
supports = result['supports']
|
||||
if 'txhash' in result['proof'] and 'nOut' in result['proof']:
|
||||
if 'transaction' in result:
|
||||
tx = transaction_class(raw=unhexlify(result['transaction']))
|
||||
nOut = result['proof']['nOut']
|
||||
if result['proof']['txhash'] == tx.id:
|
||||
if 0 <= nOut < len(tx.outputs):
|
||||
claim_output = tx.outputs[nOut]
|
||||
effective_amount = claim_output.amount + support_amount
|
||||
claim_address = hash160_to_address(claim_output.script.values['pubkey_hash'])
|
||||
claim_id = result['claim_id']
|
||||
claim_sequence = result['claim_sequence']
|
||||
claim_script = claim_output.script
|
||||
decoded_name = claim_script.values['claim_name'].decode()
|
||||
decoded_value = claim_script.values['claim']
|
||||
if decoded_name == name:
|
||||
return _build_response(name, decoded_value, claim_id,
|
||||
tx.id, nOut, claim_output.amount,
|
||||
effective_amount, claim_sequence,
|
||||
claim_address, supports)
|
||||
return {'error': 'name in proof did not match requested name'}
|
||||
outputs = len(tx['outputs'])
|
||||
return {'error': 'invalid nOut: %d (let(outputs): %d' % (nOut, outputs)}
|
||||
return {'error': "computed txid did not match given transaction: %s vs %s" %
|
||||
(tx.id, result['proof']['txhash'])
|
||||
}
|
||||
return {'error': "didn't receive a transaction with the proof"}
|
||||
return {'error': 'name is not claimed'}
|
||||
|
||||
if 'proof' in result:
|
||||
proof_name = name
|
||||
if 'name' in result:
|
||||
proof_name = result['name']
|
||||
name = result['name']
|
||||
if 'normalized_name' in result:
|
||||
proof_name = result['normalized_name']
|
||||
try:
|
||||
verify_proof(result['proof'], claim_trie_root, proof_name)
|
||||
except InvalidProofError:
|
||||
return {'error': "Proof was invalid"}
|
||||
return _parse_proof_result(name, result)
|
||||
else:
|
||||
return {'error': "proof not in result"}
|
||||
|
||||
|
||||
def validate_claim_signature_and_get_channel_name(claim, certificate_claim,
|
||||
claim_address, name, decoded_certificate=None):
|
||||
if not certificate_claim:
|
||||
return False, None
|
||||
if 'value' not in certificate_claim:
|
||||
log.warning('Got an invalid claim while parsing certificates, please report: %s', certificate_claim)
|
||||
return False, None
|
||||
certificate = decoded_certificate or smart_decode(certificate_claim['value'])
|
||||
if not isinstance(certificate, ClaimDict):
|
||||
raise TypeError("Certificate is not a ClaimDict: %s" % str(type(certificate)))
|
||||
if _validate_signed_claim(claim, claim_address, name, certificate):
|
||||
return True, certificate_claim['name']
|
||||
return False, None
|
||||
|
||||
|
||||
def _validate_signed_claim(claim, claim_address, name, certificate):
|
||||
if not claim.has_signature:
|
||||
raise Exception("Claim is not signed")
|
||||
if not is_address(claim_address):
|
||||
raise Exception("Not given a valid claim address")
|
||||
try:
|
||||
if claim.validate_signature(claim_address, certificate.protobuf, name):
|
||||
return True
|
||||
except BadSignatureError:
|
||||
# print_msg("Signature for %s is invalid" % claim_id)
|
||||
return False
|
||||
except Exception as err:
|
||||
log.error("Signature for %s is invalid, reason: %s - %s", claim_address,
|
||||
str(type(err)), err)
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
# TODO: The following came from code handling lbryum results. Now that it's all in one place a refactor should unify it.
|
||||
def _decode_claim_result(claim):
|
||||
if 'has_signature' in claim and claim['has_signature']:
|
||||
if not claim['signature_is_valid']:
|
||||
log.warning("lbry://%s#%s has an invalid signature",
|
||||
claim['name'], claim['claim_id'])
|
||||
if 'value' not in claim:
|
||||
log.warning('Got an invalid claim while parsing, please report: %s', claim)
|
||||
claim['hex'] = None
|
||||
claim['value'] = None
|
||||
backend_message = ' SDK message: ' + claim['error'] if 'error' in claim else ''
|
||||
claim['error'] = "Failed to parse: missing value." + backend_message
|
||||
return claim
|
||||
try:
|
||||
decoded = smart_decode(claim['value'])
|
||||
claim_dict = decoded.claim_dict
|
||||
claim['value'] = claim_dict
|
||||
claim['hex'] = hexlify(decoded.serialized)
|
||||
except DecodeError:
|
||||
claim['hex'] = claim['value']
|
||||
claim['value'] = None
|
||||
claim['error'] = "Failed to decode value"
|
||||
return claim
|
||||
|
||||
|
||||
def _handle_claim_result(results):
|
||||
if not results:
|
||||
#TODO: cannot determine what name we searched for here
|
||||
# we should fix lbryum commands that return None
|
||||
raise UnknownNameError("")
|
||||
|
||||
if 'error' in results:
|
||||
if results['error'] in ['name is not claimed', 'claim not found']:
|
||||
if 'claim_id' in results:
|
||||
raise UnknownClaimID(results['claim_id'])
|
||||
if 'name' in results:
|
||||
raise UnknownNameError(results['name'])
|
||||
if 'uri' in results:
|
||||
raise UnknownURI(results['uri'])
|
||||
if 'outpoint' in results:
|
||||
raise UnknownOutpoint(results['outpoint'])
|
||||
raise Exception(results['error'])
|
||||
|
||||
# case where return value is {'certificate':{'txid', 'value',...},...}
|
||||
if 'certificate' in results:
|
||||
results['certificate'] = _decode_claim_result(results['certificate'])
|
||||
|
||||
# case where return value is {'claim':{'txid','value',...},...}
|
||||
if 'claim' in results:
|
||||
results['claim'] = _decode_claim_result(results['claim'])
|
||||
|
||||
# case where return value is {'txid','value',...}
|
||||
# returned by queries that are not name resolve related
|
||||
# (getclaimbyoutpoint, getclaimbyid, getclaimsfromtx)
|
||||
elif 'value' in results:
|
||||
results = _decode_claim_result(results)
|
||||
|
||||
# case where there is no 'certificate', 'value', or 'claim' key
|
||||
elif 'certificate' not in results:
|
||||
msg = f'result in unexpected format:{results}'
|
||||
assert False, msg
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def pick_winner_from_channel_path_collision(claims_in_channel):
|
||||
# we should be doing this by effective amount so we pick the controlling claim, however changing the resolved
|
||||
# claim triggers another issue where 2 claims cant be saved for the same file. This code picks the oldest, so it
|
||||
# stays the same. Using effective amount would change the resolved claims for a channel path on takeovers,
|
||||
# potentially triggering that.
|
||||
winner = None
|
||||
for claim in claims_in_channel:
|
||||
if not claim['signature_is_valid']:
|
||||
continue
|
||||
if winner is None:
|
||||
winner = claim
|
||||
elif claim['height'] < winner['height'] or \
|
||||
(claim['height'] == winner['height'] and claim['nout'] < winner['nout']):
|
||||
winner = claim
|
||||
return winner
|
||||
@@ -1,169 +0,0 @@
|
||||
import hashlib
|
||||
import struct
|
||||
from binascii import unhexlify
|
||||
|
||||
import msgpack
|
||||
from torba.server.hash import hash_to_hex_str
|
||||
|
||||
from torba.server.block_processor import BlockProcessor
|
||||
from lbrynet.schema.uri import parse_lbry_uri
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
|
||||
from lbrynet.extras.wallet.server.model import NameClaim, ClaimInfo, ClaimUpdate, ClaimSupport
|
||||
|
||||
|
||||
class LBRYBlockProcessor(BlockProcessor):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
if self.env.coin.NET == "regtest":
|
||||
self.prefetcher.polling_delay = 0.5
|
||||
|
||||
self.should_validate_signatures = self.env.boolean('VALIDATE_CLAIM_SIGNATURES', False)
|
||||
self.logger.info("LbryumX Block Processor - Validating signatures: {}".format(self.should_validate_signatures))
|
||||
|
||||
def advance_blocks(self, blocks):
|
||||
# save height, advance blocks as usual, then hook our claim tx processing
|
||||
height = self.height + 1
|
||||
super().advance_blocks(blocks)
|
||||
pending_undo = []
|
||||
for index, block in enumerate(blocks):
|
||||
undo = self.advance_claim_txs(block.transactions, height + index)
|
||||
pending_undo.append((height+index, undo,))
|
||||
self.db.write_undo(pending_undo)
|
||||
|
||||
def advance_claim_txs(self, txs, height):
|
||||
# TODO: generate claim undo info!
|
||||
undo_info = []
|
||||
add_undo = undo_info.append
|
||||
update_inputs = set()
|
||||
for tx, txid in txs:
|
||||
update_inputs.clear()
|
||||
if tx.has_claims:
|
||||
for index, output in enumerate(tx.outputs):
|
||||
claim = output.claim
|
||||
if isinstance(claim, NameClaim):
|
||||
add_undo(self.advance_claim_name_transaction(output, height, txid, index))
|
||||
elif isinstance(claim, ClaimUpdate):
|
||||
update_input = self.db.get_update_input(claim, tx.inputs)
|
||||
if update_input:
|
||||
update_inputs.add(update_input)
|
||||
add_undo(self.advance_update_claim(output, height, txid, index))
|
||||
else:
|
||||
info = (hash_to_hex_str(txid), hash_to_hex_str(claim.claim_id),)
|
||||
self.logger.error("REJECTED: {} updating {}".format(*info))
|
||||
elif isinstance(claim, ClaimSupport):
|
||||
self.advance_support(claim, txid, index, height, output.value)
|
||||
for txin in tx.inputs:
|
||||
if txin not in update_inputs:
|
||||
abandoned_claim_id = self.db.abandon_spent(txin.prev_hash, txin.prev_idx)
|
||||
if abandoned_claim_id:
|
||||
add_undo((abandoned_claim_id, self.db.get_claim_info(abandoned_claim_id)))
|
||||
return undo_info
|
||||
|
||||
def advance_update_claim(self, output, height, txid, nout):
|
||||
claim_id = output.claim.claim_id
|
||||
claim_info = self.claim_info_from_output(output, txid, nout, height)
|
||||
old_claim_info = self.db.get_claim_info(claim_id)
|
||||
self.db.put_claim_id_for_outpoint(old_claim_info.txid, old_claim_info.nout, None)
|
||||
if old_claim_info.cert_id:
|
||||
self.db.remove_claim_from_certificate_claims(old_claim_info.cert_id, claim_id)
|
||||
if claim_info.cert_id:
|
||||
self.db.put_claim_id_signed_by_cert_id(claim_info.cert_id, claim_id)
|
||||
self.db.put_claim_info(claim_id, claim_info)
|
||||
self.db.put_claim_id_for_outpoint(txid, nout, claim_id)
|
||||
return claim_id, old_claim_info
|
||||
|
||||
def advance_claim_name_transaction(self, output, height, txid, nout):
|
||||
claim_id = claim_id_hash(txid, nout)
|
||||
claim_info = self.claim_info_from_output(output, txid, nout, height)
|
||||
if claim_info.cert_id:
|
||||
self.db.put_claim_id_signed_by_cert_id(claim_info.cert_id, claim_id)
|
||||
self.db.put_claim_info(claim_id, claim_info)
|
||||
self.db.put_claim_id_for_outpoint(txid, nout, claim_id)
|
||||
return claim_id, None
|
||||
|
||||
def backup_from_undo_info(self, claim_id, undo_claim_info):
|
||||
"""
|
||||
Undo information holds a claim state **before** a transaction changes it
|
||||
There are 4 possibilities when processing it, of which only 3 are valid ones:
|
||||
1. the claim is known and the undo info has info, it was an update
|
||||
2. the claim is known and the undo info doesn't hold any info, it was claimed
|
||||
3. the claim in unknown and the undo info has info, it was abandoned
|
||||
4. the claim is unknown and the undo info does't hold info, error!
|
||||
"""
|
||||
|
||||
undo_claim_info = ClaimInfo(*undo_claim_info) if undo_claim_info else None
|
||||
current_claim_info = self.db.get_claim_info(claim_id)
|
||||
if current_claim_info and undo_claim_info:
|
||||
# update, remove current claim
|
||||
self.db.remove_claim_id_for_outpoint(current_claim_info.txid, current_claim_info.nout)
|
||||
if current_claim_info.cert_id:
|
||||
self.db.remove_claim_from_certificate_claims(current_claim_info.cert_id, claim_id)
|
||||
elif current_claim_info and not undo_claim_info:
|
||||
# claim, abandon it
|
||||
self.db.abandon_spent(current_claim_info.txid, current_claim_info.nout)
|
||||
elif not current_claim_info and undo_claim_info:
|
||||
# abandon, reclaim it (happens below)
|
||||
pass
|
||||
else:
|
||||
# should never happen, unless the database got into an inconsistent state
|
||||
raise Exception("Unexpected situation occurred on backup, this means the database is inconsistent. "
|
||||
"Please report. Resetting the data folder (reindex) solves it for now.")
|
||||
if undo_claim_info:
|
||||
self.db.put_claim_info(claim_id, undo_claim_info)
|
||||
if undo_claim_info.cert_id:
|
||||
cert_id = self._checksig(undo_claim_info.name, undo_claim_info.value, undo_claim_info.address)
|
||||
self.db.put_claim_id_signed_by_cert_id(cert_id, claim_id)
|
||||
self.db.put_claim_id_for_outpoint(undo_claim_info.txid, undo_claim_info.nout, claim_id)
|
||||
|
||||
def backup_txs(self, txs):
|
||||
self.logger.info("Reorg at height {} with {} transactions.".format(self.height, len(txs)))
|
||||
undo_info = msgpack.loads(self.db.claim_undo_db.get(struct.pack(">I", self.height)), use_list=False)
|
||||
for claim_id, undo_claim_info in reversed(undo_info):
|
||||
self.backup_from_undo_info(claim_id, undo_claim_info)
|
||||
return super().backup_txs(txs)
|
||||
|
||||
def backup_blocks(self, raw_blocks):
|
||||
self.db.batched_flush_claims()
|
||||
super().backup_blocks(raw_blocks=raw_blocks)
|
||||
self.db.batched_flush_claims()
|
||||
|
||||
async def flush(self, flush_utxos):
|
||||
self.db.batched_flush_claims()
|
||||
return await super().flush(flush_utxos)
|
||||
|
||||
def advance_support(self, claim_support, txid, nout, height, amount):
|
||||
# TODO: check for more controller claim rules, like takeover or ordering
|
||||
pass
|
||||
|
||||
def claim_info_from_output(self, output, txid, nout, height):
|
||||
amount = output.value
|
||||
address = self.coin.address_from_script(output.pk_script)
|
||||
name, value, cert_id = output.claim.name, output.claim.value, None
|
||||
assert txid and address
|
||||
cert_id = self._checksig(name, value, address)
|
||||
return ClaimInfo(name, value, txid, nout, amount, address, height, cert_id)
|
||||
|
||||
def _checksig(self, name, value, address):
|
||||
try:
|
||||
parse_lbry_uri(name.decode()) # skip invalid names
|
||||
claim_dict = smart_decode(value)
|
||||
cert_id = unhexlify(claim_dict.certificate_id)[::-1]
|
||||
if not self.should_validate_signatures:
|
||||
return cert_id
|
||||
if cert_id:
|
||||
cert_claim = self.db.get_claim_info(cert_id)
|
||||
if cert_claim:
|
||||
certificate = smart_decode(cert_claim.value)
|
||||
claim_dict.validate_signature(address, certificate)
|
||||
return cert_id
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
def claim_id_hash(txid, n):
|
||||
# TODO: This should be in lbryschema
|
||||
packed = txid + struct.pack('>I', n)
|
||||
md = hashlib.new('ripemd160')
|
||||
md.update(hashlib.sha256(packed).digest())
|
||||
return md.digest()
|
||||
@@ -1,177 +0,0 @@
|
||||
import msgpack
|
||||
import struct
|
||||
|
||||
import time
|
||||
from torba.server.hash import hash_to_hex_str
|
||||
|
||||
from torba.server.db import DB
|
||||
|
||||
from lbrynet.extras.wallet.server.model import ClaimInfo
|
||||
|
||||
|
||||
class LBRYDB(DB):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.claim_cache = {}
|
||||
self.claims_signed_by_cert_cache = {}
|
||||
self.outpoint_to_claim_id_cache = {}
|
||||
self.claims_db = self.signatures_db = self.outpoint_to_claim_id_db = self.claim_undo_db = None
|
||||
# stores deletes not yet flushed to disk
|
||||
self.pending_abandons = {}
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def close(self):
|
||||
self.batched_flush_claims()
|
||||
self.claims_db.close()
|
||||
self.signatures_db.close()
|
||||
self.outpoint_to_claim_id_db.close()
|
||||
self.claim_undo_db.close()
|
||||
self.utxo_db.close()
|
||||
super().close()
|
||||
|
||||
async def _open_dbs(self, for_sync, compacting):
|
||||
await super()._open_dbs(for_sync=for_sync, compacting=compacting)
|
||||
def log_reason(message, is_for_sync):
|
||||
reason = 'sync' if is_for_sync else 'serving'
|
||||
self.logger.info('{} for {}'.format(message, reason))
|
||||
|
||||
if self.claims_db:
|
||||
if self.claims_db.for_sync == for_sync:
|
||||
return
|
||||
log_reason('closing claim DBs to re-open', for_sync)
|
||||
self.claims_db.close()
|
||||
self.signatures_db.close()
|
||||
self.outpoint_to_claim_id_db.close()
|
||||
self.claim_undo_db.close()
|
||||
self.claims_db = self.db_class('claims', for_sync)
|
||||
self.signatures_db = self.db_class('signatures', for_sync)
|
||||
self.outpoint_to_claim_id_db = self.db_class('outpoint_claim_id', for_sync)
|
||||
self.claim_undo_db = self.db_class('claim_undo', for_sync)
|
||||
log_reason('opened claim DBs', self.claims_db.for_sync)
|
||||
|
||||
def flush_dbs(self, flush_data, flush_utxos, estimate_txs_remaining):
|
||||
# flush claims together with utxos as they are parsed together
|
||||
self.batched_flush_claims()
|
||||
return super().flush_dbs(flush_data, flush_utxos, estimate_txs_remaining)
|
||||
|
||||
def batched_flush_claims(self):
|
||||
with self.claims_db.write_batch() as claims_batch:
|
||||
with self.signatures_db.write_batch() as signed_claims_batch:
|
||||
with self.outpoint_to_claim_id_db.write_batch() as outpoint_batch:
|
||||
self.flush_claims(claims_batch, signed_claims_batch, outpoint_batch)
|
||||
|
||||
def flush_claims(self, batch, signed_claims_batch, outpoint_batch):
|
||||
flush_start = time.time()
|
||||
write_claim, write_cert = batch.put, signed_claims_batch.put
|
||||
write_outpoint = outpoint_batch.put
|
||||
delete_claim, delete_outpoint = batch.delete, outpoint_batch.delete
|
||||
delete_cert = signed_claims_batch.delete
|
||||
for claim_id, outpoints in self.pending_abandons.items():
|
||||
claim = self.get_claim_info(claim_id)
|
||||
if claim.cert_id:
|
||||
self.remove_claim_from_certificate_claims(claim.cert_id, claim_id)
|
||||
self.remove_certificate(claim_id)
|
||||
self.claim_cache[claim_id] = None
|
||||
for txid, tx_index in outpoints:
|
||||
self.put_claim_id_for_outpoint(txid, tx_index, None)
|
||||
for key, claim in self.claim_cache.items():
|
||||
if claim:
|
||||
write_claim(key, claim)
|
||||
else:
|
||||
delete_claim(key)
|
||||
for cert_id, claims in self.claims_signed_by_cert_cache.items():
|
||||
if not claims:
|
||||
delete_cert(cert_id)
|
||||
else:
|
||||
write_cert(cert_id, msgpack.dumps(claims))
|
||||
for key, claim_id in self.outpoint_to_claim_id_cache.items():
|
||||
if claim_id:
|
||||
write_outpoint(key, claim_id)
|
||||
else:
|
||||
delete_outpoint(key)
|
||||
self.logger.info('flushed at height {:,d} with {:,d} claims, {:,d} outpoints '
|
||||
'and {:,d} certificates added while {:,d} were abandoned in {:.1f}s, committing...'
|
||||
.format(self.db_height,
|
||||
len(self.claim_cache), len(self.outpoint_to_claim_id_cache),
|
||||
len(self.claims_signed_by_cert_cache), len(self.pending_abandons),
|
||||
time.time() - flush_start))
|
||||
self.claim_cache = {}
|
||||
self.claims_signed_by_cert_cache = {}
|
||||
self.outpoint_to_claim_id_cache = {}
|
||||
self.pending_abandons = {}
|
||||
|
||||
def assert_flushed(self, flush_data):
|
||||
super().assert_flushed(flush_data)
|
||||
assert not self.claim_cache
|
||||
assert not self.claims_signed_by_cert_cache
|
||||
assert not self.outpoint_to_claim_id_cache
|
||||
assert not self.pending_abandons
|
||||
|
||||
def abandon_spent(self, tx_hash, tx_idx):
|
||||
claim_id = self.get_claim_id_from_outpoint(tx_hash, tx_idx)
|
||||
if claim_id:
|
||||
self.logger.info("[!] Abandon: {}".format(hash_to_hex_str(claim_id)))
|
||||
self.pending_abandons.setdefault(claim_id, []).append((tx_hash, tx_idx,))
|
||||
return claim_id
|
||||
|
||||
def put_claim_id_for_outpoint(self, tx_hash, tx_idx, claim_id):
|
||||
self.logger.info("[+] Adding outpoint: {}:{} for {}.".format(hash_to_hex_str(tx_hash), tx_idx,
|
||||
hash_to_hex_str(claim_id) if claim_id else None))
|
||||
self.outpoint_to_claim_id_cache[tx_hash + struct.pack('>I', tx_idx)] = claim_id
|
||||
|
||||
def remove_claim_id_for_outpoint(self, tx_hash, tx_idx):
|
||||
self.logger.info("[-] Remove outpoint: {}:{}.".format(hash_to_hex_str(tx_hash), tx_idx))
|
||||
self.outpoint_to_claim_id_cache[tx_hash + struct.pack('>I', tx_idx)] = None
|
||||
|
||||
def get_claim_id_from_outpoint(self, tx_hash, tx_idx):
|
||||
key = tx_hash + struct.pack('>I', tx_idx)
|
||||
return self.outpoint_to_claim_id_cache.get(key) or self.outpoint_to_claim_id_db.get(key)
|
||||
|
||||
def get_signed_claim_ids_by_cert_id(self, cert_id):
|
||||
if cert_id in self.claims_signed_by_cert_cache:
|
||||
return self.claims_signed_by_cert_cache[cert_id]
|
||||
db_claims = self.signatures_db.get(cert_id)
|
||||
return msgpack.loads(db_claims, use_list=True) if db_claims else []
|
||||
|
||||
def put_claim_id_signed_by_cert_id(self, cert_id, claim_id):
|
||||
msg = "[+] Adding signature: {} - {}".format(hash_to_hex_str(claim_id), hash_to_hex_str(cert_id))
|
||||
self.logger.info(msg)
|
||||
certs = self.get_signed_claim_ids_by_cert_id(cert_id)
|
||||
certs.append(claim_id)
|
||||
self.claims_signed_by_cert_cache[cert_id] = certs
|
||||
|
||||
def remove_certificate(self, cert_id):
|
||||
msg = "[-] Removing certificate: {}".format(hash_to_hex_str(cert_id))
|
||||
self.logger.info(msg)
|
||||
self.claims_signed_by_cert_cache[cert_id] = []
|
||||
|
||||
def remove_claim_from_certificate_claims(self, cert_id, claim_id):
|
||||
msg = "[-] Removing signature: {} - {}".format(hash_to_hex_str(claim_id), hash_to_hex_str(cert_id))
|
||||
self.logger.info(msg)
|
||||
certs = self.get_signed_claim_ids_by_cert_id(cert_id)
|
||||
if claim_id in certs:
|
||||
certs.remove(claim_id)
|
||||
self.claims_signed_by_cert_cache[cert_id] = certs
|
||||
|
||||
def get_claim_info(self, claim_id):
|
||||
serialized = self.claim_cache.get(claim_id) or self.claims_db.get(claim_id)
|
||||
return ClaimInfo.from_serialized(serialized) if serialized else None
|
||||
|
||||
def put_claim_info(self, claim_id, claim_info):
|
||||
self.logger.info("[+] Adding claim info for: {}".format(hash_to_hex_str(claim_id)))
|
||||
self.claim_cache[claim_id] = claim_info.serialized
|
||||
|
||||
def get_update_input(self, claim, inputs):
|
||||
claim_id = claim.claim_id
|
||||
claim_info = self.get_claim_info(claim_id)
|
||||
if not claim_info:
|
||||
return False
|
||||
for input in inputs:
|
||||
if input.prev_hash == claim_info.txid and input.prev_idx == claim_info.nout:
|
||||
return input
|
||||
return False
|
||||
|
||||
def write_undo(self, pending_undo):
|
||||
with self.claim_undo_db.write_batch() as writer:
|
||||
for height, undo_info in pending_undo:
|
||||
writer.put(struct.pack(">I", height), msgpack.dumps(undo_info))
|
||||
@@ -1,47 +0,0 @@
|
||||
from collections import namedtuple
|
||||
import msgpack
|
||||
from torba.server.util import cachedproperty
|
||||
# Classes representing data and their serializers, if any.
|
||||
|
||||
|
||||
class ClaimInfo(namedtuple("NameClaim", "name value txid nout amount address height cert_id")):
|
||||
'''Claim information as its stored on database'''
|
||||
|
||||
@classmethod
|
||||
def from_serialized(cls, serialized):
|
||||
return cls(*msgpack.loads(serialized))
|
||||
|
||||
@property
|
||||
def serialized(self):
|
||||
return msgpack.dumps(self)
|
||||
|
||||
|
||||
class NameClaim(namedtuple("NameClaim", "name value")):
|
||||
pass
|
||||
|
||||
|
||||
class ClaimUpdate(namedtuple("ClaimUpdate", "name claim_id value")):
|
||||
pass
|
||||
|
||||
|
||||
class ClaimSupport(namedtuple("ClaimSupport", "name claim_id")):
|
||||
pass
|
||||
|
||||
|
||||
class LBRYTx(namedtuple("Tx", "version inputs outputs locktime")):
|
||||
'''Transaction that can contain claim, update or support in its outputs.'''
|
||||
|
||||
@cachedproperty
|
||||
def is_coinbase(self):
|
||||
return self.inputs[0].is_coinbase
|
||||
|
||||
@cachedproperty
|
||||
def has_claims(self):
|
||||
for output in self.outputs:
|
||||
if output.claim:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class TxClaimOutput(namedtuple("TxClaimOutput", "value pk_script claim")):
|
||||
pass
|
||||
@@ -1,126 +0,0 @@
|
||||
import struct
|
||||
from torba.server.enum import Enumeration
|
||||
from lbrynet.extras.wallet.server.model import NameClaim, ClaimSupport, ClaimUpdate
|
||||
# TODO: Take this to lbryschema (it's also on lbryum and lbryum-server)
|
||||
|
||||
|
||||
opcodes = Enumeration("Opcodes", [
|
||||
("OP_0", 0), ("OP_PUSHDATA1", 76), "OP_PUSHDATA2", "OP_PUSHDATA4", "OP_1NEGATE", "OP_RESERVED",
|
||||
"OP_1", "OP_2", "OP_3", "OP_4", "OP_5", "OP_6", "OP_7",
|
||||
"OP_8", "OP_9", "OP_10", "OP_11", "OP_12", "OP_13", "OP_14", "OP_15", "OP_16",
|
||||
"OP_NOP", "OP_VER", "OP_IF", "OP_NOTIF", "OP_VERIF", "OP_VERNOTIF", "OP_ELSE", "OP_ENDIF",
|
||||
"OP_VERIFY",
|
||||
"OP_RETURN", "OP_TOALTSTACK", "OP_FROMALTSTACK", "OP_2DROP", "OP_2DUP", "OP_3DUP", "OP_2OVER",
|
||||
"OP_2ROT", "OP_2SWAP",
|
||||
"OP_IFDUP", "OP_DEPTH", "OP_DROP", "OP_DUP", "OP_NIP", "OP_OVER", "OP_PICK", "OP_ROLL",
|
||||
"OP_ROT",
|
||||
"OP_SWAP", "OP_TUCK", "OP_CAT", "OP_SUBSTR", "OP_LEFT", "OP_RIGHT", "OP_SIZE", "OP_INVERT",
|
||||
"OP_AND",
|
||||
"OP_OR", "OP_XOR", "OP_EQUAL", "OP_EQUALVERIFY", "OP_RESERVED1", "OP_RESERVED2", "OP_1ADD",
|
||||
"OP_1SUB", "OP_2MUL",
|
||||
"OP_2DIV", "OP_NEGATE", "OP_ABS", "OP_NOT", "OP_0NOTEQUAL", "OP_ADD", "OP_SUB", "OP_MUL",
|
||||
"OP_DIV",
|
||||
"OP_MOD", "OP_LSHIFT", "OP_RSHIFT", "OP_BOOLAND", "OP_BOOLOR",
|
||||
"OP_NUMEQUAL", "OP_NUMEQUALVERIFY", "OP_NUMNOTEQUAL", "OP_LESSTHAN",
|
||||
"OP_GREATERTHAN", "OP_LESSTHANOREQUAL", "OP_GREATERTHANOREQUAL", "OP_MIN", "OP_MAX",
|
||||
"OP_WITHIN", "OP_RIPEMD160", "OP_SHA1", "OP_SHA256", "OP_HASH160",
|
||||
"OP_HASH256", "OP_CODESEPARATOR", "OP_CHECKSIG", "OP_CHECKSIGVERIFY", "OP_CHECKMULTISIG",
|
||||
"OP_CHECKMULTISIGVERIFY", "OP_NOP1", "OP_NOP2", "OP_NOP3", "OP_NOP4", "OP_NOP5",
|
||||
"OP_CLAIM_NAME",
|
||||
"OP_SUPPORT_CLAIM", "OP_UPDATE_CLAIM",
|
||||
("OP_SINGLEBYTE_END", 0xF0),
|
||||
("OP_DOUBLEBYTE_BEGIN", 0xF000),
|
||||
"OP_PUBKEY", "OP_PUBKEYHASH",
|
||||
("OP_INVALIDOPCODE", 0xFFFF),
|
||||
])
|
||||
|
||||
|
||||
def script_GetOp(bytes):
|
||||
i = 0
|
||||
while i < len(bytes):
|
||||
vch = None
|
||||
opcode = bytes[i]
|
||||
i += 1
|
||||
if opcode <= opcodes.OP_PUSHDATA4:
|
||||
nSize = opcode
|
||||
if opcode == opcodes.OP_PUSHDATA1:
|
||||
nSize = bytes[i]
|
||||
i += 1
|
||||
elif opcode == opcodes.OP_PUSHDATA2:
|
||||
(nSize,) = struct.unpack_from('<H', bytes, i)
|
||||
i += 2
|
||||
elif opcode == opcodes.OP_PUSHDATA4:
|
||||
(nSize,) = struct.unpack_from('<I', bytes, i)
|
||||
i += 4
|
||||
if i + nSize > len(bytes):
|
||||
vch = "_INVALID_" + bytes[i:]
|
||||
i = len(bytes)
|
||||
else:
|
||||
vch = bytes[i:i + nSize]
|
||||
i += nSize
|
||||
yield (opcode, vch, i)
|
||||
|
||||
|
||||
def decode_claim_script(bytes_script):
|
||||
try:
|
||||
decoded_script = [x for x in script_GetOp(bytes_script)]
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return None
|
||||
if len(decoded_script) <= 6:
|
||||
return False
|
||||
op = 0
|
||||
claim_type = decoded_script[op][0]
|
||||
if claim_type == opcodes.OP_UPDATE_CLAIM:
|
||||
if len(decoded_script) <= 7:
|
||||
return False
|
||||
if claim_type not in [
|
||||
opcodes.OP_CLAIM_NAME,
|
||||
opcodes.OP_SUPPORT_CLAIM,
|
||||
opcodes.OP_UPDATE_CLAIM
|
||||
]:
|
||||
return False
|
||||
op += 1
|
||||
value = None
|
||||
claim_id = None
|
||||
claim = None
|
||||
if not 0 <= decoded_script[op][0] <= opcodes.OP_PUSHDATA4:
|
||||
return False
|
||||
name = decoded_script[op][1]
|
||||
op += 1
|
||||
if not 0 <= decoded_script[op][0] <= opcodes.OP_PUSHDATA4:
|
||||
return False
|
||||
if decoded_script[0][0] in [
|
||||
opcodes.OP_SUPPORT_CLAIM,
|
||||
opcodes.OP_UPDATE_CLAIM
|
||||
]:
|
||||
claim_id = decoded_script[op][1]
|
||||
if len(claim_id) != 20:
|
||||
return False
|
||||
else:
|
||||
value = decoded_script[op][1]
|
||||
op += 1
|
||||
if decoded_script[0][0] == opcodes.OP_UPDATE_CLAIM:
|
||||
value = decoded_script[op][1]
|
||||
op += 1
|
||||
if decoded_script[op][0] != opcodes.OP_2DROP:
|
||||
return False
|
||||
op += 1
|
||||
if decoded_script[op][0] != opcodes.OP_DROP and decoded_script[0][0] == opcodes.OP_CLAIM_NAME:
|
||||
return False
|
||||
elif decoded_script[op][0] != opcodes.OP_2DROP and decoded_script[0][0] == opcodes.OP_UPDATE_CLAIM:
|
||||
return False
|
||||
op += 1
|
||||
if decoded_script[0][0] == opcodes.OP_CLAIM_NAME:
|
||||
if name is None or value is None:
|
||||
return False
|
||||
claim = NameClaim(name, value)
|
||||
elif decoded_script[0][0] == opcodes.OP_UPDATE_CLAIM:
|
||||
if name is None or value is None or claim_id is None:
|
||||
return False
|
||||
claim = ClaimUpdate(name, claim_id, value)
|
||||
elif decoded_script[0][0] == opcodes.OP_SUPPORT_CLAIM:
|
||||
if name is None or claim_id is None:
|
||||
return False
|
||||
claim = ClaimSupport(name, claim_id)
|
||||
return claim, decoded_script[op:]
|
||||
@@ -1,21 +0,0 @@
|
||||
from torba.server.tx import Deserializer
|
||||
from lbrynet.extras.wallet.server.opcodes import decode_claim_script
|
||||
from lbrynet.extras.wallet.server.model import TxClaimOutput, LBRYTx
|
||||
|
||||
|
||||
class LBRYDeserializer(Deserializer):
|
||||
|
||||
def _read_output(self):
|
||||
value = self._read_le_int64()
|
||||
script = self._read_varbytes() # pk_script
|
||||
claim = decode_claim_script(script)
|
||||
claim = claim[0] if claim else None
|
||||
return TxClaimOutput(value, script, claim)
|
||||
|
||||
def read_tx(self):
|
||||
return LBRYTx(
|
||||
self._read_le_int32(), # version
|
||||
self._read_inputs(), # inputs
|
||||
self._read_outputs(), # outputs
|
||||
self._read_le_uint32() # locktime
|
||||
)
|
||||
@@ -1,6 +1,5 @@
|
||||
build:
|
||||
rm proto/* -rf
|
||||
cd proto && protoc --python_out=. -I ../../../../types/proto/ ../../../../types/proto/*.proto
|
||||
sed -e 's/^import\ \(.*\)_pb2\ /from . import\ \1_pb2\ /g' -i proto/*.py
|
||||
touch proto/__init__.py
|
||||
|
||||
rm types/v2/* -rf
|
||||
touch types/v2/__init__.py
|
||||
cd types/v2/ && protoc --python_out=. -I ../../../../../types/v2/proto/ ../../../../../types/v2/proto/*.proto
|
||||
sed -e 's/^import\ \(.*\)_pb2\ /from . import\ \1_pb2\ /g' -i types/v2/*.py
|
||||
@@ -1,13 +1 @@
|
||||
import os
|
||||
|
||||
__version__ = "0.0.16"
|
||||
|
||||
BLOCKCHAIN_NAME_ENVVAR = "LBRYSCHEMA_BLOCKCHAIN_NAME"
|
||||
if BLOCKCHAIN_NAME_ENVVAR in os.environ:
|
||||
if os.environ[BLOCKCHAIN_NAME_ENVVAR] in ['lbrycrd_main', 'lbrycrd_regtest',
|
||||
'lbrycrd_testnet']:
|
||||
BLOCKCHAIN_NAME = os.environ[BLOCKCHAIN_NAME_ENVVAR]
|
||||
else:
|
||||
raise OSError("invalid blockchain name: %s" % os.environ[BLOCKCHAIN_NAME_ENVVAR])
|
||||
else:
|
||||
BLOCKCHAIN_NAME = "lbrycrd_main"
|
||||
from .claim import Claim
|
||||
@@ -1,64 +0,0 @@
|
||||
import lbrynet.schema
|
||||
from lbrynet.schema.base import b58encode, b58decode, validate_b58_checksum
|
||||
from lbrynet.schema.hashing import double_sha256, hash160
|
||||
from lbrynet.schema.error import InvalidAddress
|
||||
from lbrynet.schema.schema import ADDRESS_LENGTH, ADDRESS_PREFIXES, PUBKEY_ADDRESS, SCRIPT_ADDRESS
|
||||
|
||||
|
||||
def validate_address_length(addr_bytes):
|
||||
if len(addr_bytes) != ADDRESS_LENGTH:
|
||||
raise InvalidAddress("Invalid address length: %i" % len(addr_bytes))
|
||||
|
||||
|
||||
def validate_address_prefix(addr_bytes):
|
||||
prefix = addr_bytes[0]
|
||||
if prefix not in ADDRESS_PREFIXES[lbrynet.schema.BLOCKCHAIN_NAME].values():
|
||||
raise InvalidAddress("Invalid address prefix: %.2X" % prefix)
|
||||
|
||||
|
||||
def validate_lbrycrd_address_bytes(addr_bytes):
|
||||
validate_address_length(addr_bytes)
|
||||
validate_address_prefix(addr_bytes)
|
||||
validate_b58_checksum(addr_bytes)
|
||||
return addr_bytes
|
||||
|
||||
|
||||
def decode_address(v):
|
||||
"""decode and validate a b58 address"""
|
||||
return validate_lbrycrd_address_bytes(b58decode(v))
|
||||
|
||||
|
||||
def encode_address(addr_bytes):
|
||||
"""validate and encode an address as b58"""
|
||||
v = validate_lbrycrd_address_bytes(addr_bytes)
|
||||
return b58encode(v)
|
||||
|
||||
|
||||
def hash_160_bytes_to_address(h160, addrtype=PUBKEY_ADDRESS):
|
||||
if addrtype == PUBKEY_ADDRESS:
|
||||
prefix = chr(ADDRESS_PREFIXES[lbrynet.schema.BLOCKCHAIN_NAME][PUBKEY_ADDRESS])
|
||||
elif addrtype == SCRIPT_ADDRESS:
|
||||
prefix = chr(ADDRESS_PREFIXES[lbrynet.schema.BLOCKCHAIN_NAME][SCRIPT_ADDRESS])
|
||||
else:
|
||||
raise Exception("Invalid address prefix")
|
||||
return b58encode(prefix + h160 + double_sha256(prefix + h160)[0:4])
|
||||
|
||||
|
||||
def public_key_to_address(public_key):
|
||||
return hash_160_bytes_to_address(hash160(public_key))
|
||||
|
||||
|
||||
def address_to_hash_160(addr):
|
||||
bytes = decode_address(addr)
|
||||
prefix, pubkey_bytes, addr_checksum = bytes[0], bytes[1:21], bytes[21:]
|
||||
if prefix == chr(ADDRESS_PREFIXES[lbrynet.schema.BLOCKCHAIN_NAME][PUBKEY_ADDRESS]):
|
||||
return PUBKEY_ADDRESS, pubkey_bytes
|
||||
return SCRIPT_ADDRESS, pubkey_bytes
|
||||
|
||||
|
||||
def is_address(addr):
|
||||
try:
|
||||
addr_bytes = decode_address(addr)
|
||||
return True
|
||||
except InvalidAddress:
|
||||
return False
|
||||
@@ -0,0 +1,521 @@
|
||||
import json
|
||||
import logging
|
||||
import os.path
|
||||
import hashlib
|
||||
from typing import Tuple, List
|
||||
from string import ascii_letters
|
||||
from decimal import Decimal, ROUND_UP
|
||||
from binascii import hexlify, unhexlify
|
||||
from google.protobuf.json_format import MessageToDict
|
||||
|
||||
from torba.client.hash import Base58
|
||||
from torba.client.constants import COIN
|
||||
|
||||
from lbrynet.schema.mime_types import guess_media_type
|
||||
from lbrynet.schema.base import Metadata, BaseMessageList
|
||||
from lbrynet.schema.types.v2.claim_pb2 import (
|
||||
Fee as FeeMessage,
|
||||
Location as LocationMessage,
|
||||
Language as LanguageMessage
|
||||
)
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def calculate_sha384_file_hash(file_path):
|
||||
sha384 = hashlib.sha384()
|
||||
with open(file_path, 'rb') as f:
|
||||
for chunk in iter(lambda: f.read(128 * sha384.block_size), b''):
|
||||
sha384.update(chunk)
|
||||
return sha384.digest()
|
||||
|
||||
|
||||
class Dimmensional(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def width(self) -> int:
|
||||
return self.message.width
|
||||
|
||||
@width.setter
|
||||
def width(self, width: int):
|
||||
self.message.width = width
|
||||
|
||||
@property
|
||||
def height(self) -> int:
|
||||
return self.message.height
|
||||
|
||||
@height.setter
|
||||
def height(self, height: int):
|
||||
self.message.height = height
|
||||
|
||||
@property
|
||||
def dimensions(self) -> Tuple[int, int]:
|
||||
return self.width, self.height
|
||||
|
||||
@dimensions.setter
|
||||
def dimensions(self, dimensions: Tuple[int, int]):
|
||||
self.message.width, self.message.height = dimensions
|
||||
|
||||
def _extract(self, file_metadata, field):
|
||||
try:
|
||||
setattr(self, field, file_metadata.getValues(field)[0])
|
||||
except:
|
||||
log.exception(f'Could not extract {field} from file metadata.')
|
||||
|
||||
def update(self, file_metadata=None, height=None, width=None):
|
||||
if height is not None:
|
||||
self.height = height
|
||||
elif file_metadata:
|
||||
self._extract(file_metadata, 'height')
|
||||
|
||||
if width is not None:
|
||||
self.width = width
|
||||
elif file_metadata:
|
||||
self._extract(file_metadata, 'width')
|
||||
|
||||
|
||||
class Playable(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def duration(self) -> int:
|
||||
return self.message.duration
|
||||
|
||||
@duration.setter
|
||||
def duration(self, duration: int):
|
||||
self.message.duration = duration
|
||||
|
||||
def update(self, file_metadata=None, duration=None):
|
||||
if duration is not None:
|
||||
self.duration = duration
|
||||
elif file_metadata:
|
||||
try:
|
||||
self.duration = file_metadata.getValues('duration')[0].seconds
|
||||
except:
|
||||
log.exception('Could not extract duration from file metadata.')
|
||||
|
||||
|
||||
class Image(Dimmensional):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
|
||||
class Audio(Playable):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
|
||||
class Video(Dimmensional, Playable):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def update(self, file_metadata=None, height=None, width=None, duration=None):
|
||||
Dimmensional.update(self, file_metadata, height, width)
|
||||
Playable.update(self, file_metadata, duration)
|
||||
|
||||
|
||||
class Source(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def update(self, file_path=None):
|
||||
if file_path is not None:
|
||||
self.name = os.path.basename(file_path)
|
||||
self.media_type, stream_type = guess_media_type(file_path)
|
||||
if not os.path.isfile(file_path):
|
||||
raise Exception(f"File does not exist: {file_path}")
|
||||
self.size = os.path.getsize(file_path)
|
||||
if self.size == 0:
|
||||
raise Exception(f"Cannot publish empty file: {file_path}")
|
||||
self.file_hash_bytes = calculate_sha384_file_hash(file_path)
|
||||
return stream_type
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self.message.name
|
||||
|
||||
@name.setter
|
||||
def name(self, name: str):
|
||||
self.message.name = name
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
return self.message.size
|
||||
|
||||
@size.setter
|
||||
def size(self, size: int):
|
||||
self.message.size = size
|
||||
|
||||
@property
|
||||
def media_type(self) -> str:
|
||||
return self.message.media_type
|
||||
|
||||
@media_type.setter
|
||||
def media_type(self, media_type: str):
|
||||
self.message.media_type = media_type
|
||||
|
||||
@property
|
||||
def file_hash(self) -> str:
|
||||
return hexlify(self.message.hash).decode()
|
||||
|
||||
@file_hash.setter
|
||||
def file_hash(self, file_hash: str):
|
||||
self.message.hash = unhexlify(file_hash.encode())
|
||||
|
||||
@property
|
||||
def file_hash_bytes(self) -> bytes:
|
||||
return self.message.hash
|
||||
|
||||
@file_hash_bytes.setter
|
||||
def file_hash_bytes(self, file_hash_bytes: bytes):
|
||||
self.message.hash = file_hash_bytes
|
||||
|
||||
@property
|
||||
def sd_hash(self) -> str:
|
||||
return hexlify(self.message.sd_hash).decode()
|
||||
|
||||
@sd_hash.setter
|
||||
def sd_hash(self, sd_hash: str):
|
||||
self.message.sd_hash = unhexlify(sd_hash.encode())
|
||||
|
||||
@property
|
||||
def sd_hash_bytes(self) -> bytes:
|
||||
return self.message.sd_hash
|
||||
|
||||
@sd_hash_bytes.setter
|
||||
def sd_hash_bytes(self, sd_hash: bytes):
|
||||
self.message.sd_hash = sd_hash
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
return self.message.url
|
||||
|
||||
@url.setter
|
||||
def url(self, url: str):
|
||||
self.message.url = url
|
||||
|
||||
|
||||
class Fee(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def update(self, address: str = None, currency: str = None, amount=None):
|
||||
if address is not None:
|
||||
self.address = address
|
||||
if currency is not None and amount is not None:
|
||||
currency = currency.lower()
|
||||
assert currency in ('lbc', 'btc', 'usd'), f'Unknown currency type: {currency}'
|
||||
setattr(self, currency, Decimal(amount))
|
||||
|
||||
@property
|
||||
def currency(self) -> str:
|
||||
return FeeMessage.Currency.Name(self.message.currency)
|
||||
|
||||
@property
|
||||
def address(self) -> str:
|
||||
return Base58.encode(self.message.address)
|
||||
|
||||
@address.setter
|
||||
def address(self, address: str):
|
||||
self.message.address = Base58.decode(address)
|
||||
|
||||
@property
|
||||
def address_bytes(self) -> bytes:
|
||||
return self.message.address
|
||||
|
||||
@address_bytes.setter
|
||||
def address_bytes(self, address: bytes):
|
||||
self.message.address = address
|
||||
|
||||
@property
|
||||
def amount(self) -> Decimal:
|
||||
if self.currency == 'LBC':
|
||||
return self.lbc
|
||||
if self.currency == 'BTC':
|
||||
return self.btc
|
||||
if self.currency == 'USD':
|
||||
return self.usd
|
||||
|
||||
DEWIES = Decimal(COIN)
|
||||
|
||||
@property
|
||||
def lbc(self) -> Decimal:
|
||||
if self.message.currency != FeeMessage.LBC:
|
||||
raise ValueError('LBC can only be returned for LBC fees.')
|
||||
return Decimal(self.message.amount / self.DEWIES)
|
||||
|
||||
@lbc.setter
|
||||
def lbc(self, amount: Decimal):
|
||||
self.dewies = int(amount * self.DEWIES)
|
||||
|
||||
@property
|
||||
def dewies(self) -> int:
|
||||
if self.message.currency != FeeMessage.LBC:
|
||||
raise ValueError('Dewies can only be returned for LBC fees.')
|
||||
return self.message.amount
|
||||
|
||||
@dewies.setter
|
||||
def dewies(self, amount: int):
|
||||
self.message.amount = amount
|
||||
self.message.currency = FeeMessage.LBC
|
||||
|
||||
SATOSHIES = Decimal(COIN)
|
||||
|
||||
@property
|
||||
def btc(self) -> Decimal:
|
||||
if self.message.currency != FeeMessage.BTC:
|
||||
raise ValueError('BTC can only be returned for BTC fees.')
|
||||
return Decimal(self.message.amount / self.SATOSHIES)
|
||||
|
||||
@btc.setter
|
||||
def btc(self, amount: Decimal):
|
||||
self.satoshis = int(amount * self.SATOSHIES)
|
||||
|
||||
@property
|
||||
def satoshis(self) -> int:
|
||||
if self.message.currency != FeeMessage.BTC:
|
||||
raise ValueError('Satoshies can only be returned for BTC fees.')
|
||||
return self.message.amount
|
||||
|
||||
@satoshis.setter
|
||||
def satoshis(self, amount: int):
|
||||
self.message.amount = amount
|
||||
self.message.currency = FeeMessage.BTC
|
||||
|
||||
PENNIES = Decimal('100.0')
|
||||
PENNY = Decimal('0.01')
|
||||
|
||||
@property
|
||||
def usd(self) -> Decimal:
|
||||
if self.message.currency != FeeMessage.USD:
|
||||
raise ValueError('USD can only be returned for USD fees.')
|
||||
return Decimal(self.message.amount / self.PENNIES)
|
||||
|
||||
@usd.setter
|
||||
def usd(self, amount: Decimal):
|
||||
self.pennies = int(amount.quantize(self.PENNY, ROUND_UP) * self.PENNIES)
|
||||
|
||||
@property
|
||||
def pennies(self) -> int:
|
||||
if self.message.currency != FeeMessage.USD:
|
||||
raise ValueError('Pennies can only be returned for USD fees.')
|
||||
return self.message.amount
|
||||
|
||||
@pennies.setter
|
||||
def pennies(self, amount: int):
|
||||
self.message.amount = amount
|
||||
self.message.currency = FeeMessage.USD
|
||||
|
||||
|
||||
class ClaimReference(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def claim_id(self) -> str:
|
||||
return hexlify(self.claim_hash[::-1]).decode()
|
||||
|
||||
@claim_id.setter
|
||||
def claim_id(self, claim_id: str):
|
||||
self.claim_hash = unhexlify(claim_id)[::-1]
|
||||
|
||||
@property
|
||||
def claim_hash(self) -> bytes:
|
||||
return self.message.claim_hash
|
||||
|
||||
@claim_hash.setter
|
||||
def claim_hash(self, claim_hash: bytes):
|
||||
self.message.claim_hash = claim_hash
|
||||
|
||||
|
||||
class ClaimList(BaseMessageList[ClaimReference]):
|
||||
|
||||
__slots__ = ()
|
||||
item_class = ClaimReference
|
||||
|
||||
@property
|
||||
def _message(self):
|
||||
return self.message.claim_references
|
||||
|
||||
def append(self, value):
|
||||
self.add().claim_id = value
|
||||
|
||||
@property
|
||||
def ids(self) -> List[str]:
|
||||
return [c.claim_id for c in self]
|
||||
|
||||
|
||||
class Language(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def langtag(self) -> str:
|
||||
langtag = []
|
||||
if self.language:
|
||||
langtag.append(self.language)
|
||||
if self.script:
|
||||
langtag.append(self.script)
|
||||
if self.region:
|
||||
langtag.append(self.region)
|
||||
return '-'.join(langtag)
|
||||
|
||||
@langtag.setter
|
||||
def langtag(self, langtag: str):
|
||||
parts = langtag.split('-')
|
||||
self.language = parts.pop(0)
|
||||
if parts and len(parts[0]) == 4:
|
||||
self.script = parts.pop(0)
|
||||
if parts and len(parts[0]) == 2:
|
||||
self.region = parts.pop(0)
|
||||
assert not parts, f"Failed to parse language tag: {langtag}"
|
||||
|
||||
@property
|
||||
def language(self) -> str:
|
||||
if self.message.language:
|
||||
return LanguageMessage.Language.Name(self.message.language)
|
||||
|
||||
@language.setter
|
||||
def language(self, language: str):
|
||||
self.message.language = LanguageMessage.Language.Value(language)
|
||||
|
||||
@property
|
||||
def script(self) -> str:
|
||||
if self.message.script:
|
||||
return LanguageMessage.Script.Name(self.message.script)
|
||||
|
||||
@script.setter
|
||||
def script(self, script: str):
|
||||
self.message.script = LanguageMessage.Script.Value(script)
|
||||
|
||||
@property
|
||||
def region(self) -> str:
|
||||
if self.message.region:
|
||||
return LocationMessage.Country.Name(self.message.region)
|
||||
|
||||
@region.setter
|
||||
def region(self, region: str):
|
||||
self.message.region = LocationMessage.Country.Value(region)
|
||||
|
||||
|
||||
class LanguageList(BaseMessageList[Language]):
|
||||
__slots__ = ()
|
||||
item_class = Language
|
||||
|
||||
def append(self, value: str):
|
||||
self.add().langtag = value
|
||||
|
||||
|
||||
class Location(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def from_value(self, value):
|
||||
if isinstance(value, str) and value.startswith('{'):
|
||||
value = json.loads(value)
|
||||
|
||||
if isinstance(value, dict):
|
||||
for key, val in value.items():
|
||||
setattr(self, key, val)
|
||||
|
||||
elif isinstance(value, str):
|
||||
parts = value.split(':')
|
||||
if len(parts) > 2 or (parts[0] and parts[0][0] in ascii_letters):
|
||||
country = parts and parts.pop(0)
|
||||
if country:
|
||||
self.country = country
|
||||
state = parts and parts.pop(0)
|
||||
if state:
|
||||
self.state = state
|
||||
city = parts and parts.pop(0)
|
||||
if city:
|
||||
self.city = city
|
||||
code = parts and parts.pop(0)
|
||||
if code:
|
||||
self.code = code
|
||||
latitude = parts and parts.pop(0)
|
||||
if latitude:
|
||||
self.latitude = latitude
|
||||
longitude = parts and parts.pop(0)
|
||||
if longitude:
|
||||
self.longitude = longitude
|
||||
|
||||
else:
|
||||
raise ValueError(f'Could not parse country value: {value}')
|
||||
|
||||
def to_dict(self):
|
||||
d = MessageToDict(self.message)
|
||||
if self.message.longitude:
|
||||
d['longitude'] = self.longitude
|
||||
if self.message.latitude:
|
||||
d['latitude'] = self.latitude
|
||||
return d
|
||||
|
||||
@property
|
||||
def country(self) -> str:
|
||||
if self.message.country:
|
||||
return LocationMessage.Country.Name(self.message.country)
|
||||
|
||||
@country.setter
|
||||
def country(self, country: str):
|
||||
self.message.country = LocationMessage.Country.Value(country)
|
||||
|
||||
@property
|
||||
def state(self) -> str:
|
||||
return self.message.state
|
||||
|
||||
@state.setter
|
||||
def state(self, state: str):
|
||||
self.message.state = state
|
||||
|
||||
@property
|
||||
def city(self) -> str:
|
||||
return self.message.city
|
||||
|
||||
@city.setter
|
||||
def city(self, city: str):
|
||||
self.message.city = city
|
||||
|
||||
@property
|
||||
def code(self) -> str:
|
||||
return self.message.code
|
||||
|
||||
@code.setter
|
||||
def code(self, code: str):
|
||||
self.message.code = code
|
||||
|
||||
GPS_PRECISION = Decimal('10000000')
|
||||
|
||||
@property
|
||||
def latitude(self) -> str:
|
||||
if self.message.latitude:
|
||||
return str(Decimal(self.message.latitude) / self.GPS_PRECISION)
|
||||
|
||||
@latitude.setter
|
||||
def latitude(self, latitude: str):
|
||||
latitude = Decimal(latitude)
|
||||
assert -90 <= latitude <= 90, "Latitude must be between -90 and 90 degrees."
|
||||
self.message.latitude = int(latitude * self.GPS_PRECISION)
|
||||
|
||||
@property
|
||||
def longitude(self) -> str:
|
||||
if self.message.longitude:
|
||||
return str(Decimal(self.message.longitude) / self.GPS_PRECISION)
|
||||
|
||||
@longitude.setter
|
||||
def longitude(self, longitude: str):
|
||||
longitude = Decimal(longitude)
|
||||
assert -180 <= longitude <= 180, "Longitude must be between -180 and 180 degrees."
|
||||
self.message.longitude = int(longitude * self.GPS_PRECISION)
|
||||
|
||||
|
||||
class LocationList(BaseMessageList[Location]):
|
||||
__slots__ = ()
|
||||
item_class = Location
|
||||
|
||||
def append(self, value):
|
||||
self.add().from_value(value)
|
||||
+103
-72
@@ -1,90 +1,121 @@
|
||||
from lbrynet.schema.schema import ADDRESS_CHECKSUM_LENGTH
|
||||
from lbrynet.schema.hashing import double_sha256
|
||||
from lbrynet.schema.error import InvalidAddress
|
||||
from binascii import hexlify, unhexlify
|
||||
from typing import List, Iterator, TypeVar, Generic
|
||||
|
||||
from google.protobuf.message import DecodeError
|
||||
from google.protobuf.json_format import MessageToDict
|
||||
|
||||
|
||||
alphabet = b'123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
|
||||
class Signable:
|
||||
|
||||
__slots__ = (
|
||||
'message', 'version', 'signature',
|
||||
'signature_type', 'unsigned_payload', 'signing_channel_hash'
|
||||
)
|
||||
|
||||
message_class = None
|
||||
|
||||
def __init__(self, message=None):
|
||||
self.message = message or self.message_class()
|
||||
self.version = 2
|
||||
self.signature = None
|
||||
self.signature_type = 'SECP256k1'
|
||||
self.unsigned_payload = None
|
||||
self.signing_channel_hash = None
|
||||
|
||||
def clear_signature(self):
|
||||
self.signature = None
|
||||
self.unsigned_payload = None
|
||||
self.signing_channel_hash = None
|
||||
|
||||
@property
|
||||
def signing_channel_id(self):
|
||||
return hexlify(self.signing_channel_hash[::-1]).decode() if self.signing_channel_hash else None
|
||||
|
||||
@signing_channel_id.setter
|
||||
def signing_channel_id(self, channel_id: str):
|
||||
self.signing_channel_hash = unhexlify(channel_id)[::-1]
|
||||
|
||||
@property
|
||||
def is_signed(self):
|
||||
return self.signature is not None
|
||||
|
||||
def to_dict(self):
|
||||
return MessageToDict(self.message)
|
||||
|
||||
def to_message_bytes(self) -> bytes:
|
||||
return self.message.SerializeToString()
|
||||
|
||||
def to_bytes(self) -> bytes:
|
||||
pieces = bytearray()
|
||||
if self.is_signed:
|
||||
pieces.append(1)
|
||||
pieces.extend(self.signing_channel_hash)
|
||||
pieces.extend(self.signature)
|
||||
else:
|
||||
pieces.append(0)
|
||||
pieces.extend(self.to_message_bytes())
|
||||
return bytes(pieces)
|
||||
|
||||
@classmethod
|
||||
def from_bytes(cls, data: bytes):
|
||||
signable = cls()
|
||||
if data[0] == 0:
|
||||
signable.message.ParseFromString(data[1:])
|
||||
elif data[0] == 1:
|
||||
signable.signing_channel_hash = data[1:21]
|
||||
signable.signature = data[21:85]
|
||||
signable.message.ParseFromString(data[85:])
|
||||
else:
|
||||
raise DecodeError('Could not determine message format version.')
|
||||
return signable
|
||||
|
||||
def __len__(self):
|
||||
return len(self.to_bytes())
|
||||
|
||||
def __bytes__(self):
|
||||
return self.to_bytes()
|
||||
|
||||
|
||||
def scrub_input(v):
|
||||
if isinstance(v, str) and not isinstance(v, bytes):
|
||||
v = v.encode('ascii')
|
||||
return v
|
||||
class Metadata:
|
||||
|
||||
__slots__ = 'message',
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
|
||||
|
||||
def b58encode_int(i, default_one=True):
|
||||
'''Encode an integer using Base58'''
|
||||
if not i and default_one:
|
||||
return alphabet[0:1]
|
||||
string = b""
|
||||
while i:
|
||||
i, idx = divmod(i, 58)
|
||||
string = alphabet[idx:idx+1] + string
|
||||
return string
|
||||
I = TypeVar('I')
|
||||
|
||||
|
||||
def b58encode(v):
|
||||
'''Encode a string using Base58'''
|
||||
class BaseMessageList(Metadata, Generic[I]):
|
||||
|
||||
v = scrub_input(v)
|
||||
__slots__ = ()
|
||||
|
||||
nPad = len(v)
|
||||
v = v.lstrip(b'\0')
|
||||
nPad -= len(v)
|
||||
item_class = None
|
||||
|
||||
p, acc = 1, 0
|
||||
for c in reversed(v):
|
||||
acc += p * c
|
||||
p = p << 8
|
||||
@property
|
||||
def _message(self):
|
||||
return self.message
|
||||
|
||||
result = b58encode_int(acc, default_one=False)
|
||||
def add(self) -> I:
|
||||
return self.item_class(self._message.add())
|
||||
|
||||
return alphabet[0:1] * nPad + result
|
||||
def extend(self, values: List[str]):
|
||||
for value in values:
|
||||
self.append(value)
|
||||
|
||||
def append(self, value: str):
|
||||
raise NotImplemented
|
||||
|
||||
def b58decode_int(v):
|
||||
'''Decode a Base58 encoded string as an integer'''
|
||||
def __len__(self):
|
||||
return len(self._message)
|
||||
|
||||
v = scrub_input(v)
|
||||
def __iter__(self) -> Iterator[I]:
|
||||
for item in self._message:
|
||||
yield self.item_class(item)
|
||||
|
||||
decimal = 0
|
||||
for char in v:
|
||||
decimal = decimal * 58 + alphabet.index(char)
|
||||
return decimal
|
||||
def __getitem__(self, item) -> I:
|
||||
return self.item_class(self._message[item])
|
||||
|
||||
|
||||
def b58decode(v):
|
||||
'''Decode a Base58 encoded string'''
|
||||
|
||||
v = scrub_input(v)
|
||||
|
||||
origlen = len(v)
|
||||
v = v.lstrip(alphabet[0:1])
|
||||
newlen = len(v)
|
||||
|
||||
acc = b58decode_int(v)
|
||||
|
||||
result = []
|
||||
while acc > 0:
|
||||
acc, mod = divmod(acc, 256)
|
||||
result.append(mod)
|
||||
|
||||
return b'\0' * (origlen - newlen) + bytes(reversed(result))
|
||||
|
||||
|
||||
def validate_b58_checksum(addr_bytes):
|
||||
addr_without_checksum = addr_bytes[:-ADDRESS_CHECKSUM_LENGTH]
|
||||
addr_checksum = addr_bytes[-ADDRESS_CHECKSUM_LENGTH:]
|
||||
if double_sha256(addr_without_checksum)[:ADDRESS_CHECKSUM_LENGTH] != addr_checksum:
|
||||
raise InvalidAddress("Invalid address checksum")
|
||||
|
||||
|
||||
def b58decode_strip_checksum(v):
|
||||
addr_bytes = b58decode(v)
|
||||
validate_b58_checksum(addr_bytes)
|
||||
return addr_bytes[:-ADDRESS_CHECKSUM_LENGTH]
|
||||
|
||||
|
||||
def b58encode_with_checksum(addr_bytes):
|
||||
addr_checksum = double_sha256(addr_bytes)[:ADDRESS_CHECKSUM_LENGTH]
|
||||
return b58encode(addr_bytes + addr_checksum)
|
||||
def __delitem__(self, key):
|
||||
del self._message[key]
|
||||
+361
-162
@@ -1,198 +1,397 @@
|
||||
import json
|
||||
import binascii
|
||||
from google.protobuf import json_format # pylint: disable=no-name-in-module
|
||||
from google.protobuf.message import DecodeError as DecodeError_pb # pylint: disable=no-name-in-module,import-error
|
||||
import logging
|
||||
from typing import List
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
from collections import OrderedDict
|
||||
from google.protobuf.json_format import MessageToDict
|
||||
from google.protobuf.message import DecodeError
|
||||
from hachoir.core.log import log as hachoir_log
|
||||
from hachoir.parser import createParser as binary_file_parser
|
||||
from hachoir.metadata import extractMetadata as binary_file_metadata
|
||||
|
||||
from lbrynet.schema.schema.claim import Claim
|
||||
from lbrynet.schema.proto import claim_pb2
|
||||
from lbrynet.schema.signature import Signature
|
||||
from lbrynet.schema.validator import get_validator
|
||||
from lbrynet.schema.signer import get_signer
|
||||
from lbrynet.schema.schema import NIST256p, CURVE_NAMES, CLAIM_TYPE_NAMES, SECP256k1
|
||||
from lbrynet.schema.encoding import decode_fields, decode_b64_fields, encode_fields
|
||||
from lbrynet.schema.error import DecodeError
|
||||
from lbrynet.schema.fee import Fee
|
||||
from lbrynet.schema import compat
|
||||
from lbrynet.schema.base import Signable
|
||||
from lbrynet.schema.mime_types import guess_media_type, guess_stream_type
|
||||
from lbrynet.schema.attrs import (
|
||||
Source, Playable, Dimmensional, Fee, Image, Video, Audio,
|
||||
LanguageList, LocationList, ClaimList, ClaimReference
|
||||
)
|
||||
from lbrynet.schema.types.v2.claim_pb2 import Claim as ClaimMessage
|
||||
|
||||
|
||||
class ClaimDict(OrderedDict):
|
||||
def __init__(self, claim_dict=None, detached_signature: Signature=None):
|
||||
if isinstance(claim_dict, claim_pb2.Claim):
|
||||
raise Exception("To initialize %s with a Claim protobuf use %s.load_protobuf" %
|
||||
(self.__class__.__name__, self.__class__.__name__))
|
||||
self.detached_signature = detached_signature
|
||||
OrderedDict.__init__(self, claim_dict or [])
|
||||
hachoir_log.use_print = False
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Claim(Signable):
|
||||
|
||||
STREAM = 'stream'
|
||||
CHANNEL = 'channel'
|
||||
COLLECTION = 'collection'
|
||||
REPOST = 'repost'
|
||||
|
||||
__slots__ = 'version',
|
||||
|
||||
message_class = ClaimMessage
|
||||
|
||||
def __init__(self, message=None):
|
||||
super().__init__(message)
|
||||
self.version = 2
|
||||
|
||||
@property
|
||||
def protobuf_dict(self):
|
||||
"""Claim dictionary using base64 to represent bytes"""
|
||||
def claim_type(self) -> str:
|
||||
return self.message.WhichOneof('type')
|
||||
|
||||
return json.loads(json_format.MessageToJson(self.protobuf, True))
|
||||
|
||||
@property
|
||||
def protobuf(self):
|
||||
"""Claim message object"""
|
||||
|
||||
return Claim.load(self)
|
||||
|
||||
@property
|
||||
def serialized(self):
|
||||
"""Serialized Claim protobuf"""
|
||||
if self.detached_signature:
|
||||
return self.detached_signature.serialized
|
||||
return self.protobuf.SerializeToString()
|
||||
|
||||
@property
|
||||
def serialized_no_signature(self):
|
||||
"""Serialized Claim protobuf without publisherSignature field"""
|
||||
claim = self.protobuf
|
||||
claim.ClearField("publisherSignature")
|
||||
return ClaimDict.load_protobuf(claim).serialized
|
||||
|
||||
@property
|
||||
def has_signature(self):
|
||||
return self.protobuf.HasField("publisherSignature") or (
|
||||
self.detached_signature and self.detached_signature.raw_signature
|
||||
)
|
||||
|
||||
@property
|
||||
def is_certificate(self):
|
||||
claim = self.protobuf
|
||||
return CLAIM_TYPE_NAMES[claim.claimType] == "certificate"
|
||||
def get_message(self, type_name):
|
||||
message = getattr(self.message, type_name)
|
||||
if self.claim_type is None:
|
||||
message.SetInParent()
|
||||
if self.claim_type != type_name:
|
||||
raise ValueError(f'Claim is not a {type_name}.')
|
||||
return message
|
||||
|
||||
@property
|
||||
def is_stream(self):
|
||||
claim = self.protobuf
|
||||
return CLAIM_TYPE_NAMES[claim.claimType] == "stream"
|
||||
return self.claim_type == self.STREAM
|
||||
|
||||
@property
|
||||
def source_hash(self):
|
||||
claim = self.protobuf
|
||||
if not CLAIM_TYPE_NAMES[claim.claimType] == "stream":
|
||||
return None
|
||||
return binascii.hexlify(claim.stream.source.source)
|
||||
def stream(self) -> 'Stream':
|
||||
return Stream(self)
|
||||
|
||||
@property
|
||||
def has_fee(self):
|
||||
claim = self.protobuf
|
||||
if not CLAIM_TYPE_NAMES[claim.claimType] == "stream":
|
||||
return None
|
||||
if claim.stream.metadata.HasField("fee"):
|
||||
return True
|
||||
return False
|
||||
def is_channel(self):
|
||||
return self.claim_type == self.CHANNEL
|
||||
|
||||
@property
|
||||
def source_fee(self):
|
||||
claim = self.protobuf
|
||||
if not CLAIM_TYPE_NAMES[claim.claimType] == "stream":
|
||||
return None
|
||||
if claim.stream.metadata.HasField("fee"):
|
||||
return Fee.load_protobuf(claim.stream.metadata.fee)
|
||||
return None
|
||||
def channel(self) -> 'Channel':
|
||||
return Channel(self)
|
||||
|
||||
@property
|
||||
def certificate_id(self) -> str:
|
||||
if self.protobuf.HasField("publisherSignature"):
|
||||
return binascii.hexlify(self.protobuf.publisherSignature.certificateId).decode()
|
||||
if self.detached_signature and self.detached_signature.certificate_id:
|
||||
return binascii.hexlify(self.detached_signature.certificate_id).decode()
|
||||
def is_repost(self):
|
||||
return self.claim_type == self.REPOST
|
||||
|
||||
@property
|
||||
def signature(self):
|
||||
if not self.has_signature:
|
||||
return None
|
||||
return binascii.hexlify(self.protobuf.publisherSignature.signature)
|
||||
def repost(self) -> 'Repost':
|
||||
return Repost(self)
|
||||
|
||||
@property
|
||||
def protobuf_len(self):
|
||||
"""Length of serialized string"""
|
||||
|
||||
return self.protobuf.ByteSize()
|
||||
def is_collection(self):
|
||||
return self.claim_type == self.COLLECTION
|
||||
|
||||
@property
|
||||
def json_len(self):
|
||||
"""Length of json encoded string"""
|
||||
|
||||
return len(json.dumps(self.claim_dict))
|
||||
|
||||
@property
|
||||
def claim_dict(self):
|
||||
"""Claim dictionary with bytes represented as hex and base58"""
|
||||
|
||||
return dict(encode_fields(self, self.detached_signature))
|
||||
def collection(self) -> 'Collection':
|
||||
return Collection(self)
|
||||
|
||||
@classmethod
|
||||
def load_protobuf_dict(cls, protobuf_dict, detached_signature=None):
|
||||
"""
|
||||
Load a ClaimDict from a dictionary with base64 encoded bytes
|
||||
(as returned by the protobuf json formatter)
|
||||
"""
|
||||
|
||||
return cls(decode_b64_fields(protobuf_dict), detached_signature=detached_signature)
|
||||
|
||||
@classmethod
|
||||
def load_protobuf(cls, protobuf_claim, detached_signature=None):
|
||||
"""Load ClaimDict from a protobuf Claim message"""
|
||||
return cls.load_protobuf_dict(json.loads(json_format.MessageToJson(protobuf_claim, True)), detached_signature)
|
||||
|
||||
@classmethod
|
||||
def load_dict(cls, claim_dict):
|
||||
"""Load ClaimDict from a dictionary with hex and base58 encoded bytes"""
|
||||
def from_bytes(cls, data: bytes) -> 'Claim':
|
||||
try:
|
||||
claim_dict, detached_signature = decode_fields(claim_dict)
|
||||
return cls.load_protobuf(cls(claim_dict).protobuf, detached_signature)
|
||||
except json_format.ParseError as err:
|
||||
raise DecodeError(str(err))
|
||||
return super().from_bytes(data)
|
||||
except DecodeError:
|
||||
claim = cls()
|
||||
if data[0] == ord('{'):
|
||||
claim.version = 0
|
||||
compat.from_old_json_schema(claim, data)
|
||||
elif data[0] not in (0, 1):
|
||||
claim.version = 1
|
||||
compat.from_types_v1(claim, data)
|
||||
else:
|
||||
raise
|
||||
return claim
|
||||
|
||||
@classmethod
|
||||
def deserialize(cls, serialized):
|
||||
"""Load a ClaimDict from a serialized protobuf string"""
|
||||
detached_signature = Signature.flagged_parse(serialized)
|
||||
|
||||
temp_claim = claim_pb2.Claim()
|
||||
try:
|
||||
temp_claim.ParseFromString(detached_signature.payload)
|
||||
except DecodeError_pb:
|
||||
raise DecodeError(DecodeError_pb)
|
||||
return cls.load_protobuf(temp_claim, detached_signature=detached_signature)
|
||||
class BaseClaim:
|
||||
|
||||
@classmethod
|
||||
def generate_certificate(cls, private_key, curve=SECP256k1):
|
||||
signer = get_signer(curve).load_pem(private_key)
|
||||
return cls.load_protobuf(signer.certificate)
|
||||
__slots__ = 'claim', 'message'
|
||||
|
||||
def sign(self, private_key, claim_address, cert_claim_id, curve=SECP256k1, name=None, force_detached=False):
|
||||
signer = get_signer(curve).load_pem(private_key)
|
||||
signed, signature = signer.sign_stream_claim(self, claim_address, cert_claim_id, name, force_detached)
|
||||
return ClaimDict.load_protobuf(signed, signature)
|
||||
claim_type = None
|
||||
object_fields = 'thumbnail',
|
||||
repeat_fields = 'tags', 'languages', 'locations'
|
||||
|
||||
def validate_signature(self, claim_address, certificate, name=None):
|
||||
if isinstance(certificate, ClaimDict):
|
||||
certificate = certificate.protobuf
|
||||
curve = CURVE_NAMES[certificate.certificate.keyType]
|
||||
validator = get_validator(curve).load_from_certificate(certificate, self.certificate_id)
|
||||
return validator.validate_claim_signature(self, claim_address, name)
|
||||
def __init__(self, claim: Claim = None):
|
||||
self.claim = claim or Claim()
|
||||
self.message = self.claim.get_message(self.claim_type)
|
||||
|
||||
def validate_private_key(self, private_key, certificate_id):
|
||||
certificate = self.protobuf
|
||||
if CLAIM_TYPE_NAMES[certificate.claimType] != "certificate":
|
||||
return
|
||||
curve = CURVE_NAMES[certificate.certificate.keyType]
|
||||
validator = get_validator(curve).load_from_certificate(certificate, certificate_id)
|
||||
signing_key = validator.signing_key_from_pem(private_key)
|
||||
return validator.validate_private_key(signing_key)
|
||||
def to_dict(self):
|
||||
claim = MessageToDict(self.claim.message, preserving_proto_field_name=True)
|
||||
claim.update(claim.pop(self.claim_type))
|
||||
if 'languages' in claim:
|
||||
claim['languages'] = self.langtags
|
||||
if 'locations' in claim:
|
||||
claim['locations'] = [l.to_dict() for l in self.locations]
|
||||
return claim
|
||||
|
||||
def get_validator(self, certificate_id):
|
||||
"""
|
||||
Get a lbrynet.schema.validator.Validator object for a certificate claim
|
||||
def update(self, **kwargs):
|
||||
for key in list(kwargs):
|
||||
for field in self.object_fields:
|
||||
if key.startswith(f'{field}_'):
|
||||
attr = getattr(self, field)
|
||||
setattr(attr, key[len(f'{field}_'):], kwargs.pop(key))
|
||||
continue
|
||||
|
||||
:param certificate_id: claim id of this certificate claim
|
||||
:return: None or lbrynet.schema.validator.Validator object
|
||||
"""
|
||||
for l in self.repeat_fields:
|
||||
field = getattr(self, l)
|
||||
if kwargs.pop(f'clear_{l}', False):
|
||||
del field[:]
|
||||
items = kwargs.pop(l, None)
|
||||
if items is not None:
|
||||
if isinstance(items, str):
|
||||
field.append(items)
|
||||
elif isinstance(items, list):
|
||||
field.extend(items)
|
||||
else:
|
||||
raise ValueError(f"Unknown {l} value: {items}")
|
||||
|
||||
claim = self.protobuf
|
||||
if CLAIM_TYPE_NAMES[claim.claimType] != "certificate":
|
||||
return
|
||||
curve = CURVE_NAMES[claim.certificate.keyType]
|
||||
return get_validator(curve).load_from_certificate(claim, certificate_id)
|
||||
for key, value in kwargs.items():
|
||||
setattr(self, key, value)
|
||||
|
||||
@property
|
||||
def title(self) -> str:
|
||||
return self.claim.message.title
|
||||
|
||||
@title.setter
|
||||
def title(self, title: str):
|
||||
self.claim.message.title = title
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
return self.claim.message.description
|
||||
|
||||
@description.setter
|
||||
def description(self, description: str):
|
||||
self.claim.message.description = description
|
||||
|
||||
@property
|
||||
def thumbnail(self) -> Source:
|
||||
return Source(self.claim.message.thumbnail)
|
||||
|
||||
@property
|
||||
def tags(self) -> List:
|
||||
return self.claim.message.tags
|
||||
|
||||
@property
|
||||
def languages(self) -> LanguageList:
|
||||
return LanguageList(self.claim.message.languages)
|
||||
|
||||
@property
|
||||
def langtags(self) -> List[str]:
|
||||
return [l.langtag for l in self.languages]
|
||||
|
||||
@property
|
||||
def locations(self) -> LocationList:
|
||||
return LocationList(self.claim.message.locations)
|
||||
|
||||
|
||||
class Stream(BaseClaim):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
claim_type = Claim.STREAM
|
||||
|
||||
object_fields = BaseClaim.object_fields + ('source',)
|
||||
|
||||
def to_dict(self):
|
||||
claim = super().to_dict()
|
||||
if 'source' in claim:
|
||||
if 'hash' in claim['source']:
|
||||
claim['source']['hash'] = self.source.file_hash
|
||||
if 'sd_hash' in claim['source']:
|
||||
claim['source']['sd_hash'] = self.source.sd_hash
|
||||
if 'media_type' in claim['source']:
|
||||
claim['stream_type'] = guess_stream_type(claim['source']['media_type'])
|
||||
fee = claim.get('fee', {})
|
||||
if 'address' in fee:
|
||||
fee['address'] = self.fee.address
|
||||
if 'amount' in fee:
|
||||
fee['amount'] = str(self.fee.amount)
|
||||
return claim
|
||||
|
||||
def update(self, file_path=None, height=None, width=None, duration=None, **kwargs):
|
||||
if kwargs.pop('clear_fee', False):
|
||||
self.message.ClearField('fee')
|
||||
else:
|
||||
self.fee.update(
|
||||
kwargs.pop('fee_address', None),
|
||||
kwargs.pop('fee_currency', None),
|
||||
kwargs.pop('fee_amount', None)
|
||||
)
|
||||
|
||||
if 'sd_hash' in kwargs:
|
||||
self.source.sd_hash = kwargs.pop('sd_hash')
|
||||
if 'file_name' in kwargs:
|
||||
self.source.name = kwargs.pop('file_name')
|
||||
if 'file_hash' in kwargs:
|
||||
self.source.file_hash = kwargs.pop('file_hash')
|
||||
|
||||
stream_type = None
|
||||
if file_path is not None:
|
||||
stream_type = self.source.update(file_path=file_path)
|
||||
elif self.source.name:
|
||||
self.source.media_type, stream_type = guess_media_type(self.source.name)
|
||||
elif self.source.media_type:
|
||||
stream_type = guess_stream_type(self.source.media_type)
|
||||
|
||||
if 'file_size' in kwargs:
|
||||
self.source.size = kwargs.pop('file_size')
|
||||
|
||||
if stream_type in ('image', 'video', 'audio'):
|
||||
media = getattr(self, stream_type)
|
||||
media_args = {'file_metadata': None}
|
||||
if file_path is not None:
|
||||
try:
|
||||
media_args['file_metadata'] = binary_file_metadata(binary_file_parser(file_path))
|
||||
except:
|
||||
log.exception('Could not read file metadata.')
|
||||
if isinstance(media, Playable):
|
||||
media_args['duration'] = duration
|
||||
if isinstance(media, Dimmensional):
|
||||
media_args['height'] = height
|
||||
media_args['width'] = width
|
||||
media.update(**media_args)
|
||||
|
||||
super().update(**kwargs)
|
||||
|
||||
@property
|
||||
def author(self) -> str:
|
||||
return self.message.author
|
||||
|
||||
@author.setter
|
||||
def author(self, author: str):
|
||||
self.message.author = author
|
||||
|
||||
@property
|
||||
def license(self) -> str:
|
||||
return self.message.license
|
||||
|
||||
@license.setter
|
||||
def license(self, license: str):
|
||||
self.message.license = license
|
||||
|
||||
@property
|
||||
def license_url(self) -> str:
|
||||
return self.message.license_url
|
||||
|
||||
@license_url.setter
|
||||
def license_url(self, license_url: str):
|
||||
self.message.license_url = license_url
|
||||
|
||||
@property
|
||||
def release_time(self) -> int:
|
||||
return self.message.release_time
|
||||
|
||||
@release_time.setter
|
||||
def release_time(self, release_time: int):
|
||||
self.message.release_time = release_time
|
||||
|
||||
@property
|
||||
def fee(self) -> Fee:
|
||||
return Fee(self.message.fee)
|
||||
|
||||
@property
|
||||
def has_fee(self) -> bool:
|
||||
return self.message.HasField('fee')
|
||||
|
||||
@property
|
||||
def source(self) -> Source:
|
||||
return Source(self.message.source)
|
||||
|
||||
@property
|
||||
def stream_type(self) -> str:
|
||||
return self.message.WhichOneof('type')
|
||||
|
||||
@property
|
||||
def image(self) -> Image:
|
||||
return Image(self.message.image)
|
||||
|
||||
@property
|
||||
def video(self) -> Video:
|
||||
return Video(self.message.video)
|
||||
|
||||
@property
|
||||
def audio(self) -> Audio:
|
||||
return Audio(self.message.audio)
|
||||
|
||||
|
||||
class Channel(BaseClaim):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
claim_type = Claim.CHANNEL
|
||||
|
||||
object_fields = BaseClaim.object_fields + ('cover',)
|
||||
repeat_fields = BaseClaim.repeat_fields + ('featured',)
|
||||
|
||||
def to_dict(self):
|
||||
claim = super().to_dict()
|
||||
claim['public_key'] = self.public_key
|
||||
if 'featured' in claim:
|
||||
claim['featured'] = self.featured.ids
|
||||
return claim
|
||||
|
||||
@property
|
||||
def public_key(self) -> str:
|
||||
return hexlify(self.message.public_key).decode()
|
||||
|
||||
@public_key.setter
|
||||
def public_key(self, sd_public_key: str):
|
||||
self.message.public_key = unhexlify(sd_public_key.encode())
|
||||
|
||||
@property
|
||||
def public_key_bytes(self) -> bytes:
|
||||
return self.message.public_key
|
||||
|
||||
@public_key_bytes.setter
|
||||
def public_key_bytes(self, public_key: bytes):
|
||||
self.message.public_key = public_key
|
||||
|
||||
@property
|
||||
def email(self) -> str:
|
||||
return self.message.email
|
||||
|
||||
@email.setter
|
||||
def email(self, email: str):
|
||||
self.message.email = email
|
||||
|
||||
@property
|
||||
def website_url(self) -> str:
|
||||
return self.message.website_url
|
||||
|
||||
@website_url.setter
|
||||
def website_url(self, website_url: str):
|
||||
self.message.website_url = website_url
|
||||
|
||||
@property
|
||||
def cover(self) -> Source:
|
||||
return Source(self.message.cover)
|
||||
|
||||
@property
|
||||
def featured(self) -> ClaimList:
|
||||
return ClaimList(self.message.featured)
|
||||
|
||||
|
||||
class Repost(BaseClaim):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
claim_type = Claim.REPOST
|
||||
|
||||
@property
|
||||
def reference(self) -> ClaimReference:
|
||||
return ClaimReference(self.message)
|
||||
|
||||
|
||||
class Collection(BaseClaim):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
claim_type = Claim.COLLECTION
|
||||
|
||||
repeat_fields = BaseClaim.repeat_fields + ('claims',)
|
||||
|
||||
def to_dict(self):
|
||||
claim = super().to_dict()
|
||||
if 'claim_references' in claim:
|
||||
claim['claim_references'] = self.claims.ids
|
||||
return claim
|
||||
|
||||
@property
|
||||
def claims(self) -> ClaimList:
|
||||
return ClaimList(self.message)
|
||||
@@ -0,0 +1,93 @@
|
||||
import json
|
||||
from decimal import Decimal
|
||||
|
||||
from google.protobuf.message import DecodeError
|
||||
|
||||
from lbrynet.schema.types.v1.legacy_claim_pb2 import Claim as OldClaimMessage
|
||||
from lbrynet.schema.types.v1.certificate_pb2 import KeyType
|
||||
from lbrynet.schema.types.v1.fee_pb2 import Fee as FeeMessage
|
||||
|
||||
|
||||
def from_old_json_schema(claim, payload: bytes):
|
||||
try:
|
||||
value = json.loads(payload)
|
||||
except:
|
||||
raise DecodeError('Could not parse JSON.')
|
||||
stream = claim.stream
|
||||
stream.source.sd_hash = value['sources']['lbry_sd_hash']
|
||||
stream.source.media_type = (
|
||||
value.get('content_type', value.get('content-type')) or
|
||||
'application/octet-stream'
|
||||
)
|
||||
stream.title = value.get('title', '')
|
||||
stream.description = value.get('description', '')
|
||||
if value.get('thumbnail', ''):
|
||||
stream.thumbnail.url = value.get('thumbnail', '')
|
||||
stream.author = value.get('author', '')
|
||||
stream.license = value.get('license', '')
|
||||
stream.license_url = value.get('license_url', '')
|
||||
language = value.get('language', '')
|
||||
if language:
|
||||
if language.lower() == 'english':
|
||||
language = 'en'
|
||||
try:
|
||||
stream.languages.append(language)
|
||||
except:
|
||||
pass
|
||||
if value.get('nsfw', False):
|
||||
stream.tags.append('mature')
|
||||
if "fee" in value:
|
||||
fee = value["fee"]
|
||||
currency = list(fee.keys())[0]
|
||||
if currency == 'LBC':
|
||||
stream.fee.lbc = Decimal(fee[currency]['amount'])
|
||||
elif currency == 'USD':
|
||||
stream.fee.usd = Decimal(fee[currency]['amount'])
|
||||
elif currency == 'BTC':
|
||||
stream.fee.btc = Decimal(fee[currency]['amount'])
|
||||
else:
|
||||
raise DecodeError(f'Unknown currency: {currency}')
|
||||
stream.fee.address = fee[currency]['address']
|
||||
return claim
|
||||
|
||||
|
||||
def from_types_v1(claim, payload: bytes):
|
||||
old = OldClaimMessage()
|
||||
old.ParseFromString(payload)
|
||||
if old.claimType == 2:
|
||||
channel = claim.channel
|
||||
channel.public_key_bytes = old.certificate.publicKey
|
||||
else:
|
||||
stream = claim.stream
|
||||
stream.title = old.stream.metadata.title
|
||||
stream.description = old.stream.metadata.description
|
||||
stream.author = old.stream.metadata.author
|
||||
stream.license = old.stream.metadata.license
|
||||
stream.license_url = old.stream.metadata.licenseUrl
|
||||
stream.thumbnail.url = old.stream.metadata.thumbnail
|
||||
if old.stream.metadata.HasField('language'):
|
||||
stream.languages.add().message.language = old.stream.metadata.language
|
||||
stream.source.media_type = old.stream.source.contentType
|
||||
stream.source.sd_hash_bytes = old.stream.source.source
|
||||
if old.stream.metadata.nsfw:
|
||||
stream.tags.append('mature')
|
||||
if old.stream.metadata.HasField('fee'):
|
||||
fee = old.stream.metadata.fee
|
||||
stream.fee.address_bytes = fee.address
|
||||
currency = FeeMessage.Currency.Name(fee.currency)
|
||||
if currency == 'LBC':
|
||||
stream.fee.lbc = Decimal(fee.amount)
|
||||
elif currency == 'USD':
|
||||
stream.fee.usd = Decimal(fee.amount)
|
||||
elif currency == 'BTC':
|
||||
stream.fee.btc = Decimal(fee.amount)
|
||||
else:
|
||||
raise DecodeError(f'Unsupported currency: {currency}')
|
||||
if old.HasField('publisherSignature'):
|
||||
sig = old.publisherSignature
|
||||
claim.signature = sig.signature
|
||||
claim.signature_type = KeyType.Name(sig.signatureType)
|
||||
claim.signing_channel_hash = sig.certificateId[::-1]
|
||||
old.ClearField("publisherSignature")
|
||||
claim.unsigned_payload = old.SerializeToString()
|
||||
return claim
|
||||
@@ -1,61 +0,0 @@
|
||||
import json
|
||||
import binascii
|
||||
from google.protobuf import json_format # pylint: disable=no-name-in-module
|
||||
|
||||
from lbrynet.schema.error import DecodeError, InvalidAddress
|
||||
from lbrynet.schema.legacy.migrate import migrate as schema_migrator
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
|
||||
|
||||
|
||||
def migrate_json_claim_value(decoded_json):
|
||||
try:
|
||||
if 'fee' in decoded_json:
|
||||
old_fee = decoded_json['fee']
|
||||
if not old_fee[list(old_fee.keys())[0]]['amount']:
|
||||
del decoded_json['fee']
|
||||
return migrate_json_claim_value(decoded_json)
|
||||
except (TypeError, AttributeError, InvalidAddress):
|
||||
raise DecodeError("Failed to decode claim")
|
||||
try:
|
||||
pb_migrated = schema_migrator(decoded_json)
|
||||
return pb_migrated
|
||||
except json_format.ParseError as parse_error:
|
||||
raise DecodeError("Failed to parse protobuf: %s" % parse_error)
|
||||
except Exception as err:
|
||||
raise DecodeError("Failed to migrate claim: %s" % err)
|
||||
|
||||
|
||||
def smart_decode(claim_value):
|
||||
"""
|
||||
Decode a claim value
|
||||
|
||||
Try decoding claim protobuf, if this fails try decoding json and migrating it.
|
||||
If unable to decode or migrate, raise DecodeError
|
||||
"""
|
||||
|
||||
# if already decoded, return
|
||||
if isinstance(claim_value, ClaimDict):
|
||||
return claim_value
|
||||
elif isinstance(claim_value, dict):
|
||||
return ClaimDict.load_dict(claim_value)
|
||||
|
||||
try:
|
||||
claim_value = binascii.unhexlify(claim_value)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
if claim_value[0] in ['{', ord('{')]:
|
||||
try:
|
||||
if isinstance(claim_value, bytes):
|
||||
claim_value = claim_value.decode()
|
||||
decoded_json = json.loads(claim_value)
|
||||
return migrate_json_claim_value(decoded_json)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
try:
|
||||
if isinstance(claim_value, str):
|
||||
claim_value = claim_value.encode()
|
||||
return ClaimDict.deserialize(claim_value)
|
||||
except (DecodeError, InvalidAddress, KeyError, TypeError):
|
||||
raise DecodeError(claim_value)
|
||||
@@ -1,90 +0,0 @@
|
||||
import base64, binascii
|
||||
from copy import deepcopy
|
||||
from lbrynet.schema.address import decode_address, encode_address
|
||||
from lbrynet.schema.schema import CLAIM_TYPES, CLAIM_TYPE, STREAM_TYPE, CERTIFICATE_TYPE
|
||||
from lbrynet.schema.schema import SIGNATURE
|
||||
from lbrynet.schema.error import DecodeError, InvalidAddress
|
||||
from lbrynet.schema.signature import Signature
|
||||
|
||||
|
||||
def encode_fields(claim_dictionary, detached_signature: Signature):
|
||||
"""Encode bytes to hex and b58 for return by ClaimDict"""
|
||||
claim_dictionary = deepcopy(claim_dictionary)
|
||||
claim_type = CLAIM_TYPES[claim_dictionary[CLAIM_TYPE]]
|
||||
claim_value = claim_dictionary[claim_type]
|
||||
if claim_type == CLAIM_TYPES[STREAM_TYPE]:
|
||||
claim_value['source']['source'] = binascii.hexlify(claim_value['source']['source']).decode()
|
||||
if 'fee' in claim_value['metadata']:
|
||||
try:
|
||||
address = encode_address(claim_value['metadata']['fee']['address'])
|
||||
except InvalidAddress as err:
|
||||
raise DecodeError("Invalid fee address: %s" % err)
|
||||
claim_value['metadata']['fee']['address'] = address
|
||||
elif claim_type == CLAIM_TYPES[CERTIFICATE_TYPE]:
|
||||
public_key = claim_value["publicKey"]
|
||||
claim_value["publicKey"] = binascii.hexlify(public_key).decode()
|
||||
if SIGNATURE in claim_dictionary:
|
||||
encoded_sig = binascii.hexlify(claim_dictionary[SIGNATURE]['signature']).decode()
|
||||
encoded_cert_id = binascii.hexlify(claim_dictionary[SIGNATURE]['certificateId']).decode()
|
||||
claim_dictionary[SIGNATURE]['signature'] = encoded_sig
|
||||
claim_dictionary[SIGNATURE]['certificateId'] = encoded_cert_id
|
||||
elif detached_signature and detached_signature.raw_signature:
|
||||
claim_dictionary[SIGNATURE] = {
|
||||
'detached_signature': binascii.hexlify(detached_signature.serialized).decode(),
|
||||
'certificateId': binascii.hexlify(detached_signature.certificate_id).decode()
|
||||
}
|
||||
|
||||
claim_dictionary[claim_type] = claim_value
|
||||
return claim_dictionary
|
||||
|
||||
|
||||
def decode_fields(claim_dictionary):
|
||||
"""Decode hex and b58 encoded bytes in dictionaries given to ClaimDict"""
|
||||
detached_signature = None
|
||||
claim_dictionary = deepcopy(claim_dictionary)
|
||||
claim_type = CLAIM_TYPES[claim_dictionary[CLAIM_TYPE]]
|
||||
claim_value = claim_dictionary[claim_type]
|
||||
if claim_type == CLAIM_TYPES[STREAM_TYPE]:
|
||||
claim_value['source']['source'] = binascii.unhexlify(claim_value['source']['source'])
|
||||
if 'fee' in claim_value['metadata']:
|
||||
try:
|
||||
address = decode_address(claim_value['metadata']['fee']['address'])
|
||||
except InvalidAddress as err:
|
||||
raise DecodeError("Invalid fee address: %s" % err)
|
||||
claim_value['metadata']['fee']['address'] = address
|
||||
elif claim_type == CLAIM_TYPES[CERTIFICATE_TYPE]:
|
||||
public_key = binascii.unhexlify(claim_value["publicKey"])
|
||||
claim_value["publicKey"] = public_key
|
||||
if SIGNATURE in claim_dictionary and not claim_dictionary[SIGNATURE].get('detached_signature'):
|
||||
decoded_sig = binascii.unhexlify(claim_dictionary[SIGNATURE]['signature'])
|
||||
decoded_cert_id = binascii.unhexlify(claim_dictionary[SIGNATURE]['certificateId'])
|
||||
claim_dictionary[SIGNATURE]['signature'] = decoded_sig
|
||||
claim_dictionary[SIGNATURE]['certificateId'] = decoded_cert_id
|
||||
elif claim_dictionary.get(SIGNATURE, {}).get('detached_signature'):
|
||||
hex_detached_signature = claim_dictionary[SIGNATURE]['detached_signature']
|
||||
detached_signature = Signature.flagged_parse(binascii.unhexlify(hex_detached_signature))
|
||||
del claim_dictionary[SIGNATURE]
|
||||
claim_dictionary[claim_type] = claim_value
|
||||
return claim_dictionary, detached_signature
|
||||
|
||||
|
||||
def decode_b64_fields(claim_dictionary):
|
||||
"""Decode b64 encoded bytes in protobuf generated dictionary to be given to ClaimDict"""
|
||||
claim_dictionary = deepcopy(claim_dictionary)
|
||||
claim_type = CLAIM_TYPES[claim_dictionary[CLAIM_TYPE]]
|
||||
claim_value = claim_dictionary[claim_type]
|
||||
if claim_type == CLAIM_TYPES[STREAM_TYPE]:
|
||||
claim_value['source']['source'] = base64.b64decode(claim_value['source']['source'])
|
||||
if 'fee' in claim_value['metadata']:
|
||||
address = base64.b64decode(claim_value['metadata']['fee']['address'])
|
||||
claim_value['metadata']['fee']['address'] = address
|
||||
elif claim_type == CLAIM_TYPES[CERTIFICATE_TYPE]:
|
||||
public_key = base64.b64decode(claim_value["publicKey"])
|
||||
claim_value["publicKey"] = public_key
|
||||
if SIGNATURE in claim_dictionary:
|
||||
encoded_sig = base64.b64decode(claim_dictionary[SIGNATURE]['signature'])
|
||||
encoded_cert_id = base64.b64decode(claim_dictionary[SIGNATURE]['certificateId'])
|
||||
claim_dictionary[SIGNATURE]['signature'] = encoded_sig
|
||||
claim_dictionary[SIGNATURE]['certificateId'] = encoded_cert_id
|
||||
claim_dictionary[claim_type] = claim_value
|
||||
return claim_dictionary
|
||||
@@ -1,22 +0,0 @@
|
||||
class UnknownSourceType(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidSourceHashLength(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class DecodeError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class URIParseError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class CertificateError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidAddress(Exception):
|
||||
pass
|
||||
@@ -1,79 +0,0 @@
|
||||
from collections import OrderedDict
|
||||
|
||||
from lbrynet.schema.address import encode_address, decode_address
|
||||
from lbrynet.schema.schema import CURRENCY_NAMES, CURRENCY_MAP
|
||||
from lbrynet.schema.schema.fee import Fee as FeeHelper
|
||||
from lbrynet.schema.proto import fee_pb2
|
||||
|
||||
|
||||
def migrate(fee):
|
||||
if len(list(fee.keys())) == 3 and 'currency' in fee and 'amount' in fee and 'address' in fee:
|
||||
return FeeHelper.load({
|
||||
"version": "_0_0_1",
|
||||
"currency": fee['currency'],
|
||||
"amount": fee['amount'],
|
||||
"address": decode_address(fee['address'])
|
||||
})
|
||||
if len(list(fee.keys())) > 1:
|
||||
raise Exception("Invalid fee")
|
||||
|
||||
currency = list(fee.keys())[0]
|
||||
amount = fee[currency]['amount']
|
||||
address = fee[currency]['address']
|
||||
|
||||
return FeeHelper.load({
|
||||
"version": "_0_0_1",
|
||||
"currency": currency,
|
||||
"amount": amount,
|
||||
"address": decode_address(address)
|
||||
})
|
||||
|
||||
|
||||
class Fee(OrderedDict):
|
||||
def __init__(self, fee):
|
||||
if (len(fee) == 4 and "version" in fee and "currency" in fee
|
||||
and "amount" in fee and "address" in fee):
|
||||
OrderedDict.__init__(self, fee)
|
||||
else:
|
||||
OrderedDict.__init__(self, Fee.load_protobuf(migrate(fee)))
|
||||
|
||||
@property
|
||||
def currency(self):
|
||||
return self['currency']
|
||||
|
||||
@property
|
||||
def address(self):
|
||||
return self['address']
|
||||
|
||||
@property
|
||||
def amount(self):
|
||||
return self['amount']
|
||||
|
||||
@property
|
||||
def version(self):
|
||||
return self['version']
|
||||
|
||||
@property
|
||||
def protobuf(self):
|
||||
pb = {
|
||||
"version": self.version,
|
||||
"currency": CURRENCY_MAP[self.currency],
|
||||
"address": decode_address(self.address),
|
||||
"amount": self.amount
|
||||
}
|
||||
return FeeHelper.load(pb)
|
||||
|
||||
@classmethod
|
||||
def load_protobuf(cls, pb):
|
||||
return cls({
|
||||
"version": pb.version,
|
||||
"currency": CURRENCY_NAMES[pb.currency],
|
||||
"address": encode_address(pb.address),
|
||||
"amount": pb.amount
|
||||
})
|
||||
|
||||
@classmethod
|
||||
def deserialize(cls, serialized):
|
||||
pb = fee_pb2.Fee()
|
||||
pb.ParseFromString(serialized)
|
||||
return cls.load_protobuf(pb)
|
||||
@@ -1,23 +0,0 @@
|
||||
import hashlib
|
||||
|
||||
|
||||
def sha256(x):
|
||||
if isinstance(x, str):
|
||||
x = x.encode('utf-8')
|
||||
return hashlib.sha256(x).digest()
|
||||
|
||||
|
||||
def double_sha256(x):
|
||||
return sha256(sha256(x))
|
||||
|
||||
|
||||
def ripemd160(x):
|
||||
if isinstance(x, str):
|
||||
x = x.encode('utf-8')
|
||||
md = hashlib.new('ripemd160')
|
||||
md.update(x)
|
||||
return md.digest()
|
||||
|
||||
|
||||
def hash160(x):
|
||||
return ripemd160(sha256(x))
|
||||
@@ -1,59 +0,0 @@
|
||||
import jsonschema
|
||||
|
||||
|
||||
class StructuredDict(dict):
|
||||
"""
|
||||
A dictionary that enforces a structure specified by a schema, and supports
|
||||
migration between different versions of the schema.
|
||||
"""
|
||||
|
||||
# To be specified in sub-classes, an array in the format
|
||||
# [(version, schema, migration), ...]
|
||||
_versions = []
|
||||
|
||||
# Used internally to allow schema lookups by version number
|
||||
_schemas = {}
|
||||
|
||||
version = None
|
||||
|
||||
def __init__(self, value, starting_version, migrate=True, target_version=None):
|
||||
dict.__init__(self, value)
|
||||
|
||||
self.version = starting_version
|
||||
self._schemas = dict([(version, schema) for (version, schema, _) in self._versions])
|
||||
|
||||
self.validate(starting_version)
|
||||
|
||||
if migrate:
|
||||
self.migrate(target_version)
|
||||
|
||||
def _upgrade_version_range(self, start_version, end_version):
|
||||
after_starting_version = False
|
||||
for version, schema, migration in self._versions:
|
||||
if not after_starting_version:
|
||||
if version == self.version:
|
||||
after_starting_version = True
|
||||
continue
|
||||
|
||||
yield version, schema, migration
|
||||
|
||||
if end_version and version == end_version:
|
||||
break
|
||||
|
||||
def validate(self, version):
|
||||
jsonschema.validate(self, self._schemas[version])
|
||||
|
||||
def migrate(self, target_version=None):
|
||||
if target_version:
|
||||
assert self._versions.index(target_version) > self._versions.index(self.version), \
|
||||
"Current version is above target version"
|
||||
|
||||
for version, schema, migration in self._upgrade_version_range(self.version, target_version):
|
||||
migration(self)
|
||||
try:
|
||||
self.validate(version)
|
||||
except jsonschema.ValidationError as e:
|
||||
raise jsonschema.ValidationError(
|
||||
"Could not migrate to version %s due to validation error: %s" %
|
||||
(version, e.message))
|
||||
self.version = version
|
||||
@@ -1,276 +0,0 @@
|
||||
VER_001 = {
|
||||
'$schema': 'http://json-schema.org/draft-04/schema#',
|
||||
'title': 'LBRY metadata schema 0.0.1',
|
||||
'definitions': {
|
||||
'fee_info': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'amount': {
|
||||
'type': 'number',
|
||||
'minimum': 0,
|
||||
'exclusiveMinimum': True,
|
||||
},
|
||||
'address': {
|
||||
'type': 'string'
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
'type': 'object',
|
||||
|
||||
'properties': {
|
||||
'ver': {
|
||||
'type': 'string',
|
||||
'default': '0.0.1'
|
||||
},
|
||||
'title': {
|
||||
'type': 'string'
|
||||
},
|
||||
'description': {
|
||||
'type': 'string'
|
||||
},
|
||||
'author': {
|
||||
'type': 'string'
|
||||
},
|
||||
'language': {
|
||||
'type': 'string'
|
||||
},
|
||||
'license': {
|
||||
'type': 'string'
|
||||
},
|
||||
'content-type': {
|
||||
'type': 'string'
|
||||
},
|
||||
'sources': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'lbry_sd_hash': {
|
||||
'type': 'string'
|
||||
},
|
||||
'btih': {
|
||||
'type': 'string'
|
||||
},
|
||||
'url': {
|
||||
'type': 'string'
|
||||
}
|
||||
},
|
||||
'additionalProperties': False
|
||||
},
|
||||
'thumbnail': {
|
||||
'type': 'string'
|
||||
},
|
||||
'preview': {
|
||||
'type': 'string'
|
||||
},
|
||||
'fee': {
|
||||
'properties': {
|
||||
'LBC': {'$ref': '#/definitions/fee_info'},
|
||||
'BTC': {'$ref': '#/definitions/fee_info'},
|
||||
'USD': {'$ref': '#/definitions/fee_info'}
|
||||
}
|
||||
},
|
||||
'contact': {
|
||||
'type': 'number'
|
||||
},
|
||||
'pubkey': {
|
||||
'type': 'string'
|
||||
},
|
||||
},
|
||||
'required': [
|
||||
'title', 'description', 'author', 'language', 'license', 'content-type', 'sources'],
|
||||
'additionalProperties': False
|
||||
}
|
||||
|
||||
|
||||
VER_002 = {
|
||||
'$schema': 'http://json-schema.org/draft-04/schema#',
|
||||
'title': 'LBRY metadata schema 0.0.2',
|
||||
'definitions': {
|
||||
'fee_info': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'amount': {
|
||||
'type': 'number',
|
||||
'minimum': 0,
|
||||
'exclusiveMinimum': True,
|
||||
},
|
||||
'address': {
|
||||
'type': 'string'
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
'type': 'object',
|
||||
|
||||
'properties': {
|
||||
'ver': {
|
||||
'type': 'string',
|
||||
'enum': ['0.0.2'],
|
||||
},
|
||||
'title': {
|
||||
'type': 'string'
|
||||
},
|
||||
'description': {
|
||||
'type': 'string'
|
||||
},
|
||||
'author': {
|
||||
'type': 'string'
|
||||
},
|
||||
'language': {
|
||||
'type': 'string'
|
||||
},
|
||||
'license': {
|
||||
'type': 'string'
|
||||
},
|
||||
'content-type': {
|
||||
'type': 'string'
|
||||
},
|
||||
'sources': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'lbry_sd_hash': {
|
||||
'type': 'string'
|
||||
},
|
||||
'btih': {
|
||||
'type': 'string'
|
||||
},
|
||||
'url': {
|
||||
'type': 'string'
|
||||
}
|
||||
},
|
||||
'additionalProperties': False
|
||||
},
|
||||
'thumbnail': {
|
||||
'type': 'string'
|
||||
},
|
||||
'preview': {
|
||||
'type': 'string'
|
||||
},
|
||||
'fee': {
|
||||
'properties': {
|
||||
'LBC': {'$ref': '#/definitions/fee_info'},
|
||||
'BTC': {'$ref': '#/definitions/fee_info'},
|
||||
'USD': {'$ref': '#/definitions/fee_info'}
|
||||
}
|
||||
},
|
||||
'contact': {
|
||||
'type': 'number'
|
||||
},
|
||||
'pubkey': {
|
||||
'type': 'string'
|
||||
},
|
||||
'license_url': {
|
||||
'type': 'string'
|
||||
},
|
||||
'nsfw': {
|
||||
'type': 'boolean',
|
||||
'default': False
|
||||
},
|
||||
|
||||
},
|
||||
'required': [
|
||||
'ver', 'title', 'description', 'author', 'language', 'license',
|
||||
'content-type', 'sources', 'nsfw'
|
||||
],
|
||||
'additionalProperties': False
|
||||
}
|
||||
|
||||
|
||||
VER_003 = {
|
||||
'$schema': 'http://json-schema.org/draft-04/schema#',
|
||||
'title': 'LBRY metadata schema 0.0.3',
|
||||
'definitions': {
|
||||
'fee_info': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'amount': {
|
||||
'type': 'number',
|
||||
'minimum': 0,
|
||||
'exclusiveMinimum': True,
|
||||
},
|
||||
'address': {
|
||||
'type': 'string'
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
'type': 'object',
|
||||
|
||||
'properties': {
|
||||
'ver': {
|
||||
'type': 'string',
|
||||
'enum': ['0.0.3'],
|
||||
},
|
||||
'title': {
|
||||
'type': 'string'
|
||||
},
|
||||
'description': {
|
||||
'type': 'string'
|
||||
},
|
||||
'author': {
|
||||
'type': 'string'
|
||||
},
|
||||
'language': {
|
||||
'type': 'string'
|
||||
},
|
||||
'license': {
|
||||
'type': 'string'
|
||||
},
|
||||
'content_type': {
|
||||
'type': 'string'
|
||||
},
|
||||
'sources': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'lbry_sd_hash': {
|
||||
'type': 'string'
|
||||
},
|
||||
'btih': {
|
||||
'type': 'string'
|
||||
},
|
||||
'url': {
|
||||
'type': 'string'
|
||||
}
|
||||
},
|
||||
'additionalProperties': False
|
||||
},
|
||||
'thumbnail': {
|
||||
'type': 'string'
|
||||
},
|
||||
'preview': {
|
||||
'type': 'string'
|
||||
},
|
||||
'fee': {
|
||||
'properties': {
|
||||
'LBC': {'$ref': '#/definitions/fee_info'},
|
||||
'BTC': {'$ref': '#/definitions/fee_info'},
|
||||
'USD': {'$ref': '#/definitions/fee_info'}
|
||||
}
|
||||
},
|
||||
'contact': {
|
||||
'type': 'number'
|
||||
},
|
||||
'pubkey': {
|
||||
'type': 'string'
|
||||
},
|
||||
'license_url': {
|
||||
'type': 'string'
|
||||
},
|
||||
'nsfw': {
|
||||
'type': 'boolean',
|
||||
'default': False
|
||||
},
|
||||
'sig': {
|
||||
'type': 'string'
|
||||
}
|
||||
},
|
||||
'required': [
|
||||
'ver', 'title', 'description', 'author', 'language', 'license',
|
||||
'content_type', 'sources', 'nsfw'
|
||||
],
|
||||
'additionalProperties': False,
|
||||
'dependencies': {
|
||||
'pubkey': ['sig'],
|
||||
'sig': ['pubkey']
|
||||
}
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
"""
|
||||
migrate claim json schema (0.0.1-3) to protobuf (0.1.0)
|
||||
"""
|
||||
|
||||
from lbrynet.schema.legacy import metadata_schemas
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from .StructuredDict import StructuredDict
|
||||
|
||||
|
||||
def migrate_001_to_002(metadata):
|
||||
metadata['ver'] = '0.0.2'
|
||||
metadata['nsfw'] = False
|
||||
|
||||
|
||||
def migrate_002_to_003(metadata):
|
||||
metadata['ver'] = '0.0.3'
|
||||
if 'content-type' in metadata:
|
||||
metadata['content_type'] = metadata['content-type']
|
||||
del metadata['content-type']
|
||||
|
||||
|
||||
class LegacyMetadata(StructuredDict):
|
||||
current_version = '0.0.3'
|
||||
|
||||
_versions = [
|
||||
('0.0.1', metadata_schemas.VER_001, None),
|
||||
('0.0.2', metadata_schemas.VER_002, migrate_001_to_002),
|
||||
('0.0.3', metadata_schemas.VER_003, migrate_002_to_003)
|
||||
]
|
||||
|
||||
def __init__(self, metadata, migrate=True, target_version=None):
|
||||
if not isinstance(metadata, dict):
|
||||
raise TypeError("{} is not a dictionary".format(metadata))
|
||||
starting_version = metadata.get('ver', '0.0.1')
|
||||
|
||||
StructuredDict.__init__(self, metadata, starting_version, migrate, target_version)
|
||||
|
||||
|
||||
def migrate_003_to_010(value):
|
||||
migrated_to_003 = LegacyMetadata(value)
|
||||
metadata = {
|
||||
"version": "_0_1_0"
|
||||
}
|
||||
for k in ["author", "description", "language", "license", "nsfw", "thumbnail", "title",
|
||||
"preview"]:
|
||||
if k in migrated_to_003:
|
||||
metadata.update({k: migrated_to_003[k]})
|
||||
|
||||
if 'license_url' in migrated_to_003:
|
||||
metadata['licenseUrl'] = migrated_to_003['license_url']
|
||||
|
||||
if "fee" in migrated_to_003:
|
||||
fee = migrated_to_003["fee"]
|
||||
currency = list(fee.keys())[0]
|
||||
amount = fee[currency]['amount']
|
||||
address = fee[currency]['address']
|
||||
metadata.update(dict(fee={"currency": currency, "version": "_0_0_1",
|
||||
"amount": amount, "address": address}))
|
||||
source = {
|
||||
"source": migrated_to_003['sources']['lbry_sd_hash'],
|
||||
"contentType": migrated_to_003['content_type'],
|
||||
"sourceType": "lbry_sd_hash",
|
||||
"version": "_0_0_1"
|
||||
}
|
||||
|
||||
migrated = {
|
||||
"version": "_0_0_1",
|
||||
"claimType": "streamType",
|
||||
"stream": {
|
||||
"version": "_0_0_1",
|
||||
"metadata": metadata,
|
||||
"source": source
|
||||
}
|
||||
}
|
||||
return ClaimDict.load_dict(migrated)
|
||||
|
||||
|
||||
def migrate(value):
|
||||
return migrate_003_to_010(value)
|
||||
@@ -0,0 +1,175 @@
|
||||
import os
|
||||
|
||||
types_map = {
|
||||
# http://www.iana.org/assignments/media-types
|
||||
# Type mapping for automated metadata extraction (video, audio, image, document, binary, model)
|
||||
'.a': ('application/octet-stream', 'binary'),
|
||||
'.ai': ('application/postscript', 'image'),
|
||||
'.aif': ('audio/x-aiff', 'audio'),
|
||||
'.aifc': ('audio/x-aiff', 'audio'),
|
||||
'.aiff': ('audio/x-aiff', 'audio'),
|
||||
'.au': ('audio/basic', 'audio'),
|
||||
'.avi': ('video/x-msvideo', 'video'),
|
||||
'.bat': ('text/plain', 'document'),
|
||||
'.bcpio': ('application/x-bcpio', 'binary'),
|
||||
'.bin': ('application/octet-stream', 'binary'),
|
||||
'.bmp': ('image/bmp', 'image'),
|
||||
'.c': ('text/plain', 'document'),
|
||||
'.cdf': ('application/x-netcdf', 'binary'),
|
||||
'.cpio': ('application/x-cpio', 'binary'),
|
||||
'.csh': ('application/x-csh', 'binary'),
|
||||
'.css': ('text/css', 'document'),
|
||||
'.csv': ('text/csv', 'document'),
|
||||
'.dll': ('application/octet-stream', 'binary'),
|
||||
'.doc': ('application/msword', 'document'),
|
||||
'.dot': ('application/msword', 'document'),
|
||||
'.dvi': ('application/x-dvi', 'binary'),
|
||||
'.eml': ('message/rfc822', 'document'),
|
||||
'.eps': ('application/postscript', 'document'),
|
||||
'.epub': ('application/epub+zip', 'document'),
|
||||
'.etx': ('text/x-setext', 'document'),
|
||||
'.exe': ('application/octet-stream', 'binary'),
|
||||
'.gif': ('image/gif', 'image'),
|
||||
'.gtar': ('application/x-gtar', 'binary'),
|
||||
'.h': ('text/plain', 'document'),
|
||||
'.hdf': ('application/x-hdf', 'binary'),
|
||||
'.htm': ('text/html', 'document'),
|
||||
'.html': ('text/html', 'document'),
|
||||
'.ico': ('image/vnd.microsoft.icon', 'image'),
|
||||
'.ief': ('image/ief', 'image'),
|
||||
'.iges': ('model/iges', 'model'),
|
||||
'.jpe': ('image/jpeg', 'image'),
|
||||
'.jpeg': ('image/jpeg', 'image'),
|
||||
'.jpg': ('image/jpeg', 'image'),
|
||||
'.js': ('application/javascript', 'document'),
|
||||
'.json': ('application/json', 'document'),
|
||||
'.ksh': ('text/plain', 'document'),
|
||||
'.latex': ('application/x-latex', 'binary'),
|
||||
'.m1v': ('video/mpeg', 'video'),
|
||||
'.m3u': ('application/vnd.apple.mpegurl', 'audio'),
|
||||
'.m3u8': ('application/vnd.apple.mpegurl', 'audio'),
|
||||
'.man': ('application/x-troff-man', 'document'),
|
||||
'.markdown': ('text/markdown', 'document'),
|
||||
'.md': ('text/markdown', 'document'),
|
||||
'.me': ('application/x-troff-me', 'binary'),
|
||||
'.mht': ('message/rfc822', 'document'),
|
||||
'.mhtml': ('message/rfc822', 'document'),
|
||||
'.mif': ('application/x-mif', 'binary'),
|
||||
'.mov': ('video/quicktime', 'video'),
|
||||
'.movie': ('video/x-sgi-movie', 'video'),
|
||||
'.mp2': ('audio/mpeg', 'audio'),
|
||||
'.mp3': ('audio/mpeg', 'audio'),
|
||||
'.mp4': ('video/mp4', 'video'),
|
||||
'.mpa': ('video/mpeg', 'video'),
|
||||
'.mpe': ('video/mpeg', 'video'),
|
||||
'.mpeg': ('video/mpeg', 'video'),
|
||||
'.mpg': ('video/mpeg', 'video'),
|
||||
'.ms': ('application/x-troff-ms', 'binary'),
|
||||
'.nc': ('application/x-netcdf', 'binary'),
|
||||
'.nws': ('message/rfc822', 'document'),
|
||||
'.o': ('application/octet-stream', 'binary'),
|
||||
'.obj': ('application/octet-stream', 'model'),
|
||||
'.oda': ('application/oda', 'binary'),
|
||||
'.p12': ('application/x-pkcs12', 'binary'),
|
||||
'.p7c': ('application/pkcs7-mime', 'binary'),
|
||||
'.pbm': ('image/x-portable-bitmap', 'image'),
|
||||
'.pdf': ('application/pdf', 'document'),
|
||||
'.pfx': ('application/x-pkcs12', 'binary'),
|
||||
'.pgm': ('image/x-portable-graymap', 'image'),
|
||||
'.pl': ('text/plain', 'document'),
|
||||
'.png': ('image/png', 'image'),
|
||||
'.pnm': ('image/x-portable-anymap', 'image'),
|
||||
'.pot': ('application/vnd.ms-powerpoint', 'document'),
|
||||
'.ppa': ('application/vnd.ms-powerpoint', 'document'),
|
||||
'.ppm': ('image/x-portable-pixmap', 'image'),
|
||||
'.pps': ('application/vnd.ms-powerpoint', 'document'),
|
||||
'.ppt': ('application/vnd.ms-powerpoint', 'document'),
|
||||
'.ps': ('application/postscript', 'document'),
|
||||
'.pwz': ('application/vnd.ms-powerpoint', 'document'),
|
||||
'.py': ('text/x-python', 'document'),
|
||||
'.pyc': ('application/x-python-code', 'binary'),
|
||||
'.pyo': ('application/x-python-code', 'binary'),
|
||||
'.qt': ('video/quicktime', 'video'),
|
||||
'.ra': ('audio/x-pn-realaudio', 'audio'),
|
||||
'.ram': ('application/x-pn-realaudio', 'audio'),
|
||||
'.ras': ('image/x-cmu-raster', 'image'),
|
||||
'.rdf': ('application/xml', 'binary'),
|
||||
'.rgb': ('image/x-rgb', 'image'),
|
||||
'.roff': ('application/x-troff', 'binary'),
|
||||
'.rtx': ('text/richtext', 'document'),
|
||||
'.sgm': ('text/x-sgml', 'document'),
|
||||
'.sgml': ('text/x-sgml', 'document'),
|
||||
'.sh': ('application/x-sh', 'document'),
|
||||
'.shar': ('application/x-shar', 'binary'),
|
||||
'.snd': ('audio/basic', 'audio'),
|
||||
'.so': ('application/octet-stream', 'binary'),
|
||||
'.src': ('application/x-wais-source', 'binary'),
|
||||
'.stl': ('model/stl', 'model'),
|
||||
'.sv4cpio': ('application/x-sv4cpio', 'binary'),
|
||||
'.sv4crc': ('application/x-sv4crc', 'binary'),
|
||||
'.svg': ('image/svg+xml', 'image'),
|
||||
'.swf': ('application/x-shockwave-flash', 'binary'),
|
||||
'.t': ('application/x-troff', 'binary'),
|
||||
'.tar': ('application/x-tar', 'binary'),
|
||||
'.tcl': ('application/x-tcl', 'binary'),
|
||||
'.tex': ('application/x-tex', 'binary'),
|
||||
'.texi': ('application/x-texinfo', 'binary'),
|
||||
'.texinfo': ('application/x-texinfo', 'binary'),
|
||||
'.tif': ('image/tiff', 'image'),
|
||||
'.tiff': ('image/tiff', 'image'),
|
||||
'.tr': ('application/x-troff', 'binary'),
|
||||
'.tsv': ('text/tab-separated-values', 'document'),
|
||||
'.txt': ('text/plain', 'document'),
|
||||
'.ustar': ('application/x-ustar', 'binary'),
|
||||
'.vcf': ('text/x-vcard', 'document'),
|
||||
'.wav': ('audio/x-wav', 'audio'),
|
||||
'.webm': ('video/webm', 'video'),
|
||||
'.wiz': ('application/msword', 'document'),
|
||||
'.wsdl': ('application/xml', 'document'),
|
||||
'.xbm': ('image/x-xbitmap', 'image'),
|
||||
'.xlb': ('application/vnd.ms-excel', 'document'),
|
||||
'.xls': ('application/vnd.ms-excel', 'document'),
|
||||
'.xml': ('text/xml', 'document'),
|
||||
'.xpdl': ('application/xml', 'document'),
|
||||
'.xpm': ('image/x-xpixmap', 'image'),
|
||||
'.xsl': ('application/xml', 'document'),
|
||||
'.xwd': ('image/x-xwindowdump', 'image'),
|
||||
'.zip': ('application/zip', 'binary'),
|
||||
|
||||
# These are non-standard types, commonly found in the wild.
|
||||
'.cbr': ('application/vnd.comicbook-rar', 'document'),
|
||||
'.cbz': ('application/vnd.comicbook+zip', 'document'),
|
||||
'.lbry': ('application/x-ext-lbry', 'document'),
|
||||
'.m4v': ('video/m4v', 'video'),
|
||||
'.mid': ('audio/midi', 'audio'),
|
||||
'.midi': ('audio/midi', 'audio'),
|
||||
'.mkv': ('video/x-matroska', 'video'),
|
||||
'.mobi': ('application/x-mobipocket-ebook', 'document'),
|
||||
'.pct': ('image/pict', 'image'),
|
||||
'.pic': ('image/pict', 'image'),
|
||||
'.pict': ('image/pict', 'image'),
|
||||
'.prc': ('application/x-mobipocket-ebook', 'document'),
|
||||
'.rtf': ('application/rtf', 'document'),
|
||||
'.xul': ('text/xul', 'document'),
|
||||
|
||||
# microsoft is special and has its own 'standard'
|
||||
# https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
|
||||
'.wmv': ('video/x-ms-wmv', 'video')
|
||||
}
|
||||
|
||||
|
||||
def guess_media_type(path):
|
||||
_, ext = os.path.splitext(path)
|
||||
extension = ext.strip().lower()
|
||||
if extension[1:]:
|
||||
if extension in types_map:
|
||||
return types_map[extension]
|
||||
return f'application/x-ext-{extension[1:]}', 'binary'
|
||||
return 'application/octet-stream', 'binary'
|
||||
|
||||
|
||||
def guess_stream_type(media_type):
|
||||
for media, stream in types_map.values():
|
||||
if media == media_type:
|
||||
return stream
|
||||
return 'binary'
|
||||
@@ -1,18 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
enum KeyType {
|
||||
UNKNOWN_PUBLIC_KEY_TYPE = 0;
|
||||
NIST256p = 1;
|
||||
NIST384p = 2;
|
||||
SECP256k1 = 3;
|
||||
}
|
||||
|
||||
message Certificate {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
}
|
||||
required Version version = 1;
|
||||
required KeyType keyType = 2;
|
||||
required bytes publicKey = 4;
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
import "lbrynet/schema/proto/stream.proto";
|
||||
import "lbrynet/schema/proto/certificate.proto";
|
||||
import "lbrynet/schema/proto/signature.proto";
|
||||
|
||||
message Claim {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
}
|
||||
required Version version = 1;
|
||||
enum ClaimType {
|
||||
UNKNOWN_CLAIM_TYPE = 0;
|
||||
streamType = 1;
|
||||
certificateType = 2;
|
||||
}
|
||||
required ClaimType claimType = 2;
|
||||
optional Stream stream = 3;
|
||||
optional Certificate certificate = 4;
|
||||
optional Signature publisherSignature = 5;
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
message Fee {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
}
|
||||
enum Currency {
|
||||
UNKNOWN_CURRENCY = 0;
|
||||
LBC = 1;
|
||||
BTC = 2;
|
||||
USD = 3;
|
||||
}
|
||||
required Version version = 1;
|
||||
required Currency currency = 2;
|
||||
required bytes address = 3;
|
||||
required float amount = 4;
|
||||
}
|
||||
@@ -1,212 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
import "lbrynet/schema/proto/fee.proto";
|
||||
|
||||
message Metadata {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
_0_0_2 = 2;
|
||||
_0_0_3 = 3;
|
||||
_0_1_0 = 4;
|
||||
}
|
||||
enum Language {
|
||||
UNKNOWN_LANGUAGE = 0;
|
||||
en = 1;
|
||||
aa = 2;
|
||||
ab = 3;
|
||||
ae = 4;
|
||||
af = 5;
|
||||
ak = 6;
|
||||
am = 7;
|
||||
an = 8;
|
||||
ar = 9;
|
||||
as = 10;
|
||||
av = 11;
|
||||
ay = 12;
|
||||
az = 13;
|
||||
ba = 14;
|
||||
be = 15;
|
||||
bg = 16;
|
||||
bh = 17;
|
||||
bi = 18;
|
||||
bm = 19;
|
||||
bn = 20;
|
||||
bo = 21;
|
||||
br = 22;
|
||||
bs = 23;
|
||||
ca = 24;
|
||||
ce = 25;
|
||||
ch = 26;
|
||||
co = 27;
|
||||
cr = 28;
|
||||
cs = 29;
|
||||
cu = 30;
|
||||
cv = 31;
|
||||
cy = 32;
|
||||
da = 33;
|
||||
de = 34;
|
||||
dv = 35;
|
||||
dz = 36;
|
||||
ee = 37;
|
||||
el = 38;
|
||||
eo = 39;
|
||||
es = 40;
|
||||
et = 41;
|
||||
eu = 42;
|
||||
fa = 43;
|
||||
ff = 44;
|
||||
fi = 45;
|
||||
fj = 46;
|
||||
fo = 47;
|
||||
fr = 48;
|
||||
fy = 49;
|
||||
ga = 50;
|
||||
gd = 51;
|
||||
gl = 52;
|
||||
gn = 53;
|
||||
gu = 54;
|
||||
gv = 55;
|
||||
ha = 56;
|
||||
he = 57;
|
||||
hi = 58;
|
||||
ho = 59;
|
||||
hr = 60;
|
||||
ht = 61;
|
||||
hu = 62;
|
||||
hy = 63;
|
||||
hz = 64;
|
||||
ia = 65;
|
||||
id = 66;
|
||||
ie = 67;
|
||||
ig = 68;
|
||||
ii = 69;
|
||||
ik = 70;
|
||||
io = 71;
|
||||
is = 72;
|
||||
it = 73;
|
||||
iu = 74;
|
||||
ja = 75;
|
||||
jv = 76;
|
||||
ka = 77;
|
||||
kg = 78;
|
||||
ki = 79;
|
||||
kj = 80;
|
||||
kk = 81;
|
||||
kl = 82;
|
||||
km = 83;
|
||||
kn = 84;
|
||||
ko = 85;
|
||||
kr = 86;
|
||||
ks = 87;
|
||||
ku = 88;
|
||||
kv = 89;
|
||||
kw = 90;
|
||||
ky = 91;
|
||||
la = 92;
|
||||
lb = 93;
|
||||
lg = 94;
|
||||
li = 95;
|
||||
ln = 96;
|
||||
lo = 97;
|
||||
lt = 98;
|
||||
lu = 99;
|
||||
lv = 100;
|
||||
mg = 101;
|
||||
mh = 102;
|
||||
mi = 103;
|
||||
mk = 104;
|
||||
ml = 105;
|
||||
mn = 106;
|
||||
mr = 107;
|
||||
ms = 108;
|
||||
mt = 109;
|
||||
my = 110;
|
||||
na = 111;
|
||||
nb = 112;
|
||||
nd = 113;
|
||||
ne = 114;
|
||||
ng = 115;
|
||||
nl = 116;
|
||||
nn = 117;
|
||||
no = 118;
|
||||
nr = 119;
|
||||
nv = 120;
|
||||
ny = 121;
|
||||
oc = 122;
|
||||
oj = 123;
|
||||
om = 124;
|
||||
or = 125;
|
||||
os = 126;
|
||||
pa = 127;
|
||||
pi = 128;
|
||||
pl = 129;
|
||||
ps = 130;
|
||||
pt = 131;
|
||||
qu = 132;
|
||||
rm = 133;
|
||||
rn = 134;
|
||||
ro = 135;
|
||||
ru = 136;
|
||||
rw = 137;
|
||||
sa = 138;
|
||||
sc = 139;
|
||||
sd = 140;
|
||||
se = 141;
|
||||
sg = 142;
|
||||
si = 143;
|
||||
sk = 144;
|
||||
sl = 145;
|
||||
sm = 146;
|
||||
sn = 147;
|
||||
so = 148;
|
||||
sq = 149;
|
||||
sr = 150;
|
||||
ss = 151;
|
||||
st = 152;
|
||||
su = 153;
|
||||
sv = 154;
|
||||
sw = 155;
|
||||
ta = 156;
|
||||
te = 157;
|
||||
tg = 158;
|
||||
th = 159;
|
||||
ti = 160;
|
||||
tk = 161;
|
||||
tl = 162;
|
||||
tn = 163;
|
||||
to = 164;
|
||||
tr = 165;
|
||||
ts = 166;
|
||||
tt = 167;
|
||||
tw = 168;
|
||||
ty = 169;
|
||||
ug = 170;
|
||||
uk = 171;
|
||||
ur = 172;
|
||||
uz = 173;
|
||||
ve = 174;
|
||||
vi = 175;
|
||||
vo = 176;
|
||||
wa = 177;
|
||||
wo = 178;
|
||||
xh = 179;
|
||||
yi = 180;
|
||||
yo = 181;
|
||||
za = 182;
|
||||
zh = 183;
|
||||
zu = 184;
|
||||
}
|
||||
required Version version = 1;
|
||||
required Language language = 2;
|
||||
required string title = 3;
|
||||
required string description = 4;
|
||||
required string author = 5;
|
||||
required string license = 6;
|
||||
required bool nsfw = 7;
|
||||
|
||||
optional Fee fee = 8;
|
||||
optional string thumbnail = 9;
|
||||
optional string preview = 10;
|
||||
optional string licenseUrl = 11;
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
import "lbrynet/schema/proto/certificate.proto";
|
||||
|
||||
message Signature {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
}
|
||||
required Version version = 1;
|
||||
required KeyType signatureType = 2;
|
||||
required bytes signature = 3;
|
||||
required bytes certificateId = 4;
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
message Source {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
}
|
||||
required Version version = 1;
|
||||
enum SourceTypes {
|
||||
UNKNOWN_SOURCE_TYPE = 0;
|
||||
lbry_sd_hash = 1;
|
||||
}
|
||||
required SourceTypes sourceType = 2;
|
||||
required bytes source = 3;
|
||||
required string contentType = 4;
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
import "lbrynet/schema/proto/metadata.proto";
|
||||
import "lbrynet/schema/proto/source.proto";
|
||||
|
||||
|
||||
message Stream {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
}
|
||||
required Version version = 1;
|
||||
required Metadata metadata = 2;
|
||||
required Source source = 3;
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
import base64
|
||||
import struct
|
||||
from typing import List
|
||||
from binascii import hexlify
|
||||
from itertools import chain
|
||||
|
||||
from lbrynet.schema.types.v2.result_pb2 import Outputs as OutputsMessage
|
||||
|
||||
|
||||
class Outputs:
|
||||
|
||||
__slots__ = 'txos', 'extra_txos', 'txs', 'offset', 'total'
|
||||
|
||||
def __init__(self, txos: List, extra_txos: List, txs: set, offset: int, total: int):
|
||||
self.txos = txos
|
||||
self.txs = txs
|
||||
self.extra_txos = extra_txos
|
||||
self.offset = offset
|
||||
self.total = total
|
||||
|
||||
def inflate(self, txs):
|
||||
tx_map = {tx.hash: tx for tx in txs}
|
||||
for txo_message in self.extra_txos:
|
||||
self.message_to_txo(txo_message, tx_map)
|
||||
return [self.message_to_txo(txo_message, tx_map) for txo_message in self.txos]
|
||||
|
||||
def message_to_txo(self, txo_message, tx_map):
|
||||
if txo_message.WhichOneof('meta') == 'error':
|
||||
return None
|
||||
txo = tx_map[txo_message.tx_hash].outputs[txo_message.nout]
|
||||
if txo_message.WhichOneof('meta') == 'claim':
|
||||
claim = txo_message.claim
|
||||
txo.meta = {
|
||||
'short_url': f'lbry://{claim.short_url}',
|
||||
'canonical_url': f'lbry://{claim.canonical_url or claim.short_url}',
|
||||
'is_controlling': claim.is_controlling,
|
||||
'creation_height': claim.creation_height,
|
||||
'activation_height': claim.activation_height,
|
||||
'expiration_height': claim.expiration_height,
|
||||
'effective_amount': claim.effective_amount,
|
||||
'support_amount': claim.support_amount,
|
||||
'trending_group': claim.trending_group,
|
||||
'trending_mixed': claim.trending_mixed,
|
||||
'trending_local': claim.trending_local,
|
||||
'trending_global': claim.trending_global,
|
||||
}
|
||||
if claim.HasField('channel'):
|
||||
txo.channel = tx_map[claim.channel.tx_hash].outputs[claim.channel.nout]
|
||||
try:
|
||||
if txo.claim.is_channel:
|
||||
txo.meta['claims_in_channel'] = claim.claims_in_channel
|
||||
except:
|
||||
pass
|
||||
return txo
|
||||
|
||||
@classmethod
|
||||
def from_base64(cls, data: str) -> 'Outputs':
|
||||
return cls.from_bytes(base64.b64decode(data))
|
||||
|
||||
@classmethod
|
||||
def from_bytes(cls, data: bytes) -> 'Outputs':
|
||||
outputs = OutputsMessage()
|
||||
outputs.ParseFromString(data)
|
||||
txs = set()
|
||||
for txo_message in chain(outputs.txos, outputs.extra_txos):
|
||||
if txo_message.WhichOneof('meta') == 'error':
|
||||
continue
|
||||
txs.add((hexlify(txo_message.tx_hash[::-1]).decode(), txo_message.height))
|
||||
return cls(outputs.txos, outputs.extra_txos, txs, outputs.offset, outputs.total)
|
||||
|
||||
@classmethod
|
||||
def to_base64(cls, txo_rows, extra_txo_rows, offset=0, total=None) -> str:
|
||||
return base64.b64encode(cls.to_bytes(txo_rows, extra_txo_rows, offset, total)).decode()
|
||||
|
||||
@classmethod
|
||||
def to_bytes(cls, txo_rows, extra_txo_rows, offset=0, total=None) -> bytes:
|
||||
page = OutputsMessage()
|
||||
page.offset = offset
|
||||
page.total = total or len(txo_rows)
|
||||
for row in txo_rows:
|
||||
cls.row_to_message(row, page.txos.add())
|
||||
for row in extra_txo_rows:
|
||||
cls.row_to_message(row, page.extra_txos.add())
|
||||
return page.SerializeToString()
|
||||
|
||||
@classmethod
|
||||
def row_to_message(cls, txo, txo_message):
|
||||
if isinstance(txo, Exception):
|
||||
txo_message.error.text = txo.args[0]
|
||||
if isinstance(txo, ValueError):
|
||||
txo_message.error.code = txo_message.error.INVALID
|
||||
elif isinstance(txo, LookupError):
|
||||
txo_message.error.code = txo_message.error.NOT_FOUND
|
||||
return
|
||||
txo_message.tx_hash = txo['txo_hash'][:32]
|
||||
txo_message.nout, = struct.unpack('<I', txo['txo_hash'][32:])
|
||||
txo_message.height = txo['height']
|
||||
txo_message.claim.short_url = txo['short_url']
|
||||
if txo['canonical_url'] is not None:
|
||||
txo_message.claim.canonical_url = txo['canonical_url']
|
||||
txo_message.claim.is_controlling = bool(txo['is_controlling'])
|
||||
txo_message.claim.creation_height = txo['creation_height']
|
||||
txo_message.claim.activation_height = txo['activation_height']
|
||||
txo_message.claim.expiration_height = txo['expiration_height']
|
||||
if txo['claims_in_channel'] is not None:
|
||||
txo_message.claim.claims_in_channel = txo['claims_in_channel']
|
||||
txo_message.claim.effective_amount = txo['effective_amount']
|
||||
txo_message.claim.support_amount = txo['support_amount']
|
||||
txo_message.claim.trending_group = txo['trending_group']
|
||||
txo_message.claim.trending_mixed = txo['trending_mixed']
|
||||
txo_message.claim.trending_local = txo['trending_local']
|
||||
txo_message.claim.trending_global = txo['trending_global']
|
||||
if txo['channel_txo_hash']:
|
||||
channel = txo_message.claim.channel
|
||||
channel.tx_hash = txo['channel_txo_hash'][:32]
|
||||
channel.nout, = struct.unpack('<I', txo['channel_txo_hash'][32:])
|
||||
channel.height = txo['channel_height']
|
||||
@@ -1,102 +0,0 @@
|
||||
V_0_0_1 = "_0_0_1"
|
||||
V_0_0_2 = "_0_0_2"
|
||||
V_0_0_3 = "_0_0_3"
|
||||
V_0_1_0 = "_0_1_0"
|
||||
|
||||
|
||||
VERSION_MAP = {
|
||||
V_0_0_1: 1,
|
||||
V_0_0_2: 2,
|
||||
V_0_0_3: 3,
|
||||
V_0_1_0: 4,
|
||||
}
|
||||
|
||||
VERSION_NAMES = {
|
||||
1: V_0_0_1,
|
||||
2: V_0_0_2,
|
||||
3: V_0_0_3,
|
||||
4: V_0_1_0
|
||||
}
|
||||
|
||||
LBC = "LBC"
|
||||
BTC = "BTC"
|
||||
USD = "USD"
|
||||
|
||||
CURRENCY_MAP = {
|
||||
LBC: 1,
|
||||
BTC: 2,
|
||||
USD: 3
|
||||
}
|
||||
|
||||
CURRENCY_NAMES = {
|
||||
1: LBC,
|
||||
2: BTC,
|
||||
3: USD
|
||||
}
|
||||
|
||||
ADDRESS_LENGTH = 25
|
||||
ADDRESS_CHECKSUM_LENGTH = 4
|
||||
|
||||
VERSION = "version"
|
||||
STREAM_TYPE = "streamType"
|
||||
CERTIFICATE_TYPE = "certificateType"
|
||||
CLAIM_TYPE = "claimType"
|
||||
SIGNATURE = "publisherSignature"
|
||||
|
||||
CLAIM_TYPES = {
|
||||
STREAM_TYPE: "stream",
|
||||
CERTIFICATE_TYPE: "certificate"
|
||||
}
|
||||
|
||||
CLAIM_TYPE_NAMES = {
|
||||
1: "stream",
|
||||
2: "certificate"
|
||||
}
|
||||
|
||||
LBRY_SD_HASH = "lbry_sd_hash"
|
||||
LBRY_SD_HASH_LENGTH = 48
|
||||
|
||||
SOURCE_TYPES = {
|
||||
LBRY_SD_HASH: 1
|
||||
}
|
||||
|
||||
NIST256p = "NIST256p"
|
||||
NIST384p = "NIST384p"
|
||||
SECP256k1 = "SECP256k1"
|
||||
|
||||
ECDSA_CURVES = {
|
||||
NIST256p: 1,
|
||||
NIST384p: 2,
|
||||
SECP256k1: 3
|
||||
}
|
||||
|
||||
CURVE_NAMES = {
|
||||
1: NIST256p,
|
||||
2: NIST384p,
|
||||
3: SECP256k1
|
||||
}
|
||||
|
||||
SHA256 = "sha256"
|
||||
SHA384 = "sha384"
|
||||
|
||||
|
||||
B58_CHARS = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
|
||||
assert len(B58_CHARS) == 58
|
||||
|
||||
PUBKEY_ADDRESS = 0
|
||||
SCRIPT_ADDRESS = 5
|
||||
|
||||
ADDRESS_PREFIXES = {
|
||||
"lbrycrd_main": {
|
||||
PUBKEY_ADDRESS: 85,
|
||||
SCRIPT_ADDRESS: 122
|
||||
},
|
||||
"lbrycrd_regtest": {
|
||||
PUBKEY_ADDRESS: 111,
|
||||
SCRIPT_ADDRESS: 196
|
||||
},
|
||||
"lbrycrd_testnet": {
|
||||
PUBKEY_ADDRESS: 111,
|
||||
SCRIPT_ADDRESS: 196
|
||||
},
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
from copy import deepcopy
|
||||
|
||||
from lbrynet.schema.proto import certificate_pb2 as cert_pb
|
||||
from lbrynet.schema.schema.schema import Schema
|
||||
from lbrynet.schema.schema import VERSION_MAP, V_0_0_1, ECDSA_CURVES, CURVE_NAMES
|
||||
from lbrynet.schema.validator import get_key_type_from_dem
|
||||
|
||||
|
||||
class _ECDSAKeyHelper(object):
|
||||
def __init__(self, key):
|
||||
self._key = key
|
||||
|
||||
@property
|
||||
def der(self):
|
||||
return self._key.to_der()
|
||||
|
||||
@property
|
||||
def curve_name(self):
|
||||
return self._key.curve.name
|
||||
|
||||
|
||||
class Certificate(Schema):
|
||||
@classmethod
|
||||
def load(cls, message):
|
||||
_key = deepcopy(message)
|
||||
_message_pb = cert_pb.Certificate()
|
||||
if isinstance(_key, dict):
|
||||
_message_pb.publicKey = _key.pop("publicKey")
|
||||
_message_pb.version = VERSION_MAP[_key.pop("version")]
|
||||
_message_pb.keyType = ECDSA_CURVES[_key.pop("keyType")]
|
||||
else:
|
||||
_message_pb.version = _key.version
|
||||
_message_pb.keyType = _key.keyType
|
||||
_message_pb.publicKey = _key.publicKey
|
||||
if _message_pb.keyType not in CURVE_NAMES:
|
||||
raise Exception("Unknown curve")
|
||||
if get_key_type_from_dem(_message_pb.publicKey) != _message_pb.keyType:
|
||||
raise Exception("Curve mismatch")
|
||||
return cls._load(_key, _message_pb)
|
||||
|
||||
@classmethod
|
||||
def load_from_key_obj(cls, key, key_type):
|
||||
if key_type in ECDSA_CURVES:
|
||||
_key = _ECDSAKeyHelper(key)
|
||||
else:
|
||||
raise Exception("Unknown key type: %s" % str(type(key)))
|
||||
if key_type != _key.curve_name:
|
||||
raise Exception("Curve mismatch")
|
||||
msg = {
|
||||
"version": V_0_0_1,
|
||||
"keyType": key_type,
|
||||
"publicKey": _key.der,
|
||||
}
|
||||
return cls.load(msg)
|
||||
@@ -1,51 +0,0 @@
|
||||
from copy import deepcopy
|
||||
|
||||
from lbrynet.schema.proto import claim_pb2 as claim_pb
|
||||
from lbrynet.schema.schema import VERSION_MAP
|
||||
from lbrynet.schema.schema.signature import Signature
|
||||
from lbrynet.schema.schema.certificate import Certificate
|
||||
from lbrynet.schema.schema.schema import Schema
|
||||
from lbrynet.schema.schema.stream import Stream
|
||||
|
||||
|
||||
class Claim(Schema):
|
||||
CLAIM_TYPE_STREAM = 1
|
||||
CLAIM_TYPE_CERT = 2
|
||||
|
||||
@classmethod
|
||||
def load(cls, message):
|
||||
_claim = deepcopy(message)
|
||||
_message_pb = claim_pb.Claim()
|
||||
_message_pb.version = VERSION_MAP[_claim.pop("version")]
|
||||
|
||||
if "certificate" in _claim:
|
||||
_cert = _claim.pop("certificate")
|
||||
if isinstance(_cert, dict):
|
||||
cert = Certificate.load(_cert)
|
||||
else:
|
||||
cert = _cert
|
||||
claim_type = Claim.CLAIM_TYPE_CERT
|
||||
_message_pb.certificate.MergeFrom(cert)
|
||||
|
||||
elif "stream" in _claim:
|
||||
_stream = _claim.pop("stream")
|
||||
if isinstance(_stream, dict):
|
||||
stream = Stream.load(_stream)
|
||||
else:
|
||||
stream = _stream
|
||||
claim_type = Claim.CLAIM_TYPE_STREAM
|
||||
_message_pb.stream.MergeFrom(stream)
|
||||
else:
|
||||
raise AttributeError
|
||||
|
||||
_message_pb.claimType = claim_type
|
||||
|
||||
if "publisherSignature" in _claim:
|
||||
_publisherSignature = _claim.pop("publisherSignature")
|
||||
if isinstance(_publisherSignature, dict):
|
||||
publisherSignature = Signature.load(_publisherSignature)
|
||||
else:
|
||||
publisherSignature = _publisherSignature
|
||||
_message_pb.publisherSignature.MergeFrom(publisherSignature)
|
||||
|
||||
return cls._load(_claim, _message_pb)
|
||||
@@ -1,17 +0,0 @@
|
||||
from copy import deepcopy
|
||||
|
||||
from lbrynet.schema.schema.schema import Schema
|
||||
from lbrynet.schema.proto import fee_pb2 as fee_pb
|
||||
from lbrynet.schema.schema import VERSION_MAP, CURRENCY_MAP
|
||||
|
||||
|
||||
class Fee(Schema):
|
||||
@classmethod
|
||||
def load(cls, message):
|
||||
_fee = deepcopy(message)
|
||||
currency = CURRENCY_MAP[_fee.pop('currency')]
|
||||
_message_pb = fee_pb.Fee()
|
||||
_message_pb.version = VERSION_MAP[_fee.pop("version")]
|
||||
_message_pb.currency = currency
|
||||
_message_pb.address = _fee.pop('address')
|
||||
return cls._load(_fee, _message_pb)
|
||||
@@ -1,17 +0,0 @@
|
||||
from copy import deepcopy
|
||||
from lbrynet.schema.proto import metadata_pb2 as metadata_pb
|
||||
from lbrynet.schema.schema.fee import Fee
|
||||
from lbrynet.schema.schema.schema import Schema
|
||||
from lbrynet.schema.schema import VERSION_MAP
|
||||
|
||||
|
||||
class Metadata(Schema):
|
||||
@classmethod
|
||||
def load(cls, message):
|
||||
_metadata = deepcopy(message)
|
||||
_message_pb = metadata_pb.Metadata()
|
||||
_message_pb.version = VERSION_MAP[_metadata.pop("version")]
|
||||
if 'fee' in _metadata:
|
||||
fee_pb = Fee.load(_metadata.pop('fee'))
|
||||
_message_pb.fee.CopyFrom(fee_pb)
|
||||
return cls._load(_metadata, _message_pb)
|
||||
@@ -1,15 +0,0 @@
|
||||
import json
|
||||
import google.protobuf.json_format as json_pb # pylint: disable=no-name-in-module
|
||||
from google.protobuf.message import Message # pylint: disable=no-name-in-module,import-error
|
||||
|
||||
|
||||
class Schema(Message):
|
||||
@classmethod
|
||||
def load(cls, message):
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def _load(cls, data, message):
|
||||
if isinstance(data, dict):
|
||||
data = json.dumps(data)
|
||||
return json_pb.Parse(data, message)
|
||||
@@ -1,17 +0,0 @@
|
||||
from copy import deepcopy
|
||||
|
||||
from lbrynet.schema.proto import signature_pb2 as signature_pb
|
||||
from lbrynet.schema.schema import VERSION_MAP, ECDSA_CURVES
|
||||
from lbrynet.schema.schema.schema import Schema
|
||||
|
||||
|
||||
class Signature(Schema):
|
||||
@classmethod
|
||||
def load(cls, message):
|
||||
_signature = deepcopy(message)
|
||||
_message_pb = signature_pb.Signature()
|
||||
_message_pb.version = VERSION_MAP[_signature.pop("version")]
|
||||
_message_pb.signatureType = ECDSA_CURVES[_signature.pop("signatureType")]
|
||||
_message_pb.certificateId = _signature.pop("certificateId")
|
||||
_message_pb.signature = _signature.pop("signature")
|
||||
return cls._load(_signature, _message_pb)
|
||||
@@ -1,19 +0,0 @@
|
||||
from copy import deepcopy
|
||||
from lbrynet.schema.proto import source_pb2 as source_pb
|
||||
from lbrynet.schema.schema import SOURCE_TYPES, LBRY_SD_HASH_LENGTH, VERSION_MAP
|
||||
from lbrynet.schema.schema.schema import Schema
|
||||
from lbrynet.schema.error import InvalidSourceHashLength
|
||||
|
||||
|
||||
class Source(Schema):
|
||||
@classmethod
|
||||
def load(cls, message):
|
||||
_source = deepcopy(message)
|
||||
sd_hash = _source.pop('source')
|
||||
assert len(sd_hash) == LBRY_SD_HASH_LENGTH, InvalidSourceHashLength(len(sd_hash))
|
||||
_message_pb = source_pb.Source()
|
||||
_message_pb.version = VERSION_MAP[_source.pop("version")]
|
||||
_message_pb.sourceType = SOURCE_TYPES[_source.pop('sourceType')]
|
||||
_message_pb.source = sd_hash
|
||||
_message_pb.contentType = _source.pop('contentType')
|
||||
return cls._load(_source, _message_pb)
|
||||
@@ -1,20 +0,0 @@
|
||||
from copy import deepcopy
|
||||
|
||||
from lbrynet.schema.schema.source import Source
|
||||
from lbrynet.schema.proto import stream_pb2 as stream_pb
|
||||
from lbrynet.schema.schema import VERSION_MAP
|
||||
from lbrynet.schema.schema.metadata import Metadata
|
||||
from lbrynet.schema.schema.schema import Schema
|
||||
|
||||
|
||||
class Stream(Schema):
|
||||
@classmethod
|
||||
def load(cls, message):
|
||||
_claim = deepcopy(message)
|
||||
source = Source.load(_claim.pop('source'))
|
||||
metadata = Metadata.load(_claim.pop('metadata'))
|
||||
_message_pb = stream_pb.Stream()
|
||||
_message_pb.version = VERSION_MAP[_claim.pop("version")]
|
||||
_message_pb.source.CopyFrom(source)
|
||||
_message_pb.metadata.CopyFrom(metadata)
|
||||
return cls._load(_claim, _message_pb)
|
||||
@@ -1,44 +0,0 @@
|
||||
from collections import namedtuple
|
||||
|
||||
from typing import Union
|
||||
|
||||
LEGACY = namedtuple('Legacy', 'payload')
|
||||
NAMED_SECP256K1 = namedtuple('NamedSECP256k1', 'raw_signature certificate_id payload')
|
||||
FLAGS = {
|
||||
LEGACY: 0x80,
|
||||
NAMED_SECP256K1: 0x01
|
||||
}
|
||||
|
||||
class Signature:
|
||||
|
||||
def __init__(self, data: Union[LEGACY, NAMED_SECP256K1]):
|
||||
self.data = data
|
||||
|
||||
@property
|
||||
def payload(self):
|
||||
return self.data.payload
|
||||
|
||||
@property
|
||||
def certificate_id(self):
|
||||
if type(self.data) == NAMED_SECP256K1:
|
||||
return self.data.certificate_id
|
||||
|
||||
@property
|
||||
def raw_signature(self):
|
||||
if type(self.data) == NAMED_SECP256K1:
|
||||
return self.data.raw_signature
|
||||
|
||||
@classmethod
|
||||
def flagged_parse(cls, binary: bytes):
|
||||
flag = binary[0]
|
||||
if flag == FLAGS[NAMED_SECP256K1]:
|
||||
return cls(NAMED_SECP256K1(binary[1:65], binary[65:85], binary[85:]))
|
||||
else:
|
||||
return cls(LEGACY(binary))
|
||||
|
||||
@property
|
||||
def serialized(self):
|
||||
if isinstance(self.data, NAMED_SECP256K1):
|
||||
return (bytes([FLAGS[type(self.data)]]) + self.data.raw_signature + self.data.certificate_id + self.payload)
|
||||
elif isinstance(self.data, LEGACY):
|
||||
return self.payload
|
||||
@@ -1,121 +0,0 @@
|
||||
import ecdsa
|
||||
import hashlib
|
||||
import binascii
|
||||
from lbrynet.schema.address import decode_address
|
||||
from lbrynet.schema.encoding import decode_b64_fields
|
||||
from lbrynet.schema.schema.certificate import Certificate
|
||||
from lbrynet.schema.schema.claim import Claim
|
||||
from lbrynet.schema.signature import Signature, NAMED_SECP256K1
|
||||
from lbrynet.schema.validator import validate_claim_id
|
||||
from lbrynet.schema.schema import V_0_0_1, CLAIM_TYPE, CLAIM_TYPES, CERTIFICATE_TYPE, VERSION
|
||||
from lbrynet.schema.schema import NIST256p, NIST384p, SECP256k1, SHA256, SHA384
|
||||
|
||||
|
||||
class NIST_ECDSASigner(object):
|
||||
CURVE = None
|
||||
CURVE_NAME = None
|
||||
HASHFUNC = hashlib.sha256
|
||||
HASHFUNC_NAME = SHA256
|
||||
|
||||
def __init__(self, private_key):
|
||||
self._private_key = private_key
|
||||
|
||||
@property
|
||||
def private_key(self):
|
||||
return self._private_key
|
||||
|
||||
@property
|
||||
def public_key(self):
|
||||
return self.private_key.get_verifying_key()
|
||||
|
||||
@property
|
||||
def certificate(self):
|
||||
certificate_claim = {
|
||||
VERSION: V_0_0_1,
|
||||
CLAIM_TYPE: CERTIFICATE_TYPE,
|
||||
CLAIM_TYPES[CERTIFICATE_TYPE]: Certificate.load_from_key_obj(self.public_key,
|
||||
self.CURVE_NAME)
|
||||
}
|
||||
return Claim.load(certificate_claim)
|
||||
|
||||
@classmethod
|
||||
def load_pem(cls, pem_string):
|
||||
return cls(ecdsa.SigningKey.from_pem(pem_string, hashfunc=cls.HASHFUNC_NAME))
|
||||
|
||||
@classmethod
|
||||
def generate(cls):
|
||||
return cls(ecdsa.SigningKey.generate(curve=cls.CURVE, hashfunc=cls.HASHFUNC_NAME))
|
||||
|
||||
def sign(self, *fields):
|
||||
digest = self.HASHFUNC(bytearray(b''.join(fields))).digest()
|
||||
return self.private_key.sign_digest_deterministic(digest, hashfunc=self.HASHFUNC)
|
||||
|
||||
def sign_stream_claim(self, claim, claim_address, cert_claim_id, name, detached=False):
|
||||
validate_claim_id(cert_claim_id)
|
||||
raw_cert_id = binascii.unhexlify(cert_claim_id)
|
||||
decoded_addr = decode_address(claim_address)
|
||||
if detached:
|
||||
assert name, "Name is required for detached signatures"
|
||||
assert self.CURVE_NAME == SECP256k1, f"Only SECP256k1 is supported, not: {self.CURVE_NAME}"
|
||||
signature = self.sign(
|
||||
name.lower().encode(),
|
||||
decoded_addr,
|
||||
claim.serialized_no_signature,
|
||||
raw_cert_id,
|
||||
)
|
||||
else:
|
||||
signature = self.sign(decoded_addr, claim.serialized_no_signature, raw_cert_id)
|
||||
|
||||
if detached:
|
||||
return Claim.load(decode_b64_fields(claim.protobuf_dict)), Signature(NAMED_SECP256K1(
|
||||
signature,
|
||||
raw_cert_id,
|
||||
claim.serialized_no_signature
|
||||
))
|
||||
# -- Legacy signer (signature inside protobuf) --
|
||||
|
||||
if not isinstance(self.private_key, ecdsa.SigningKey):
|
||||
raise Exception("Not given a signing key")
|
||||
sig_dict = {
|
||||
"version": V_0_0_1,
|
||||
"signatureType": self.CURVE_NAME,
|
||||
"signature": signature,
|
||||
"certificateId": raw_cert_id
|
||||
}
|
||||
|
||||
msg = {
|
||||
"version": V_0_0_1,
|
||||
"stream": decode_b64_fields(claim.protobuf_dict)['stream'],
|
||||
"publisherSignature": sig_dict
|
||||
}
|
||||
|
||||
proto = Claim.load(msg)
|
||||
return proto, Signature.flagged_parse(proto.SerializeToString())
|
||||
|
||||
|
||||
class NIST256pSigner(NIST_ECDSASigner):
|
||||
CURVE = ecdsa.NIST256p
|
||||
CURVE_NAME = NIST256p
|
||||
|
||||
|
||||
class NIST384pSigner(NIST_ECDSASigner):
|
||||
CURVE = ecdsa.NIST384p
|
||||
CURVE_NAME = NIST384p
|
||||
HASHFUNC = hashlib.sha384
|
||||
HASHFUNC_NAME = SHA384
|
||||
|
||||
|
||||
class SECP256k1Signer(NIST_ECDSASigner):
|
||||
CURVE = ecdsa.SECP256k1
|
||||
CURVE_NAME = SECP256k1
|
||||
|
||||
|
||||
def get_signer(curve):
|
||||
if curve == NIST256p:
|
||||
return NIST256pSigner
|
||||
elif curve == NIST384p:
|
||||
return NIST384pSigner
|
||||
elif curve == SECP256k1:
|
||||
return SECP256k1Signer
|
||||
else:
|
||||
raise Exception("Unknown curve: %s" % str(curve))
|
||||
@@ -0,0 +1,6 @@
|
||||
from lbrynet.schema.base import Signable
|
||||
|
||||
|
||||
class Support(Signable):
|
||||
__slots__ = ()
|
||||
message_class = None # TODO: add support protobufs
|
||||
File renamed without changes.
File renamed without changes.
+18
-18
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/certificate.proto
|
||||
# source: certificate.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -16,16 +16,16 @@ _sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/certificate.proto',
|
||||
package='',
|
||||
name='certificate.proto',
|
||||
package='legacy_pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n&lbrynet/schema/proto/certificate.proto\"\x8e\x01\n\x0b\x43\x65rtificate\x12%\n\x07version\x18\x01 \x02(\x0e\x32\x14.Certificate.Version\x12\x19\n\x07keyType\x18\x02 \x02(\x0e\x32\x08.KeyType\x12\x11\n\tpublicKey\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01*Q\n\x07KeyType\x12\x1b\n\x17UNKNOWN_PUBLIC_KEY_TYPE\x10\x00\x12\x0c\n\x08NIST256p\x10\x01\x12\x0c\n\x08NIST384p\x10\x02\x12\r\n\tSECP256k1\x10\x03')
|
||||
serialized_pb=_b('\n\x11\x63\x65rtificate.proto\x12\tlegacy_pb\"\xa2\x01\n\x0b\x43\x65rtificate\x12/\n\x07version\x18\x01 \x02(\x0e\x32\x1e.legacy_pb.Certificate.Version\x12#\n\x07keyType\x18\x02 \x02(\x0e\x32\x12.legacy_pb.KeyType\x12\x11\n\tpublicKey\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01*Q\n\x07KeyType\x12\x1b\n\x17UNKNOWN_PUBLIC_KEY_TYPE\x10\x00\x12\x0c\n\x08NIST256p\x10\x01\x12\x0c\n\x08NIST384p\x10\x02\x12\r\n\tSECP256k1\x10\x03')
|
||||
)
|
||||
|
||||
_KEYTYPE = _descriptor.EnumDescriptor(
|
||||
name='KeyType',
|
||||
full_name='KeyType',
|
||||
full_name='legacy_pb.KeyType',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -48,8 +48,8 @@ _KEYTYPE = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=187,
|
||||
serialized_end=268,
|
||||
serialized_start=197,
|
||||
serialized_end=278,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_KEYTYPE)
|
||||
|
||||
@@ -62,7 +62,7 @@ SECP256k1 = 3
|
||||
|
||||
_CERTIFICATE_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Certificate.Version',
|
||||
full_name='legacy_pb.Certificate.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -77,35 +77,35 @@ _CERTIFICATE_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=143,
|
||||
serialized_end=185,
|
||||
serialized_start=153,
|
||||
serialized_end=195,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_CERTIFICATE_VERSION)
|
||||
|
||||
|
||||
_CERTIFICATE = _descriptor.Descriptor(
|
||||
name='Certificate',
|
||||
full_name='Certificate',
|
||||
full_name='legacy_pb.Certificate',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Certificate.version', index=0,
|
||||
name='version', full_name='legacy_pb.Certificate.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='keyType', full_name='Certificate.keyType', index=1,
|
||||
name='keyType', full_name='legacy_pb.Certificate.keyType', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='publicKey', full_name='Certificate.publicKey', index=2,
|
||||
name='publicKey', full_name='legacy_pb.Certificate.publicKey', index=2,
|
||||
number=4, type=12, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -124,8 +124,8 @@ _CERTIFICATE = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=43,
|
||||
serialized_end=185,
|
||||
serialized_start=33,
|
||||
serialized_end=195,
|
||||
)
|
||||
|
||||
_CERTIFICATE.fields_by_name['version'].enum_type = _CERTIFICATE_VERSION
|
||||
@@ -137,8 +137,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Certificate = _reflection.GeneratedProtocolMessageType('Certificate', (_message.Message,), dict(
|
||||
DESCRIPTOR = _CERTIFICATE,
|
||||
__module__ = 'lbrynet.schema.proto.certificate_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Certificate)
|
||||
__module__ = 'certificate_pb2'
|
||||
# @@protoc_insertion_point(class_scope:legacy_pb.Certificate)
|
||||
))
|
||||
_sym_db.RegisterMessage(Certificate)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/fee.proto
|
||||
# source: fee.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -15,18 +15,18 @@ _sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/fee.proto',
|
||||
package='',
|
||||
name='fee.proto',
|
||||
package='legacy_pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n\x1elbrynet/schema/proto/fee.proto\"\xcf\x01\n\x03\x46\x65\x65\x12\x1d\n\x07version\x18\x01 \x02(\x0e\x32\x0c.Fee.Version\x12\x1f\n\x08\x63urrency\x18\x02 \x02(\x0e\x32\r.Fee.Currency\x12\x0f\n\x07\x61\x64\x64ress\x18\x03 \x02(\x0c\x12\x0e\n\x06\x61mount\x18\x04 \x02(\x02\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\";\n\x08\x43urrency\x12\x14\n\x10UNKNOWN_CURRENCY\x10\x00\x12\x07\n\x03LBC\x10\x01\x12\x07\n\x03\x42TC\x10\x02\x12\x07\n\x03USD\x10\x03')
|
||||
serialized_pb=_b('\n\tfee.proto\x12\tlegacy_pb\"\xe3\x01\n\x03\x46\x65\x65\x12\'\n\x07version\x18\x01 \x02(\x0e\x32\x16.legacy_pb.Fee.Version\x12)\n\x08\x63urrency\x18\x02 \x02(\x0e\x32\x17.legacy_pb.Fee.Currency\x12\x0f\n\x07\x61\x64\x64ress\x18\x03 \x02(\x0c\x12\x0e\n\x06\x61mount\x18\x04 \x02(\x02\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\";\n\x08\x43urrency\x12\x14\n\x10UNKNOWN_CURRENCY\x10\x00\x12\x07\n\x03LBC\x10\x01\x12\x07\n\x03\x42TC\x10\x02\x12\x07\n\x03USD\x10\x03')
|
||||
)
|
||||
|
||||
|
||||
|
||||
_FEE_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Fee.Version',
|
||||
full_name='legacy_pb.Fee.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -41,14 +41,14 @@ _FEE_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=139,
|
||||
serialized_end=181,
|
||||
serialized_start=149,
|
||||
serialized_end=191,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_FEE_VERSION)
|
||||
|
||||
_FEE_CURRENCY = _descriptor.EnumDescriptor(
|
||||
name='Currency',
|
||||
full_name='Fee.Currency',
|
||||
full_name='legacy_pb.Fee.Currency',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -71,42 +71,42 @@ _FEE_CURRENCY = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=183,
|
||||
serialized_end=242,
|
||||
serialized_start=193,
|
||||
serialized_end=252,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_FEE_CURRENCY)
|
||||
|
||||
|
||||
_FEE = _descriptor.Descriptor(
|
||||
name='Fee',
|
||||
full_name='Fee',
|
||||
full_name='legacy_pb.Fee',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Fee.version', index=0,
|
||||
name='version', full_name='legacy_pb.Fee.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='currency', full_name='Fee.currency', index=1,
|
||||
name='currency', full_name='legacy_pb.Fee.currency', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='address', full_name='Fee.address', index=2,
|
||||
name='address', full_name='legacy_pb.Fee.address', index=2,
|
||||
number=3, type=12, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='amount', full_name='Fee.amount', index=3,
|
||||
name='amount', full_name='legacy_pb.Fee.amount', index=3,
|
||||
number=4, type=2, cpp_type=6, label=2,
|
||||
has_default_value=False, default_value=float(0),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -126,8 +126,8 @@ _FEE = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=35,
|
||||
serialized_end=242,
|
||||
serialized_start=25,
|
||||
serialized_end=252,
|
||||
)
|
||||
|
||||
_FEE.fields_by_name['version'].enum_type = _FEE_VERSION
|
||||
@@ -139,8 +139,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Fee = _reflection.GeneratedProtocolMessageType('Fee', (_message.Message,), dict(
|
||||
DESCRIPTOR = _FEE,
|
||||
__module__ = 'lbrynet.schema.proto.fee_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Fee)
|
||||
__module__ = 'fee_pb2'
|
||||
# @@protoc_insertion_point(class_scope:legacy_pb.Fee)
|
||||
))
|
||||
_sym_db.RegisterMessage(Fee)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/claim.proto
|
||||
# source: legacy_claim.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -12,25 +12,25 @@ from google.protobuf import symbol_database as _symbol_database
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
from lbrynet.schema.proto import stream_pb2 as lbrynet_dot_schema_dot_proto_dot_stream__pb2
|
||||
from lbrynet.schema.proto import certificate_pb2 as lbrynet_dot_schema_dot_proto_dot_certificate__pb2
|
||||
from lbrynet.schema.proto import signature_pb2 as lbrynet_dot_schema_dot_proto_dot_signature__pb2
|
||||
from . import stream_pb2 as stream__pb2
|
||||
from . import certificate_pb2 as certificate__pb2
|
||||
from . import signature_pb2 as signature__pb2
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/claim.proto',
|
||||
package='',
|
||||
name='legacy_claim.proto',
|
||||
package='legacy_pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n lbrynet/schema/proto/claim.proto\x1a!lbrynet/schema/proto/stream.proto\x1a&lbrynet/schema/proto/certificate.proto\x1a$lbrynet/schema/proto/signature.proto\"\xa7\x02\n\x05\x43laim\x12\x1f\n\x07version\x18\x01 \x02(\x0e\x32\x0e.Claim.Version\x12#\n\tclaimType\x18\x02 \x02(\x0e\x32\x10.Claim.ClaimType\x12\x17\n\x06stream\x18\x03 \x01(\x0b\x32\x07.Stream\x12!\n\x0b\x63\x65rtificate\x18\x04 \x01(\x0b\x32\x0c.Certificate\x12&\n\x12publisherSignature\x18\x05 \x01(\x0b\x32\n.Signature\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"H\n\tClaimType\x12\x16\n\x12UNKNOWN_CLAIM_TYPE\x10\x00\x12\x0e\n\nstreamType\x10\x01\x12\x13\n\x0f\x63\x65rtificateType\x10\x02')
|
||||
serialized_pb=_b('\n\x12legacy_claim.proto\x12\tlegacy_pb\x1a\x0cstream.proto\x1a\x11\x63\x65rtificate.proto\x1a\x0fsignature.proto\"\xd9\x02\n\x05\x43laim\x12)\n\x07version\x18\x01 \x02(\x0e\x32\x18.legacy_pb.Claim.Version\x12-\n\tclaimType\x18\x02 \x02(\x0e\x32\x1a.legacy_pb.Claim.ClaimType\x12!\n\x06stream\x18\x03 \x01(\x0b\x32\x11.legacy_pb.Stream\x12+\n\x0b\x63\x65rtificate\x18\x04 \x01(\x0b\x32\x16.legacy_pb.Certificate\x12\x30\n\x12publisherSignature\x18\x05 \x01(\x0b\x32\x14.legacy_pb.Signature\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"H\n\tClaimType\x12\x16\n\x12UNKNOWN_CLAIM_TYPE\x10\x00\x12\x0e\n\nstreamType\x10\x01\x12\x13\n\x0f\x63\x65rtificateType\x10\x02')
|
||||
,
|
||||
dependencies=[lbrynet_dot_schema_dot_proto_dot_stream__pb2.DESCRIPTOR,lbrynet_dot_schema_dot_proto_dot_certificate__pb2.DESCRIPTOR,lbrynet_dot_schema_dot_proto_dot_signature__pb2.DESCRIPTOR,])
|
||||
dependencies=[stream__pb2.DESCRIPTOR,certificate__pb2.DESCRIPTOR,signature__pb2.DESCRIPTOR,])
|
||||
|
||||
|
||||
|
||||
_CLAIM_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Claim.Version',
|
||||
full_name='legacy_pb.Claim.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -45,14 +45,14 @@ _CLAIM_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=329,
|
||||
serialized_end=371,
|
||||
serialized_start=313,
|
||||
serialized_end=355,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_CLAIM_VERSION)
|
||||
|
||||
_CLAIM_CLAIMTYPE = _descriptor.EnumDescriptor(
|
||||
name='ClaimType',
|
||||
full_name='Claim.ClaimType',
|
||||
full_name='legacy_pb.Claim.ClaimType',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -71,49 +71,49 @@ _CLAIM_CLAIMTYPE = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=373,
|
||||
serialized_end=445,
|
||||
serialized_start=357,
|
||||
serialized_end=429,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_CLAIM_CLAIMTYPE)
|
||||
|
||||
|
||||
_CLAIM = _descriptor.Descriptor(
|
||||
name='Claim',
|
||||
full_name='Claim',
|
||||
full_name='legacy_pb.Claim',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Claim.version', index=0,
|
||||
name='version', full_name='legacy_pb.Claim.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='claimType', full_name='Claim.claimType', index=1,
|
||||
name='claimType', full_name='legacy_pb.Claim.claimType', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='stream', full_name='Claim.stream', index=2,
|
||||
name='stream', full_name='legacy_pb.Claim.stream', index=2,
|
||||
number=3, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='certificate', full_name='Claim.certificate', index=3,
|
||||
name='certificate', full_name='legacy_pb.Claim.certificate', index=3,
|
||||
number=4, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='publisherSignature', full_name='Claim.publisherSignature', index=4,
|
||||
name='publisherSignature', full_name='legacy_pb.Claim.publisherSignature', index=4,
|
||||
number=5, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -133,15 +133,15 @@ _CLAIM = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=150,
|
||||
serialized_end=445,
|
||||
serialized_start=84,
|
||||
serialized_end=429,
|
||||
)
|
||||
|
||||
_CLAIM.fields_by_name['version'].enum_type = _CLAIM_VERSION
|
||||
_CLAIM.fields_by_name['claimType'].enum_type = _CLAIM_CLAIMTYPE
|
||||
_CLAIM.fields_by_name['stream'].message_type = lbrynet_dot_schema_dot_proto_dot_stream__pb2._STREAM
|
||||
_CLAIM.fields_by_name['certificate'].message_type = lbrynet_dot_schema_dot_proto_dot_certificate__pb2._CERTIFICATE
|
||||
_CLAIM.fields_by_name['publisherSignature'].message_type = lbrynet_dot_schema_dot_proto_dot_signature__pb2._SIGNATURE
|
||||
_CLAIM.fields_by_name['stream'].message_type = stream__pb2._STREAM
|
||||
_CLAIM.fields_by_name['certificate'].message_type = certificate__pb2._CERTIFICATE
|
||||
_CLAIM.fields_by_name['publisherSignature'].message_type = signature__pb2._SIGNATURE
|
||||
_CLAIM_VERSION.containing_type = _CLAIM
|
||||
_CLAIM_CLAIMTYPE.containing_type = _CLAIM
|
||||
DESCRIPTOR.message_types_by_name['Claim'] = _CLAIM
|
||||
@@ -149,8 +149,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Claim = _reflection.GeneratedProtocolMessageType('Claim', (_message.Message,), dict(
|
||||
DESCRIPTOR = _CLAIM,
|
||||
__module__ = 'lbrynet.schema.proto.claim_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Claim)
|
||||
__module__ = 'legacy_claim_pb2'
|
||||
# @@protoc_insertion_point(class_scope:legacy_pb.Claim)
|
||||
))
|
||||
_sym_db.RegisterMessage(Claim)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/metadata.proto
|
||||
# source: metadata.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -12,23 +12,23 @@ from google.protobuf import symbol_database as _symbol_database
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
from lbrynet.schema.proto import fee_pb2 as lbrynet_dot_schema_dot_proto_dot_fee__pb2
|
||||
from . import fee_pb2 as fee__pb2
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/metadata.proto',
|
||||
package='',
|
||||
name='metadata.proto',
|
||||
package='legacy_pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n#lbrynet/schema/proto/metadata.proto\x1a\x1elbrynet/schema/proto/fee.proto\"\xde\x0e\n\x08Metadata\x12\"\n\x07version\x18\x01 \x02(\x0e\x32\x11.Metadata.Version\x12$\n\x08language\x18\x02 \x02(\x0e\x32\x12.Metadata.Language\x12\r\n\x05title\x18\x03 \x02(\t\x12\x13\n\x0b\x64\x65scription\x18\x04 \x02(\t\x12\x0e\n\x06\x61uthor\x18\x05 \x02(\t\x12\x0f\n\x07license\x18\x06 \x02(\t\x12\x0c\n\x04nsfw\x18\x07 \x02(\x08\x12\x11\n\x03\x66\x65\x65\x18\x08 \x01(\x0b\x32\x04.Fee\x12\x11\n\tthumbnail\x18\t \x01(\t\x12\x0f\n\x07preview\x18\n \x01(\t\x12\x12\n\nlicenseUrl\x18\x0b \x01(\t\"N\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\x12\n\n\x06_0_0_2\x10\x02\x12\n\n\x06_0_0_3\x10\x03\x12\n\n\x06_0_1_0\x10\x04\"\x99\x0c\n\x08Language\x12\x14\n\x10UNKNOWN_LANGUAGE\x10\x00\x12\x06\n\x02\x65n\x10\x01\x12\x06\n\x02\x61\x61\x10\x02\x12\x06\n\x02\x61\x62\x10\x03\x12\x06\n\x02\x61\x65\x10\x04\x12\x06\n\x02\x61\x66\x10\x05\x12\x06\n\x02\x61k\x10\x06\x12\x06\n\x02\x61m\x10\x07\x12\x06\n\x02\x61n\x10\x08\x12\x06\n\x02\x61r\x10\t\x12\x06\n\x02\x61s\x10\n\x12\x06\n\x02\x61v\x10\x0b\x12\x06\n\x02\x61y\x10\x0c\x12\x06\n\x02\x61z\x10\r\x12\x06\n\x02\x62\x61\x10\x0e\x12\x06\n\x02\x62\x65\x10\x0f\x12\x06\n\x02\x62g\x10\x10\x12\x06\n\x02\x62h\x10\x11\x12\x06\n\x02\x62i\x10\x12\x12\x06\n\x02\x62m\x10\x13\x12\x06\n\x02\x62n\x10\x14\x12\x06\n\x02\x62o\x10\x15\x12\x06\n\x02\x62r\x10\x16\x12\x06\n\x02\x62s\x10\x17\x12\x06\n\x02\x63\x61\x10\x18\x12\x06\n\x02\x63\x65\x10\x19\x12\x06\n\x02\x63h\x10\x1a\x12\x06\n\x02\x63o\x10\x1b\x12\x06\n\x02\x63r\x10\x1c\x12\x06\n\x02\x63s\x10\x1d\x12\x06\n\x02\x63u\x10\x1e\x12\x06\n\x02\x63v\x10\x1f\x12\x06\n\x02\x63y\x10 \x12\x06\n\x02\x64\x61\x10!\x12\x06\n\x02\x64\x65\x10\"\x12\x06\n\x02\x64v\x10#\x12\x06\n\x02\x64z\x10$\x12\x06\n\x02\x65\x65\x10%\x12\x06\n\x02\x65l\x10&\x12\x06\n\x02\x65o\x10\'\x12\x06\n\x02\x65s\x10(\x12\x06\n\x02\x65t\x10)\x12\x06\n\x02\x65u\x10*\x12\x06\n\x02\x66\x61\x10+\x12\x06\n\x02\x66\x66\x10,\x12\x06\n\x02\x66i\x10-\x12\x06\n\x02\x66j\x10.\x12\x06\n\x02\x66o\x10/\x12\x06\n\x02\x66r\x10\x30\x12\x06\n\x02\x66y\x10\x31\x12\x06\n\x02ga\x10\x32\x12\x06\n\x02gd\x10\x33\x12\x06\n\x02gl\x10\x34\x12\x06\n\x02gn\x10\x35\x12\x06\n\x02gu\x10\x36\x12\x06\n\x02gv\x10\x37\x12\x06\n\x02ha\x10\x38\x12\x06\n\x02he\x10\x39\x12\x06\n\x02hi\x10:\x12\x06\n\x02ho\x10;\x12\x06\n\x02hr\x10<\x12\x06\n\x02ht\x10=\x12\x06\n\x02hu\x10>\x12\x06\n\x02hy\x10?\x12\x06\n\x02hz\x10@\x12\x06\n\x02ia\x10\x41\x12\x06\n\x02id\x10\x42\x12\x06\n\x02ie\x10\x43\x12\x06\n\x02ig\x10\x44\x12\x06\n\x02ii\x10\x45\x12\x06\n\x02ik\x10\x46\x12\x06\n\x02io\x10G\x12\x06\n\x02is\x10H\x12\x06\n\x02it\x10I\x12\x06\n\x02iu\x10J\x12\x06\n\x02ja\x10K\x12\x06\n\x02jv\x10L\x12\x06\n\x02ka\x10M\x12\x06\n\x02kg\x10N\x12\x06\n\x02ki\x10O\x12\x06\n\x02kj\x10P\x12\x06\n\x02kk\x10Q\x12\x06\n\x02kl\x10R\x12\x06\n\x02km\x10S\x12\x06\n\x02kn\x10T\x12\x06\n\x02ko\x10U\x12\x06\n\x02kr\x10V\x12\x06\n\x02ks\x10W\x12\x06\n\x02ku\x10X\x12\x06\n\x02kv\x10Y\x12\x06\n\x02kw\x10Z\x12\x06\n\x02ky\x10[\x12\x06\n\x02la\x10\\\x12\x06\n\x02lb\x10]\x12\x06\n\x02lg\x10^\x12\x06\n\x02li\x10_\x12\x06\n\x02ln\x10`\x12\x06\n\x02lo\x10\x61\x12\x06\n\x02lt\x10\x62\x12\x06\n\x02lu\x10\x63\x12\x06\n\x02lv\x10\x64\x12\x06\n\x02mg\x10\x65\x12\x06\n\x02mh\x10\x66\x12\x06\n\x02mi\x10g\x12\x06\n\x02mk\x10h\x12\x06\n\x02ml\x10i\x12\x06\n\x02mn\x10j\x12\x06\n\x02mr\x10k\x12\x06\n\x02ms\x10l\x12\x06\n\x02mt\x10m\x12\x06\n\x02my\x10n\x12\x06\n\x02na\x10o\x12\x06\n\x02nb\x10p\x12\x06\n\x02nd\x10q\x12\x06\n\x02ne\x10r\x12\x06\n\x02ng\x10s\x12\x06\n\x02nl\x10t\x12\x06\n\x02nn\x10u\x12\x06\n\x02no\x10v\x12\x06\n\x02nr\x10w\x12\x06\n\x02nv\x10x\x12\x06\n\x02ny\x10y\x12\x06\n\x02oc\x10z\x12\x06\n\x02oj\x10{\x12\x06\n\x02om\x10|\x12\x06\n\x02or\x10}\x12\x06\n\x02os\x10~\x12\x06\n\x02pa\x10\x7f\x12\x07\n\x02pi\x10\x80\x01\x12\x07\n\x02pl\x10\x81\x01\x12\x07\n\x02ps\x10\x82\x01\x12\x07\n\x02pt\x10\x83\x01\x12\x07\n\x02qu\x10\x84\x01\x12\x07\n\x02rm\x10\x85\x01\x12\x07\n\x02rn\x10\x86\x01\x12\x07\n\x02ro\x10\x87\x01\x12\x07\n\x02ru\x10\x88\x01\x12\x07\n\x02rw\x10\x89\x01\x12\x07\n\x02sa\x10\x8a\x01\x12\x07\n\x02sc\x10\x8b\x01\x12\x07\n\x02sd\x10\x8c\x01\x12\x07\n\x02se\x10\x8d\x01\x12\x07\n\x02sg\x10\x8e\x01\x12\x07\n\x02si\x10\x8f\x01\x12\x07\n\x02sk\x10\x90\x01\x12\x07\n\x02sl\x10\x91\x01\x12\x07\n\x02sm\x10\x92\x01\x12\x07\n\x02sn\x10\x93\x01\x12\x07\n\x02so\x10\x94\x01\x12\x07\n\x02sq\x10\x95\x01\x12\x07\n\x02sr\x10\x96\x01\x12\x07\n\x02ss\x10\x97\x01\x12\x07\n\x02st\x10\x98\x01\x12\x07\n\x02su\x10\x99\x01\x12\x07\n\x02sv\x10\x9a\x01\x12\x07\n\x02sw\x10\x9b\x01\x12\x07\n\x02ta\x10\x9c\x01\x12\x07\n\x02te\x10\x9d\x01\x12\x07\n\x02tg\x10\x9e\x01\x12\x07\n\x02th\x10\x9f\x01\x12\x07\n\x02ti\x10\xa0\x01\x12\x07\n\x02tk\x10\xa1\x01\x12\x07\n\x02tl\x10\xa2\x01\x12\x07\n\x02tn\x10\xa3\x01\x12\x07\n\x02to\x10\xa4\x01\x12\x07\n\x02tr\x10\xa5\x01\x12\x07\n\x02ts\x10\xa6\x01\x12\x07\n\x02tt\x10\xa7\x01\x12\x07\n\x02tw\x10\xa8\x01\x12\x07\n\x02ty\x1Line truncated
|
||||
serialized_pb=_b('\n\x0emetadata.proto\x12\tlegacy_pb\x1a\tfee.proto\"\xfc\x0e\n\x08Metadata\x12,\n\x07version\x18\x01 \x02(\x0e\x32\x1b.legacy_pb.Metadata.Version\x12.\n\x08language\x18\x02 \x02(\x0e\x32\x1c.legacy_pb.Metadata.Language\x12\r\n\x05title\x18\x03 \x02(\t\x12\x13\n\x0b\x64\x65scription\x18\x04 \x02(\t\x12\x0e\n\x06\x61uthor\x18\x05 \x02(\t\x12\x0f\n\x07license\x18\x06 \x02(\t\x12\x0c\n\x04nsfw\x18\x07 \x02(\x08\x12\x1b\n\x03\x66\x65\x65\x18\x08 \x01(\x0b\x32\x0e.legacy_pb.Fee\x12\x11\n\tthumbnail\x18\t \x01(\t\x12\x0f\n\x07preview\x18\n \x01(\t\x12\x12\n\nlicenseUrl\x18\x0b \x01(\t\"N\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\x12\n\n\x06_0_0_2\x10\x02\x12\n\n\x06_0_0_3\x10\x03\x12\n\n\x06_0_1_0\x10\x04\"\x99\x0c\n\x08Language\x12\x14\n\x10UNKNOWN_LANGUAGE\x10\x00\x12\x06\n\x02\x65n\x10\x01\x12\x06\n\x02\x61\x61\x10\x02\x12\x06\n\x02\x61\x62\x10\x03\x12\x06\n\x02\x61\x65\x10\x04\x12\x06\n\x02\x61\x66\x10\x05\x12\x06\n\x02\x61k\x10\x06\x12\x06\n\x02\x61m\x10\x07\x12\x06\n\x02\x61n\x10\x08\x12\x06\n\x02\x61r\x10\t\x12\x06\n\x02\x61s\x10\n\x12\x06\n\x02\x61v\x10\x0b\x12\x06\n\x02\x61y\x10\x0c\x12\x06\n\x02\x61z\x10\r\x12\x06\n\x02\x62\x61\x10\x0e\x12\x06\n\x02\x62\x65\x10\x0f\x12\x06\n\x02\x62g\x10\x10\x12\x06\n\x02\x62h\x10\x11\x12\x06\n\x02\x62i\x10\x12\x12\x06\n\x02\x62m\x10\x13\x12\x06\n\x02\x62n\x10\x14\x12\x06\n\x02\x62o\x10\x15\x12\x06\n\x02\x62r\x10\x16\x12\x06\n\x02\x62s\x10\x17\x12\x06\n\x02\x63\x61\x10\x18\x12\x06\n\x02\x63\x65\x10\x19\x12\x06\n\x02\x63h\x10\x1a\x12\x06\n\x02\x63o\x10\x1b\x12\x06\n\x02\x63r\x10\x1c\x12\x06\n\x02\x63s\x10\x1d\x12\x06\n\x02\x63u\x10\x1e\x12\x06\n\x02\x63v\x10\x1f\x12\x06\n\x02\x63y\x10 \x12\x06\n\x02\x64\x61\x10!\x12\x06\n\x02\x64\x65\x10\"\x12\x06\n\x02\x64v\x10#\x12\x06\n\x02\x64z\x10$\x12\x06\n\x02\x65\x65\x10%\x12\x06\n\x02\x65l\x10&\x12\x06\n\x02\x65o\x10\'\x12\x06\n\x02\x65s\x10(\x12\x06\n\x02\x65t\x10)\x12\x06\n\x02\x65u\x10*\x12\x06\n\x02\x66\x61\x10+\x12\x06\n\x02\x66\x66\x10,\x12\x06\n\x02\x66i\x10-\x12\x06\n\x02\x66j\x10.\x12\x06\n\x02\x66o\x10/\x12\x06\n\x02\x66r\x10\x30\x12\x06\n\x02\x66y\x10\x31\x12\x06\n\x02ga\x10\x32\x12\x06\n\x02gd\x10\x33\x12\x06\n\x02gl\x10\x34\x12\x06\n\x02gn\x10\x35\x12\x06\n\x02gu\x10\x36\x12\x06\n\x02gv\x10\x37\x12\x06\n\x02ha\x10\x38\x12\x06\n\x02he\x10\x39\x12\x06\n\x02hi\x10:\x12\x06\n\x02ho\x10;\x12\x06\n\x02hr\x10<\x12\x06\n\x02ht\x10=\x12\x06\n\x02hu\x10>\x12\x06\n\x02hy\x10?\x12\x06\n\x02hz\x10@\x12\x06\n\x02ia\x10\x41\x12\x06\n\x02id\x10\x42\x12\x06\n\x02ie\x10\x43\x12\x06\n\x02ig\x10\x44\x12\x06\n\x02ii\x10\x45\x12\x06\n\x02ik\x10\x46\x12\x06\n\x02io\x10G\x12\x06\n\x02is\x10H\x12\x06\n\x02it\x10I\x12\x06\n\x02iu\x10J\x12\x06\n\x02ja\x10K\x12\x06\n\x02jv\x10L\x12\x06\n\x02ka\x10M\x12\x06\n\x02kg\x10N\x12\x06\n\x02ki\x10O\x12\x06\n\x02kj\x10P\x12\x06\n\x02kk\x10Q\x12\x06\n\x02kl\x10R\x12\x06\n\x02km\x10S\x12\x06\n\x02kn\x10T\x12\x06\n\x02ko\x10U\x12\x06\n\x02kr\x10V\x12\x06\n\x02ks\x10W\x12\x06\n\x02ku\x10X\x12\x06\n\x02kv\x10Y\x12\x06\n\x02kw\x10Z\x12\x06\n\x02ky\x10[\x12\x06\n\x02la\x10\\\x12\x06\n\x02lb\x10]\x12\x06\n\x02lg\x10^\x12\x06\n\x02li\x10_\x12\x06\n\x02ln\x10`\x12\x06\n\x02lo\x10\x61\x12\x06\n\x02lt\x10\x62\x12\x06\n\x02lu\x10\x63\x12\x06\n\x02lv\x10\x64\x12\x06\n\x02mg\x10\x65\x12\x06\n\x02mh\x10\x66\x12\x06\n\x02mi\x10g\x12\x06\n\x02mk\x10h\x12\x06\n\x02ml\x10i\x12\x06\n\x02mn\x10j\x12\x06\n\x02mr\x10k\x12\x06\n\x02ms\x10l\x12\x06\n\x02mt\x10m\x12\x06\n\x02my\x10n\x12\x06\n\x02na\x10o\x12\x06\n\x02nb\x10p\x12\x06\n\x02nd\x10q\x12\x06\n\x02ne\x10r\x12\x06\n\x02ng\x10s\x12\x06\n\x02nl\x10t\x12\x06\n\x02nn\x10u\x12\x06\n\x02no\x10v\x12\x06\n\x02nr\x10w\x12\x06\n\x02nv\x10x\x12\x06\n\x02ny\x10y\x12\x06\n\x02oc\x10z\x12\x06\n\x02oj\x10{\x12\x06\n\x02om\x10|\x12\x06\n\x02or\x10}\x12\x06\n\x02os\x10~\x12\x06\n\x02pa\x10\x7f\x12\x07\n\x02pi\x10\x80\x01\x12\x07\n\x02pl\x10\x81\x01\x12\x07\n\x02ps\x10\x82\x01\x12\x07\n\x02pt\x10\x83\x01\x12\x07\n\x02qu\x10\x84\x01\x12\x07\n\x02rm\x10\x85\x01\x12\x07\n\x02rn\x10\x86\x01\x12\x07\n\x02ro\x10\x87\x01\x12\x07\n\x02ru\x10\x88\x01\x12\x07\n\x02rw\x10\x89\x01\x12\x07\n\x02sa\x10\x8a\x01\x12\x07\n\x02sc\x10\x8b\x01\x12\x07\n\x02sd\x10\x8c\x01\x12\x07\n\x02se\x10\x8d\x01\x12\x07\n\x02sg\x10\x8e\x01\x12\x07\n\x02si\x10\x8f\x01\x12\x07\n\x02sk\x10\x90\x01\x12\x07\n\x02sl\x10\x91\x01\x12\x07\n\x02sm\x10\x92\x01\x12\x07\n\x02sn\x10\x93\x01\x12\x07\n\x02so\x10\x94\x01\x12\x07\n\x02sq\x10\x95\x01\x12\x07\n\x02sr\x10\x96\x01\x12\x07\n\x02ss\x10\x97\x01\x12\x07\n\x02st\x10\x98\x01\x12\x07\n\x02su\x10\x99\x01\x12\x07\n\x02sv\x10\x9a\x01\x12\x07\n\x02sw\x10\x9b\x01\x12\x07\n\x02ta\x10\x9c\x01\x12\x07\n\x02te\x10\x9d\x01\x12\x07\n\x02tg\x10\x9e\x01\x12\x07\n\x02th\x10\x9f\x01\x12\x07\n\x02ti\x10\xa0\x01\x12\x07\n\x02tk\x10\xa1\x01\x12\x07\n\x02tl\x10\xa2\x01\x12\x07\n\x02tn\x10\xa3\x01\x12\x07\n\x02to\x10\xa4\x01\x12\x07\n\x02tr\x10\xa5\x01\x12\x07\n\x02ts\x10\xa6\x01\x12\x07\n\x02tt\x10\xa7\x01\x12\x07\n\x02tw\x10\xa8\x01\x12\x07\n\x02tyLine truncated
|
||||
,
|
||||
dependencies=[lbrynet_dot_schema_dot_proto_dot_fee__pb2.DESCRIPTOR,])
|
||||
dependencies=[fee__pb2.DESCRIPTOR,])
|
||||
|
||||
|
||||
|
||||
_METADATA_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Metadata.Version',
|
||||
full_name='legacy_pb.Metadata.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -55,14 +55,14 @@ _METADATA_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=316,
|
||||
serialized_end=394,
|
||||
serialized_start=315,
|
||||
serialized_end=393,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_METADATA_VERSION)
|
||||
|
||||
_METADATA_LANGUAGE = _descriptor.EnumDescriptor(
|
||||
name='Language',
|
||||
full_name='Metadata.Language',
|
||||
full_name='legacy_pb.Metadata.Language',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -809,91 +809,91 @@ _METADATA_LANGUAGE = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=397,
|
||||
serialized_end=1958,
|
||||
serialized_start=396,
|
||||
serialized_end=1957,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_METADATA_LANGUAGE)
|
||||
|
||||
|
||||
_METADATA = _descriptor.Descriptor(
|
||||
name='Metadata',
|
||||
full_name='Metadata',
|
||||
full_name='legacy_pb.Metadata',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Metadata.version', index=0,
|
||||
name='version', full_name='legacy_pb.Metadata.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='language', full_name='Metadata.language', index=1,
|
||||
name='language', full_name='legacy_pb.Metadata.language', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='title', full_name='Metadata.title', index=2,
|
||||
name='title', full_name='legacy_pb.Metadata.title', index=2,
|
||||
number=3, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='description', full_name='Metadata.description', index=3,
|
||||
name='description', full_name='legacy_pb.Metadata.description', index=3,
|
||||
number=4, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='author', full_name='Metadata.author', index=4,
|
||||
name='author', full_name='legacy_pb.Metadata.author', index=4,
|
||||
number=5, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='license', full_name='Metadata.license', index=5,
|
||||
name='license', full_name='legacy_pb.Metadata.license', index=5,
|
||||
number=6, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='nsfw', full_name='Metadata.nsfw', index=6,
|
||||
name='nsfw', full_name='legacy_pb.Metadata.nsfw', index=6,
|
||||
number=7, type=8, cpp_type=7, label=2,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='fee', full_name='Metadata.fee', index=7,
|
||||
name='fee', full_name='legacy_pb.Metadata.fee', index=7,
|
||||
number=8, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='thumbnail', full_name='Metadata.thumbnail', index=8,
|
||||
name='thumbnail', full_name='legacy_pb.Metadata.thumbnail', index=8,
|
||||
number=9, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='preview', full_name='Metadata.preview', index=9,
|
||||
name='preview', full_name='legacy_pb.Metadata.preview', index=9,
|
||||
number=10, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='licenseUrl', full_name='Metadata.licenseUrl', index=10,
|
||||
name='licenseUrl', full_name='legacy_pb.Metadata.licenseUrl', index=10,
|
||||
number=11, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -913,13 +913,13 @@ _METADATA = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=72,
|
||||
serialized_end=1958,
|
||||
serialized_start=41,
|
||||
serialized_end=1957,
|
||||
)
|
||||
|
||||
_METADATA.fields_by_name['version'].enum_type = _METADATA_VERSION
|
||||
_METADATA.fields_by_name['language'].enum_type = _METADATA_LANGUAGE
|
||||
_METADATA.fields_by_name['fee'].message_type = lbrynet_dot_schema_dot_proto_dot_fee__pb2._FEE
|
||||
_METADATA.fields_by_name['fee'].message_type = fee__pb2._FEE
|
||||
_METADATA_VERSION.containing_type = _METADATA
|
||||
_METADATA_LANGUAGE.containing_type = _METADATA
|
||||
DESCRIPTOR.message_types_by_name['Metadata'] = _METADATA
|
||||
@@ -927,8 +927,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Metadata = _reflection.GeneratedProtocolMessageType('Metadata', (_message.Message,), dict(
|
||||
DESCRIPTOR = _METADATA,
|
||||
__module__ = 'lbrynet.schema.proto.metadata_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Metadata)
|
||||
__module__ = 'metadata_pb2'
|
||||
# @@protoc_insertion_point(class_scope:legacy_pb.Metadata)
|
||||
))
|
||||
_sym_db.RegisterMessage(Metadata)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/signature.proto
|
||||
# source: signature.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -12,23 +12,23 @@ from google.protobuf import symbol_database as _symbol_database
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
from lbrynet.schema.proto import certificate_pb2 as lbrynet_dot_schema_dot_proto_dot_certificate__pb2
|
||||
from . import certificate_pb2 as certificate__pb2
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/signature.proto',
|
||||
package='',
|
||||
name='signature.proto',
|
||||
package='legacy_pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n$lbrynet/schema/proto/signature.proto\x1a&lbrynet/schema/proto/certificate.proto\"\xa7\x01\n\tSignature\x12#\n\x07version\x18\x01 \x02(\x0e\x32\x12.Signature.Version\x12\x1f\n\rsignatureType\x18\x02 \x02(\x0e\x32\x08.KeyType\x12\x11\n\tsignature\x18\x03 \x02(\x0c\x12\x15\n\rcertificateId\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
|
||||
serialized_pb=_b('\n\x0fsignature.proto\x12\tlegacy_pb\x1a\x11\x63\x65rtificate.proto\"\xbb\x01\n\tSignature\x12-\n\x07version\x18\x01 \x02(\x0e\x32\x1c.legacy_pb.Signature.Version\x12)\n\rsignatureType\x18\x02 \x02(\x0e\x32\x12.legacy_pb.KeyType\x12\x11\n\tsignature\x18\x03 \x02(\x0c\x12\x15\n\rcertificateId\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
|
||||
,
|
||||
dependencies=[lbrynet_dot_schema_dot_proto_dot_certificate__pb2.DESCRIPTOR,])
|
||||
dependencies=[certificate__pb2.DESCRIPTOR,])
|
||||
|
||||
|
||||
|
||||
_SIGNATURE_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Signature.Version',
|
||||
full_name='legacy_pb.Signature.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -43,42 +43,42 @@ _SIGNATURE_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=206,
|
||||
serialized_end=248,
|
||||
serialized_start=195,
|
||||
serialized_end=237,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_SIGNATURE_VERSION)
|
||||
|
||||
|
||||
_SIGNATURE = _descriptor.Descriptor(
|
||||
name='Signature',
|
||||
full_name='Signature',
|
||||
full_name='legacy_pb.Signature',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Signature.version', index=0,
|
||||
name='version', full_name='legacy_pb.Signature.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='signatureType', full_name='Signature.signatureType', index=1,
|
||||
name='signatureType', full_name='legacy_pb.Signature.signatureType', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='signature', full_name='Signature.signature', index=2,
|
||||
name='signature', full_name='legacy_pb.Signature.signature', index=2,
|
||||
number=3, type=12, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='certificateId', full_name='Signature.certificateId', index=3,
|
||||
name='certificateId', full_name='legacy_pb.Signature.certificateId', index=3,
|
||||
number=4, type=12, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -97,20 +97,20 @@ _SIGNATURE = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=81,
|
||||
serialized_end=248,
|
||||
serialized_start=50,
|
||||
serialized_end=237,
|
||||
)
|
||||
|
||||
_SIGNATURE.fields_by_name['version'].enum_type = _SIGNATURE_VERSION
|
||||
_SIGNATURE.fields_by_name['signatureType'].enum_type = lbrynet_dot_schema_dot_proto_dot_certificate__pb2._KEYTYPE
|
||||
_SIGNATURE.fields_by_name['signatureType'].enum_type = certificate__pb2._KEYTYPE
|
||||
_SIGNATURE_VERSION.containing_type = _SIGNATURE
|
||||
DESCRIPTOR.message_types_by_name['Signature'] = _SIGNATURE
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Signature = _reflection.GeneratedProtocolMessageType('Signature', (_message.Message,), dict(
|
||||
DESCRIPTOR = _SIGNATURE,
|
||||
__module__ = 'lbrynet.schema.proto.signature_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Signature)
|
||||
__module__ = 'signature_pb2'
|
||||
# @@protoc_insertion_point(class_scope:legacy_pb.Signature)
|
||||
))
|
||||
_sym_db.RegisterMessage(Signature)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/source.proto
|
||||
# source: source.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -15,18 +15,18 @@ _sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/source.proto',
|
||||
package='',
|
||||
name='source.proto',
|
||||
package='legacy_pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n!lbrynet/schema/proto/source.proto\"\xde\x01\n\x06Source\x12 \n\x07version\x18\x01 \x02(\x0e\x32\x0f.Source.Version\x12\'\n\nsourceType\x18\x02 \x02(\x0e\x32\x13.Source.SourceTypes\x12\x0e\n\x06source\x18\x03 \x02(\x0c\x12\x13\n\x0b\x63ontentType\x18\x04 \x02(\t\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"8\n\x0bSourceTypes\x12\x17\n\x13UNKNOWN_SOURCE_TYPE\x10\x00\x12\x10\n\x0clbry_sd_hash\x10\x01')
|
||||
serialized_pb=_b('\n\x0csource.proto\x12\tlegacy_pb\"\xf2\x01\n\x06Source\x12*\n\x07version\x18\x01 \x02(\x0e\x32\x19.legacy_pb.Source.Version\x12\x31\n\nsourceType\x18\x02 \x02(\x0e\x32\x1d.legacy_pb.Source.SourceTypes\x12\x0e\n\x06source\x18\x03 \x02(\x0c\x12\x13\n\x0b\x63ontentType\x18\x04 \x02(\t\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"8\n\x0bSourceTypes\x12\x17\n\x13UNKNOWN_SOURCE_TYPE\x10\x00\x12\x10\n\x0clbry_sd_hash\x10\x01')
|
||||
)
|
||||
|
||||
|
||||
|
||||
_SOURCE_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Source.Version',
|
||||
full_name='legacy_pb.Source.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -41,14 +41,14 @@ _SOURCE_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=160,
|
||||
serialized_end=202,
|
||||
serialized_start=170,
|
||||
serialized_end=212,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_SOURCE_VERSION)
|
||||
|
||||
_SOURCE_SOURCETYPES = _descriptor.EnumDescriptor(
|
||||
name='SourceTypes',
|
||||
full_name='Source.SourceTypes',
|
||||
full_name='legacy_pb.Source.SourceTypes',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -63,42 +63,42 @@ _SOURCE_SOURCETYPES = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=204,
|
||||
serialized_end=260,
|
||||
serialized_start=214,
|
||||
serialized_end=270,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_SOURCE_SOURCETYPES)
|
||||
|
||||
|
||||
_SOURCE = _descriptor.Descriptor(
|
||||
name='Source',
|
||||
full_name='Source',
|
||||
full_name='legacy_pb.Source',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Source.version', index=0,
|
||||
name='version', full_name='legacy_pb.Source.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='sourceType', full_name='Source.sourceType', index=1,
|
||||
name='sourceType', full_name='legacy_pb.Source.sourceType', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='source', full_name='Source.source', index=2,
|
||||
name='source', full_name='legacy_pb.Source.source', index=2,
|
||||
number=3, type=12, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='contentType', full_name='Source.contentType', index=3,
|
||||
name='contentType', full_name='legacy_pb.Source.contentType', index=3,
|
||||
number=4, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -118,8 +118,8 @@ _SOURCE = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=38,
|
||||
serialized_end=260,
|
||||
serialized_start=28,
|
||||
serialized_end=270,
|
||||
)
|
||||
|
||||
_SOURCE.fields_by_name['version'].enum_type = _SOURCE_VERSION
|
||||
@@ -131,8 +131,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Source = _reflection.GeneratedProtocolMessageType('Source', (_message.Message,), dict(
|
||||
DESCRIPTOR = _SOURCE,
|
||||
__module__ = 'lbrynet.schema.proto.source_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Source)
|
||||
__module__ = 'source_pb2'
|
||||
# @@protoc_insertion_point(class_scope:legacy_pb.Source)
|
||||
))
|
||||
_sym_db.RegisterMessage(Source)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/stream.proto
|
||||
# source: stream.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -12,24 +12,24 @@ from google.protobuf import symbol_database as _symbol_database
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
from lbrynet.schema.proto import metadata_pb2 as lbrynet_dot_schema_dot_proto_dot_metadata__pb2
|
||||
from lbrynet.schema.proto import source_pb2 as lbrynet_dot_schema_dot_proto_dot_source__pb2
|
||||
from . import metadata_pb2 as metadata__pb2
|
||||
from . import source_pb2 as source__pb2
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/stream.proto',
|
||||
package='',
|
||||
name='stream.proto',
|
||||
package='legacy_pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n!lbrynet/schema/proto/stream.proto\x1a#lbrynet/schema/proto/metadata.proto\x1a!lbrynet/schema/proto/source.proto\"\x8c\x01\n\x06Stream\x12 \n\x07version\x18\x01 \x02(\x0e\x32\x0f.Stream.Version\x12\x1b\n\x08metadata\x18\x02 \x02(\x0b\x32\t.Metadata\x12\x17\n\x06source\x18\x03 \x02(\x0b\x32\x07.Source\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
|
||||
serialized_pb=_b('\n\x0cstream.proto\x12\tlegacy_pb\x1a\x0emetadata.proto\x1a\x0csource.proto\"\xaa\x01\n\x06Stream\x12*\n\x07version\x18\x01 \x02(\x0e\x32\x19.legacy_pb.Stream.Version\x12%\n\x08metadata\x18\x02 \x02(\x0b\x32\x13.legacy_pb.Metadata\x12!\n\x06source\x18\x03 \x02(\x0b\x32\x11.legacy_pb.Source\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
|
||||
,
|
||||
dependencies=[lbrynet_dot_schema_dot_proto_dot_metadata__pb2.DESCRIPTOR,lbrynet_dot_schema_dot_proto_dot_source__pb2.DESCRIPTOR,])
|
||||
dependencies=[metadata__pb2.DESCRIPTOR,source__pb2.DESCRIPTOR,])
|
||||
|
||||
|
||||
|
||||
_STREAM_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Stream.Version',
|
||||
full_name='legacy_pb.Stream.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -44,35 +44,35 @@ _STREAM_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=208,
|
||||
serialized_end=250,
|
||||
serialized_start=186,
|
||||
serialized_end=228,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_STREAM_VERSION)
|
||||
|
||||
|
||||
_STREAM = _descriptor.Descriptor(
|
||||
name='Stream',
|
||||
full_name='Stream',
|
||||
full_name='legacy_pb.Stream',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Stream.version', index=0,
|
||||
name='version', full_name='legacy_pb.Stream.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='metadata', full_name='Stream.metadata', index=1,
|
||||
name='metadata', full_name='legacy_pb.Stream.metadata', index=1,
|
||||
number=2, type=11, cpp_type=10, label=2,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='source', full_name='Stream.source', index=2,
|
||||
name='source', full_name='legacy_pb.Stream.source', index=2,
|
||||
number=3, type=11, cpp_type=10, label=2,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -91,21 +91,21 @@ _STREAM = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=110,
|
||||
serialized_end=250,
|
||||
serialized_start=58,
|
||||
serialized_end=228,
|
||||
)
|
||||
|
||||
_STREAM.fields_by_name['version'].enum_type = _STREAM_VERSION
|
||||
_STREAM.fields_by_name['metadata'].message_type = lbrynet_dot_schema_dot_proto_dot_metadata__pb2._METADATA
|
||||
_STREAM.fields_by_name['source'].message_type = lbrynet_dot_schema_dot_proto_dot_source__pb2._SOURCE
|
||||
_STREAM.fields_by_name['metadata'].message_type = metadata__pb2._METADATA
|
||||
_STREAM.fields_by_name['source'].message_type = source__pb2._SOURCE
|
||||
_STREAM_VERSION.containing_type = _STREAM
|
||||
DESCRIPTOR.message_types_by_name['Stream'] = _STREAM
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Stream = _reflection.GeneratedProtocolMessageType('Stream', (_message.Message,), dict(
|
||||
DESCRIPTOR = _STREAM,
|
||||
__module__ = 'lbrynet.schema.proto.stream_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Stream)
|
||||
__module__ = 'stream_pb2'
|
||||
# @@protoc_insertion_point(class_scope:legacy_pb.Stream)
|
||||
))
|
||||
_sym_db.RegisterMessage(Stream)
|
||||
|
||||
File renamed without changes.
File diff suppressed because it is too large.
Load diff
Loaded 100 of 199 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user