Compare commits
470
Commits
v0.32.1
...
misc_fixes
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
27afc759ff | ||
|
|
7abb28cb3e | ||
|
|
7d2f592bb5 | ||
|
|
1a182bc9bd | ||
|
|
324cbdcdb2 | ||
|
|
6c0c0ee322 | ||
|
|
1e54271d8a | ||
|
|
54b9ed6cb2 | ||
|
|
558a02d187 | ||
|
|
3567b331c0 | ||
|
|
9e4cf3c995 | ||
|
|
db1d547b19 | ||
|
|
c31efd780b | ||
|
|
7cdb7987e3 | ||
|
|
ec07037e88 | ||
|
|
9bb7b0571e | ||
|
|
d98162a118 | ||
|
|
c5ae59bff7 | ||
|
|
efbf2f49a9 | ||
|
|
77c69f661d | ||
|
|
2e09d2adfa | ||
|
|
8620a0e424 | ||
|
|
e2c123456b | ||
|
|
c9be79f062 | ||
|
|
a2a1ef34ab | ||
|
|
761a2b235c | ||
|
|
3ca2fee592 | ||
|
|
a480b2d25f | ||
|
|
d7032b12d7 | ||
|
|
84381ff76c | ||
|
|
007dd43861 | ||
|
|
1bc4e4b702 | ||
|
|
b91f27219d | ||
|
|
dc4560cc9a | ||
|
|
84b471d486 | ||
|
|
c6a3c05a0c | ||
|
|
dd665a758d | ||
|
|
32dc1297ec | ||
|
|
bef244ba77 | ||
|
|
3c231d6215 | ||
|
|
a212cf6ba4 | ||
|
|
bc060f1378 | ||
|
|
22c701fd50 | ||
|
|
4f0e8fce6e | ||
|
|
f642cfe9dd | ||
|
|
24e073680b | ||
|
|
b2f63a1545 | ||
|
|
f506b3e6d4 | ||
|
|
f7412514b3 | ||
|
|
bd6a609b30 | ||
|
|
4e32b69d1d | ||
|
|
1116c7f29e | ||
|
|
d80c671cf2 | ||
|
|
14b12cbea2 | ||
|
|
1f7feafb67 | ||
|
|
cbe689ea7f | ||
|
|
9099ee2e8e | ||
|
|
b134e0c9c9 | ||
|
|
f8c0e80cfc | ||
|
|
1f266ebdad | ||
|
|
b09e1f8825 | ||
|
|
c496278a5b | ||
|
|
cacc65d45e | ||
|
|
2247b354b3 | ||
|
|
43111f0497 | ||
|
|
6c142b4477 | ||
|
|
e8683a24bd | ||
|
|
a4d17d1705 | ||
|
|
a030e13548 | ||
|
|
32b8601c38 | ||
|
|
f5bd28bfdb | ||
|
|
6cbbe23c7c | ||
|
|
d7c2e4e9b3 | ||
|
|
d61ddbb950 | ||
|
|
cf7bb6a391 | ||
|
|
68d7328918 | ||
|
|
ff2985cc80 | ||
|
|
ac69fcbc5f | ||
|
|
609cf42868 | ||
|
|
e43b29fcd1 | ||
|
|
6594b7c50c | ||
|
|
971f2a1cef | ||
|
|
fc88261cc6 | ||
|
|
895b3992c9 | ||
|
|
41951d8ec2 | ||
|
|
ae2121c5c4 | ||
|
|
771fe935f4 | ||
|
|
60f7766cf7 | ||
|
|
63bd983012 | ||
|
|
7ffce1040c | ||
|
|
ac6835fa60 | ||
|
|
3cf5c536c0 | ||
|
|
85cc1a4763 | ||
|
|
180cfcfb72 | ||
|
|
864c1dd901 | ||
|
|
aeaa7ba986 | ||
|
|
b8325c1f89 | ||
|
|
a36936b5ff | ||
|
|
7cf245a0e9 | ||
|
|
9f83cce574 | ||
|
|
30b113d64a | ||
|
|
ccd2023d37 | ||
|
|
3127cd11cf | ||
|
|
6e483cba8d | ||
|
|
46c03c0cb2 | ||
|
|
c2ca0a9467 | ||
|
|
b350e74aca | ||
|
|
de9473f75a | ||
|
|
bfc107fdb3 | ||
|
|
6a2efa54ab | ||
|
|
72f0e1ca14 | ||
|
|
9a1262d557 | ||
|
|
706a8e52dc | ||
|
|
f3c8776878 | ||
|
|
539ea04039 | ||
|
|
8f96b07559 | ||
|
|
0a1e97fe23 | ||
|
|
da637b9c0e | ||
|
|
3618de1227 | ||
|
|
3cb8a63bf9 | ||
|
|
bad99de5b3 | ||
|
|
3d1efbc9bb | ||
|
|
7bdef3146b | ||
|
|
2145bfcabf | ||
|
|
b88832d0a9 | ||
|
|
db08882f20 | ||
|
|
6f52d36b22 | ||
|
|
056a629e87 | ||
|
|
1ffb99f9f9 | ||
|
|
a8e612773d | ||
|
|
67088b2ca2 | ||
|
|
dab49e812f | ||
|
|
0f3f38d8a3 | ||
|
|
7ba433c2e5 | ||
|
|
bb7649e8ad | ||
|
|
832537a5cf | ||
|
|
3234d70270 | ||
|
|
432fe444f8 | ||
|
|
861c25716e | ||
|
|
9ba6653475 | ||
|
|
8107e97304 | ||
|
|
e2663f6d57 | ||
|
|
35479dcbd2 | ||
|
|
3a997277aa | ||
|
|
a7d94f4514 | ||
|
|
a0e3338177 | ||
|
|
d5a353066c | ||
|
|
bc29fc4465 | ||
|
|
987f175fcc | ||
|
|
5b1398b588 | ||
|
|
873b58efc9 | ||
|
|
01a81fceed | ||
|
|
a14b0aa573 | ||
|
|
a1abb7c8fb | ||
|
|
46b166952b | ||
|
|
1375153fda | ||
|
|
1a14157d37 | ||
|
|
40b9fa38b0 | ||
|
|
4d0e9e0428 | ||
|
|
3a53ef5690 | ||
|
|
33a68b5cef | ||
|
|
63bf084a6a | ||
|
|
a882f2e8c5 | ||
|
|
119e0ec408 | ||
|
|
7d782d516a | ||
|
|
d3eac48035 | ||
|
|
60c9ae64b4 | ||
|
|
2d314dce60 | ||
|
|
287b89db66 | ||
|
|
7a160f7335 | ||
|
|
04ddee7d89 | ||
|
|
3a916a8e8e | ||
|
|
f125468ebf | ||
|
|
f0e17cff9a | ||
|
|
93267efe0b | ||
|
|
c663e5a3cf | ||
|
|
676f0015aa | ||
|
|
d44a79ada2 | ||
|
|
d7ed8ffaa6 | ||
|
|
b230d8fbcb | ||
|
|
7d2428fd17 | ||
|
|
4157045a41 | ||
|
|
58987857cf | ||
|
|
fa8858325c | ||
|
|
3357a419ab | ||
|
|
a782adea75 | ||
|
|
e8991e6d52 | ||
|
|
13bd81c2b6 | ||
|
|
21241b98a4 | ||
|
|
bf467e06c5 | ||
|
|
886ed6415a | ||
|
|
97db241109 | ||
|
|
216379918b | ||
|
|
fcab4eb9bd | ||
|
|
19f300cc45 | ||
|
|
26fec098e4 | ||
|
|
7824de655e | ||
|
|
6ef1b1dd59 | ||
|
|
9fae8f5153 | ||
|
|
f660691e78 | ||
|
|
bf60683ba6 | ||
|
|
68215699f7 | ||
|
|
1644ffbf22 | ||
|
|
71f5061848 | ||
|
|
61cadb5cbe | ||
|
|
2034104cfc | ||
|
|
633f725634 | ||
|
|
8bf2e41c95 | ||
|
|
fa9a4a75c6 | ||
|
|
90bef98bc3 | ||
|
|
35c43647aa | ||
|
|
449a583286 | ||
|
|
b1f813782a | ||
|
|
60080cbd19 | ||
|
|
665dba62c7 | ||
|
|
76dea9e89b | ||
|
|
ebc9ed01c5 | ||
|
|
55116763e9 | ||
|
|
e1a6b4e3d6 | ||
|
|
911ca8c37e | ||
|
|
7f5deaf6c8 | ||
|
|
0200756218 | ||
|
|
a63442f843 | ||
|
|
74b3da79aa | ||
|
|
b6a4589da5 | ||
|
|
fbbb0a9931 | ||
|
|
a94d65d522 | ||
|
|
fa2ead002e | ||
|
|
8dfee4867a | ||
|
|
6614ab8e79 | ||
|
|
d32c1dfbb3 | ||
|
|
9f42b958fa | ||
|
|
574a912f7d | ||
|
|
2cfcc3cd45 | ||
|
|
9af625e627 | ||
|
|
1871c94003 | ||
|
|
93220c3874 | ||
|
|
362ed3f1c8 | ||
|
|
62a182b8bb | ||
|
|
6f76d9e24a | ||
|
|
55e83fe05c | ||
|
|
2f0c7d6a3f | ||
|
|
59e1df9efa | ||
|
|
fd56dc68cd | ||
|
|
a0a810edc3 | ||
|
|
c1d3b5984d | ||
|
|
1fb719dd61 | ||
|
|
b87316fb02 | ||
|
|
65f3d78084 | ||
|
|
47bc1526f7 | ||
|
|
3acabec4bb | ||
|
|
36a62d3756 | ||
|
|
2e22473ce3 | ||
|
|
0b0fd64642 | ||
|
|
ca17d8597d | ||
|
|
d16e2716c8 | ||
|
|
89c80cedfd | ||
|
|
021e30ffde | ||
|
|
eb2cd8fec1 | ||
|
|
6356818557 | ||
|
|
8bf724a78b | ||
|
|
65d0a924c1 | ||
|
|
ef7bd00f22 | ||
|
|
3c1fdb5114 | ||
|
|
83411acfbd | ||
|
|
4230812f82 | ||
|
|
7b57ba96dd | ||
|
|
668840921f | ||
|
|
a3bd335f7f | ||
|
|
4e6bca1389 | ||
|
|
c4b15920ff | ||
|
|
2a55e583e7 | ||
|
|
2c4fde06c5 | ||
|
|
eca83d7f47 | ||
|
|
87e818ac39 | ||
|
|
adab7bd7b1 | ||
|
|
750c0e4f4f | ||
|
|
5f8482bdb7 | ||
|
|
cd266477fb | ||
|
|
7860b956ff | ||
|
|
148481a5ca | ||
|
|
93faa276a5 | ||
|
|
53e67a6e81 | ||
|
|
3ad6808d87 | ||
|
|
071ab32ab7 | ||
|
|
ab87588250 | ||
|
|
22db29ee1d | ||
|
|
66bf6c61f0 | ||
|
|
9abf40d21f | ||
|
|
ea4444edbe | ||
|
|
51bb8fd283 | ||
|
|
183e3ac1c7 | ||
|
|
8087b457c6 | ||
|
|
fedb7c23d0 | ||
|
|
e2964f9728 | ||
|
|
e0cea60c87 | ||
|
|
c708c1db07 | ||
|
|
1cb696fb5c | ||
|
|
7cd0213eb9 | ||
|
|
2007c48af3 | ||
|
|
50721786d2 | ||
|
|
6d7eb259cb | ||
|
|
cc5852a9b9 | ||
|
|
0592cee567 | ||
|
|
814eaca524 | ||
|
|
e06916f5ae | ||
|
|
d61e1e9d8c | ||
|
|
25436893bc | ||
|
|
5b9d41fd5c | ||
|
|
b1720d1659 | ||
|
|
4b3a6ab875 | ||
|
|
67dcede2d8 | ||
|
|
0872a6fea3 | ||
|
|
a3aed3219f | ||
|
|
af73e1a2f5 | ||
|
|
55bbf5aa74 | ||
|
|
c1fbb02bfe | ||
|
|
d47575e8e0 | ||
|
|
8fc4f4aa73 | ||
|
|
fe9aa666e9 | ||
|
|
a9e0eeba7e | ||
|
|
0830917afb | ||
|
|
1ed086e90f | ||
|
|
56db631626 | ||
|
|
aa029955ee | ||
|
|
f2236eb7a4 | ||
|
|
df649341cd | ||
|
|
a3c3143868 | ||
|
|
d0201fe7c7 | ||
|
|
664a2a324d | ||
|
|
ea09aca175 | ||
|
|
ca92fddecf | ||
|
|
8259d7c680 | ||
|
|
6653da2973 | ||
|
|
e74998209d | ||
|
|
2bea0f697a | ||
|
|
1d68bef6f2 | ||
|
|
22bacc907b | ||
|
|
1e8ad900c7 | ||
|
|
cd2b535afb | ||
|
|
9dfe7bbcf6 | ||
|
|
120618ffd5 | ||
|
|
e8abcbb06e | ||
|
|
1ec8f0b0b4 | ||
|
|
323694fb12 | ||
|
|
d5e37acd8f | ||
|
|
129a886f2b | ||
|
|
cd15230a92 | ||
|
|
c1b7fdc68e | ||
|
|
9fd67aaad6 | ||
|
|
ce9dc1836e | ||
|
|
2e557d3a36 | ||
|
|
5df05cb37e | ||
|
|
326b62dc57 | ||
|
|
25cc7ba284 | ||
|
|
550f064302 | ||
|
|
11086b93f1 | ||
|
|
df5662136d | ||
|
|
291d65070d | ||
|
|
9b64ee33dc | ||
|
|
5404d26f2f | ||
|
|
26ade1f228 | ||
|
|
b49f1f1b54 | ||
|
|
3bafa2ad22 | ||
|
|
378ec44f3c | ||
|
|
9d0d17b422 | ||
|
|
f7c8145283 | ||
|
|
0de17730ca | ||
|
|
58743ba19b | ||
|
|
f2fefbe287 | ||
|
|
60a24f0e6e | ||
|
|
664f91bfab | ||
|
|
44f92a271f | ||
|
|
6565ca8558 | ||
|
|
3b0ba3e534 | ||
|
|
8106eb1fa5 | ||
|
|
c9b2829227 | ||
|
|
4f2f8b16f3 | ||
|
|
2ab2b846b1 | ||
|
|
2df7284c29 | ||
|
|
ab5d71ff61 | ||
|
|
99b289f135 | ||
|
|
6db02e4362 | ||
|
|
98d8b7d463 | ||
|
|
96f1883fc4 | ||
|
|
a914de155a | ||
|
|
2dee0ff0bf | ||
|
|
993cb43e5b | ||
|
|
5d212a0f82 | ||
|
|
0cc602ca53 | ||
|
|
f390ff144d | ||
|
|
b266cde511 | ||
|
|
d0a1541d0a | ||
|
|
4cbde1b08f | ||
|
|
c6740d8d70 | ||
|
|
46358142ed | ||
|
|
f0f651fc77 | ||
|
|
5d428138ed | ||
|
|
8d9da61c7d | ||
|
|
c326a9382c | ||
|
|
5bdd87c904 | ||
|
|
ad134fe8bb | ||
|
|
2f3c616acd | ||
|
|
3caa2a5039 | ||
|
|
e38a0f7a95 | ||
|
|
9de3f3ff09 | ||
|
|
df8934d472 | ||
|
|
be4a96c1b3 | ||
|
|
d97d978961 | ||
|
|
f52e44b56a | ||
|
|
10d73fe480 | ||
|
|
4fd2d994f2 | ||
|
|
d95903d219 | ||
|
|
591e66bfbb | ||
|
|
e047748961 | ||
|
|
9f071bbe36 | ||
|
|
2d8ebe25ed | ||
|
|
7a96e742f2 | ||
|
|
13c7026b20 | ||
|
|
876a3c68dd | ||
|
|
573617d6b6 | ||
|
|
e8c9abb944 | ||
|
|
8cee627dfa | ||
|
|
ab4a5cee01 | ||
|
|
5bb0a07fae | ||
|
|
5ca594b9fd | ||
|
|
3910964df1 | ||
|
|
faa1935203 | ||
|
|
ba518e2a0e | ||
|
|
2a82ff7cc5 | ||
|
|
673b958020 | ||
|
|
3f4ed0a896 | ||
|
|
f49578cdbc | ||
|
|
b834f29634 | ||
|
|
61132312dd | ||
|
|
fcdd9295b4 | ||
|
|
edb5083a5e | ||
|
|
4e7d88311f | ||
|
|
aa40337ded | ||
|
|
690e2e7e0c | ||
|
|
0c67469321 | ||
|
|
fc64d8d1a8 | ||
|
|
7fe9513857 | ||
|
|
e76107c318 | ||
|
|
3868dea819 | ||
|
|
c52541197f | ||
|
|
03cd6b7200 | ||
|
|
6915a7a553 | ||
|
|
7ad29b4bfb | ||
|
|
4294bf742d | ||
|
|
57ad9f1c52 | ||
|
|
e260de59f1 | ||
|
|
154702e6dd | ||
|
|
d06d998c81 | ||
|
|
8aff860bfe | ||
|
|
d7ba90bbea | ||
|
|
59325ad8c1 | ||
|
|
1f72e27272 | ||
|
|
aa41b8b558 | ||
|
|
2496e7ed44 | ||
|
|
ee12985135 | ||
|
|
4c7f357441 | ||
|
|
8944a36291 | ||
|
|
cb5d428d12 | ||
|
|
f175e8826c | ||
|
|
38ed212c01 | ||
|
|
d035c13883 | ||
|
|
82ce2ccee6 | ||
|
|
2cf9b2dd9e | ||
|
|
fcbdaa336f |
No files matched your search
@@ -1,9 +1,13 @@
|
||||
.DS_Store
|
||||
/build
|
||||
/dist
|
||||
/.tox
|
||||
/.idea
|
||||
/.coverage
|
||||
/lbry-venv
|
||||
|
||||
lbrynet.egg-info
|
||||
__pycache__
|
||||
_trial_temp/
|
||||
|
||||
/tests/integration/files
|
||||
+14
-15
@@ -9,9 +9,7 @@ jobs:
|
||||
- stage: code quality
|
||||
name: "pylint lbrynet"
|
||||
install:
|
||||
- pip install astroid==2.0.4 aiohttp==3.4.4
|
||||
# newer astroid and aiohttp fails in pylint so we pre-install older version
|
||||
- pip install pylint
|
||||
- pip install astroid pylint
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install -e .
|
||||
script: pylint lbrynet
|
||||
@@ -37,6 +35,17 @@ jobs:
|
||||
- coverage combine tests/
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- name: "Run Examples"
|
||||
install:
|
||||
- pip install coverage
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install -e .
|
||||
script:
|
||||
- HOME=/tmp coverage run -p --source=lbrynet scripts/generate_json_api.py
|
||||
after_success:
|
||||
- coverage combine
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- stage: build
|
||||
name: "Windows"
|
||||
language: generic
|
||||
@@ -71,7 +80,7 @@ jobs:
|
||||
install:
|
||||
- pip3 install pyinstaller
|
||||
- pip3 install git+https://github.com/lbryio/torba.git
|
||||
- python scripts/set_build.py
|
||||
- python3 scripts/set_build.py
|
||||
- pip3 install -e .
|
||||
script:
|
||||
- pyinstaller -F -n lbrynet lbrynet/extras/cli.py
|
||||
@@ -106,16 +115,6 @@ jobs:
|
||||
before_install:
|
||||
- brew upgrade python || true
|
||||
- brew upgrade python || true
|
||||
install:
|
||||
- python3 --version
|
||||
- pip3 --version
|
||||
- pip3 install pyinstaller
|
||||
- git clone https://github.com/lbryio/torba.git --depth 1
|
||||
- sed -i -e "s/'plyvel',//" torba/setup.py
|
||||
- cd torba && pip3 install -e . && cd ..
|
||||
- python3 scripts/set_build.py
|
||||
- pip3 install -e .
|
||||
|
||||
- if: tag IS present
|
||||
stage: build
|
||||
name: "Wallet Server Docker Image - Tagged Release"
|
||||
@@ -124,7 +123,7 @@ jobs:
|
||||
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
|
||||
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f scripts/Dockerfile.wallet_server .
|
||||
- docker push lbry/wallet-server:$TRAVIS_TAG
|
||||
- if: tag IS blank AND branch = master
|
||||
- if: tag IS blank AND branch = master AND NOT type IN (pull_request)
|
||||
stage: build
|
||||
name: "Wallet Server Docker Image - Master"
|
||||
script:
|
||||
|
||||
+1
-1
@@ -1,3 +1,3 @@
|
||||
## Contributing to LBRY
|
||||
|
||||
https://lbry.io/faq/contributing
|
||||
https://lbry.tech/contribute
|
||||
+11
-12
@@ -13,15 +13,6 @@ Running `lbrynet` from source requires Python 3.6 or higher (3.7 is preferred).
|
||||
|
||||
After installing python 3, you'll need to install some additional libraries depending on your operating system.
|
||||
|
||||
### Virtualenv
|
||||
|
||||
Once python 3 is installed run `python3 -m pip install virtualenv` to install virtualenv.
|
||||
|
||||
### Windows
|
||||
|
||||
Windows users will need to install `Visual C++ Build Tools`, which can be installed by [Microsoft Build Tools](Microsoft Build Tools 2015)
|
||||
|
||||
|
||||
### macOS
|
||||
|
||||
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
|
||||
@@ -29,15 +20,17 @@ macOS users will need to install [xcode command line tools](https://developer.xa
|
||||
Remaining dependencies can then be installed by running:
|
||||
|
||||
```
|
||||
brew install python3 protobuf
|
||||
brew install python protobuf
|
||||
```
|
||||
|
||||
Assistance installing Python3: https://docs.python-guide.org/starting/install3/osx/
|
||||
|
||||
### Linux
|
||||
|
||||
On Ubuntu (we recommend 18.04), install the following:
|
||||
|
||||
```
|
||||
sudo apt-get install build-essential python3.7 python3.7-dev git python-virtualenv libssl-dev python-protobuf
|
||||
sudo apt-get install build-essential python3.7 python3.7-dev git python3-venv libssl-dev python-protobuf
|
||||
```
|
||||
|
||||
On Raspbian, you will also need to install `python-pyparsing`.
|
||||
@@ -52,8 +45,14 @@ To install:
|
||||
git clone https://github.com/lbryio/lbry.git
|
||||
cd lbry
|
||||
|
||||
virtualenv lbry-venv --python=python3.7
|
||||
Creating venv:
|
||||
python -m venv lbry-venv
|
||||
|
||||
Activating lbry-venv on Linux/Mac:
|
||||
source lbry-venv/bin/activate
|
||||
|
||||
Activating lbry-venv on Windows:
|
||||
lbry-venv\Scripts\activate
|
||||
|
||||
python --version # Python 2 is not supported. Make sure you're on Python 3.7
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ LBRY SDK for Python is currently the most full featured implementation of the LB
|
||||
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
|
||||
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
|
||||
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbrynet.schema](https://github.com/lbryio/lbry/tree/master/lbrynet/schema)).
|
||||
* Wallet implementation for the LBRY blockchain ([lbrynet.extras.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/wallet)).
|
||||
* Wallet implementation for the LBRY blockchain ([lbrynet.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/wallet)).
|
||||
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbrynet.extras.daemon](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/daemon)).
|
||||
|
||||
## Installation
|
||||
@@ -33,7 +33,7 @@ Installing from source is also relatively painless. Full instructions are in [IN
|
||||
|
||||
## Contributing
|
||||
|
||||
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.io/faq/contributing) link.
|
||||
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.tech/contribute) link.
|
||||
|
||||
## License
|
||||
|
||||
@@ -41,11 +41,11 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
|
||||
|
||||
## Security
|
||||
|
||||
We take security seriously. Please contact security@lbry.io regarding any security issues. [Our GPG key is here](https://lbry.io/faq/gpg-key) if you need it.
|
||||
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.io/faq/gpg-key) if you need it.
|
||||
|
||||
## Contact
|
||||
|
||||
The primary contact for this project is [@eukreign](mailto:lex@lbry.io).
|
||||
The primary contact for this project is [@eukreign](mailto:lex@lbry.com).
|
||||
|
||||
## Additional information and links
|
||||
|
||||
|
||||
+2612
-1526
File diff suppressed because it is too large.
Load diff
+2
-4
@@ -1409,7 +1409,7 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'claim_sequence': (int) claim sequence number (or -1 if unknown),
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -1954,7 +1954,6 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -1979,7 +1978,6 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -2586,4 +2584,4 @@ Returns:
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
</html>
|
||||
+2
-4
@@ -1339,7 +1339,7 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'claim_sequence': (int) claim sequence number (or -1 if unknown),
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -1786,7 +1786,6 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -1811,7 +1810,6 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -2312,4 +2310,4 @@ Returns:
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
</html>
|
||||
@@ -12,12 +12,13 @@ blockchain_name: lbrycrd_main
|
||||
data_dir: /home/lbry/.lbrynet
|
||||
download_directory: /home/lbry/downloads
|
||||
|
||||
delete_blobs_on_remove: True
|
||||
save_blobs: true
|
||||
save_files: false
|
||||
dht_node_port: 4444
|
||||
peer_port: 3333
|
||||
use_upnp: True
|
||||
use_upnp: true
|
||||
|
||||
#components_to_skip:
|
||||
# - peer_protocol_server
|
||||
# - hash_announcer
|
||||
# - blob_server
|
||||
# - dht
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
__name__ = "lbrynet"
|
||||
__version__ = "0.32.1"
|
||||
__version__ = "0.37.0rc6"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
+270
-131
@@ -4,6 +4,8 @@ import asyncio
|
||||
import binascii
|
||||
import logging
|
||||
import typing
|
||||
import contextlib
|
||||
from io import BytesIO
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, modes
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from cryptography.hazmat.primitives.padding import PKCS7
|
||||
@@ -21,10 +23,6 @@ log = logging.getLogger(__name__)
|
||||
_hexmatch = re.compile("^[a-f,0-9]+$")
|
||||
|
||||
|
||||
def is_valid_hashcharacter(char: str) -> bool:
|
||||
return len(char) == 1 and _hexmatch.match(char)
|
||||
|
||||
|
||||
def is_valid_blobhash(blobhash: str) -> bool:
|
||||
"""Checks whether the blobhash is the correct length and contains only
|
||||
valid characters (0-9, a-f)
|
||||
@@ -46,142 +44,74 @@ def encrypt_blob_bytes(key: bytes, iv: bytes, unencrypted: bytes) -> typing.Tupl
|
||||
return encrypted, digest.hexdigest()
|
||||
|
||||
|
||||
class BlobFile:
|
||||
"""
|
||||
A chunk of data available on the network which is specified by a hashsum
|
||||
def decrypt_blob_bytes(data: bytes, length: int, key: bytes, iv: bytes) -> bytes:
|
||||
if len(data) != length:
|
||||
raise ValueError("unexpected length")
|
||||
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
|
||||
unpadder = PKCS7(AES.block_size).unpadder()
|
||||
decryptor = cipher.decryptor()
|
||||
return unpadder.update(decryptor.update(data) + decryptor.finalize()) + unpadder.finalize()
|
||||
|
||||
This class is used to create blobs on the local filesystem
|
||||
when we already know the blob hash before hand (i.e., when downloading blobs)
|
||||
Also can be used for reading from blobs on the local filesystem
|
||||
"""
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, blob_hash: str,
|
||||
length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['BlobFile'], typing.Awaitable]] = None):
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise InvalidBlobHashError(blob_hash)
|
||||
class AbstractBlob:
|
||||
"""
|
||||
A chunk of data (up to 2MB) available on the network which is specified by a sha384 hash
|
||||
|
||||
This class is non-io specific
|
||||
"""
|
||||
__slots__ = [
|
||||
'loop',
|
||||
'blob_hash',
|
||||
'length',
|
||||
'blob_completed_callback',
|
||||
'blob_directory',
|
||||
'writers',
|
||||
'verified',
|
||||
'writing',
|
||||
'readers'
|
||||
]
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None):
|
||||
self.loop = loop
|
||||
self.blob_hash = blob_hash
|
||||
self.length = length
|
||||
self.blob_dir = blob_dir
|
||||
self.file_path = os.path.join(blob_dir, self.blob_hash)
|
||||
self.writers: typing.List[HashBlobWriter] = []
|
||||
|
||||
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||
self.finished_writing = asyncio.Event(loop=loop)
|
||||
self.blob_write_lock = asyncio.Lock(loop=loop)
|
||||
if self.file_exists:
|
||||
length = int(os.stat(os.path.join(blob_dir, blob_hash)).st_size)
|
||||
self.length = length
|
||||
self.verified.set()
|
||||
self.finished_writing.set()
|
||||
self.saved_verified_blob = False
|
||||
self.blob_completed_callback = blob_completed_callback
|
||||
self.blob_directory = blob_directory
|
||||
self.writers: typing.Dict[typing.Tuple[typing.Optional[str], typing.Optional[int]], HashBlobWriter] = {}
|
||||
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||
self.writing: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||
self.readers: typing.List[typing.BinaryIO] = []
|
||||
|
||||
@property
|
||||
def file_exists(self):
|
||||
return os.path.isfile(self.file_path)
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise InvalidBlobHashError(blob_hash)
|
||||
|
||||
def writer_finished(self, writer: HashBlobWriter):
|
||||
def callback(finished: asyncio.Future):
|
||||
def __del__(self):
|
||||
if self.writers or self.readers:
|
||||
log.warning("%s not closed before being garbage collected", self.blob_hash)
|
||||
self.close()
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
raise NotImplementedError()
|
||||
|
||||
@contextlib.contextmanager
|
||||
def reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
if not self.is_readable():
|
||||
raise OSError(f"{str(type(self))} not readable, {len(self.readers)} readers {len(self.writers)} writers")
|
||||
with self._reader_context() as reader:
|
||||
try:
|
||||
error = finished.exception()
|
||||
except Exception as err:
|
||||
error = err
|
||||
if writer in self.writers: # remove this download attempt
|
||||
self.writers.remove(writer)
|
||||
if not error: # the blob downloaded, cancel all the other download attempts and set the result
|
||||
while self.writers:
|
||||
other = self.writers.pop()
|
||||
other.finished.cancel()
|
||||
t = self.loop.create_task(self.save_verified_blob(writer, finished.result()))
|
||||
t.add_done_callback(lambda *_: self.finished_writing.set())
|
||||
return
|
||||
if isinstance(error, (InvalidBlobHashError, InvalidDataError)):
|
||||
log.debug("writer error downloading %s: %s", self.blob_hash[:8], str(error))
|
||||
elif not isinstance(error, (DownloadCancelledError, asyncio.CancelledError, asyncio.TimeoutError)):
|
||||
log.exception("something else")
|
||||
raise error
|
||||
return callback
|
||||
self.readers.append(reader)
|
||||
yield reader
|
||||
finally:
|
||||
if reader in self.readers:
|
||||
self.readers.remove(reader)
|
||||
|
||||
async def save_verified_blob(self, writer, verified_bytes: bytes):
|
||||
def _save_verified():
|
||||
# log.debug(f"write blob file {self.blob_hash[:8]} from {writer.peer.address}")
|
||||
if not self.saved_verified_blob and not os.path.isfile(self.file_path):
|
||||
if self.get_length() == len(verified_bytes):
|
||||
with open(self.file_path, 'wb') as write_handle:
|
||||
write_handle.write(verified_bytes)
|
||||
self.saved_verified_blob = True
|
||||
else:
|
||||
raise Exception("length mismatch")
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
raise NotImplementedError()
|
||||
|
||||
async with self.blob_write_lock:
|
||||
if self.verified.is_set():
|
||||
return
|
||||
await self.loop.run_in_executor(None, _save_verified)
|
||||
if self.blob_completed_callback:
|
||||
await self.blob_completed_callback(self)
|
||||
self.verified.set()
|
||||
|
||||
def open_for_writing(self) -> HashBlobWriter:
|
||||
if self.file_exists:
|
||||
raise OSError(f"File already exists '{self.file_path}'")
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
|
||||
self.writers.append(writer)
|
||||
fut.add_done_callback(self.writer_finished(writer))
|
||||
return writer
|
||||
|
||||
async def sendfile(self, writer: asyncio.StreamWriter) -> int:
|
||||
"""
|
||||
Read and send the file to the writer and return the number of bytes sent
|
||||
"""
|
||||
|
||||
with open(self.file_path, 'rb') as handle:
|
||||
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
|
||||
|
||||
def close(self):
|
||||
while self.writers:
|
||||
self.writers.pop().finished.cancel()
|
||||
|
||||
def delete(self):
|
||||
self.close()
|
||||
self.saved_verified_blob = False
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
self.verified.clear()
|
||||
self.finished_writing.clear()
|
||||
|
||||
def decrypt(self, key: bytes, iv: bytes) -> bytes:
|
||||
"""
|
||||
Decrypt a BlobFile to plaintext bytes
|
||||
"""
|
||||
|
||||
with open(self.file_path, "rb") as f:
|
||||
buff = f.read()
|
||||
if len(buff) != self.length:
|
||||
raise ValueError("unexpected length")
|
||||
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
|
||||
unpadder = PKCS7(AES.block_size).unpadder()
|
||||
decryptor = cipher.decryptor()
|
||||
return unpadder.update(decryptor.update(buff) + decryptor.finalize()) + unpadder.finalize()
|
||||
|
||||
@classmethod
|
||||
async def create_from_unencrypted(cls, loop: asyncio.BaseEventLoop, blob_dir: str, key: bytes,
|
||||
iv: bytes, unencrypted: bytes, blob_num: int) -> BlobInfo:
|
||||
"""
|
||||
Create an encrypted BlobFile from plaintext bytes
|
||||
"""
|
||||
|
||||
blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted)
|
||||
length = len(blob_bytes)
|
||||
blob = cls(loop, blob_dir, blob_hash, length)
|
||||
writer = blob.open_for_writing()
|
||||
writer.write(blob_bytes)
|
||||
await blob.verified.wait()
|
||||
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash)
|
||||
|
||||
def set_length(self, length):
|
||||
def set_length(self, length) -> None:
|
||||
if self.length is not None and length == self.length:
|
||||
return
|
||||
if self.length is None and 0 <= length <= MAX_BLOB_SIZE:
|
||||
@@ -189,8 +119,217 @@ class BlobFile:
|
||||
return
|
||||
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s", self.length, length)
|
||||
|
||||
def get_length(self):
|
||||
def get_length(self) -> typing.Optional[int]:
|
||||
return self.length
|
||||
|
||||
def get_is_verified(self):
|
||||
def get_is_verified(self) -> bool:
|
||||
return self.verified.is_set()
|
||||
|
||||
def is_readable(self) -> bool:
|
||||
return self.verified.is_set()
|
||||
|
||||
def is_writeable(self) -> bool:
|
||||
return not self.writing.is_set()
|
||||
|
||||
def write_blob(self, blob_bytes: bytes):
|
||||
if not self.is_writeable():
|
||||
raise OSError("cannot open blob for writing")
|
||||
try:
|
||||
self.writing.set()
|
||||
self._write_blob(blob_bytes)
|
||||
finally:
|
||||
self.writing.clear()
|
||||
|
||||
def close(self):
|
||||
while self.writers:
|
||||
peer, writer = self.writers.popitem()
|
||||
if writer and writer.finished and not writer.finished.done() and not self.loop.is_closed():
|
||||
writer.finished.cancel()
|
||||
while self.readers:
|
||||
reader = self.readers.pop()
|
||||
if reader:
|
||||
reader.close()
|
||||
|
||||
def delete(self):
|
||||
self.close()
|
||||
self.verified.clear()
|
||||
self.length = None
|
||||
|
||||
async def sendfile(self, writer: asyncio.StreamWriter) -> int:
|
||||
"""
|
||||
Read and send the file to the writer and return the number of bytes sent
|
||||
"""
|
||||
|
||||
if not self.is_readable():
|
||||
raise OSError('blob files cannot be read')
|
||||
with self.reader_context() as handle:
|
||||
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
|
||||
|
||||
def decrypt(self, key: bytes, iv: bytes) -> bytes:
|
||||
"""
|
||||
Decrypt a BlobFile to plaintext bytes
|
||||
"""
|
||||
|
||||
with self.reader_context() as reader:
|
||||
return decrypt_blob_bytes(reader.read(), self.length, key, iv)
|
||||
|
||||
@classmethod
|
||||
async def create_from_unencrypted(
|
||||
cls, loop: asyncio.BaseEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
|
||||
unencrypted: bytes, blob_num: int,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None) -> BlobInfo:
|
||||
"""
|
||||
Create an encrypted BlobFile from plaintext bytes
|
||||
"""
|
||||
|
||||
blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted)
|
||||
length = len(blob_bytes)
|
||||
blob = cls(loop, blob_hash, length, blob_completed_callback, blob_dir)
|
||||
writer = blob.get_blob_writer()
|
||||
writer.write(blob_bytes)
|
||||
await blob.verified.wait()
|
||||
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash)
|
||||
|
||||
def save_verified_blob(self, verified_bytes: bytes):
|
||||
if self.verified.is_set():
|
||||
return
|
||||
if self.is_writeable():
|
||||
self._write_blob(verified_bytes)
|
||||
self.verified.set()
|
||||
if self.blob_completed_callback:
|
||||
self.blob_completed_callback(self)
|
||||
|
||||
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
|
||||
peer_port: typing.Optional[int] = None) -> HashBlobWriter:
|
||||
if (peer_address, peer_port) in self.writers and not self.writers[(peer_address, peer_port)].closed():
|
||||
raise OSError(f"attempted to download blob twice from {peer_address}:{peer_port}")
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
|
||||
self.writers[(peer_address, peer_port)] = writer
|
||||
|
||||
def remove_writer(_):
|
||||
if (peer_address, peer_port) in self.writers:
|
||||
del self.writers[(peer_address, peer_port)]
|
||||
|
||||
fut.add_done_callback(remove_writer)
|
||||
|
||||
def writer_finished_callback(finished: asyncio.Future):
|
||||
try:
|
||||
err = finished.exception()
|
||||
if err:
|
||||
raise err
|
||||
verified_bytes = finished.result()
|
||||
while self.writers:
|
||||
_, other = self.writers.popitem()
|
||||
if other is not writer:
|
||||
other.close_handle()
|
||||
self.save_verified_blob(verified_bytes)
|
||||
except (InvalidBlobHashError, InvalidDataError) as error:
|
||||
log.warning("writer error downloading %s: %s", self.blob_hash[:8], str(error))
|
||||
except (DownloadCancelledError, asyncio.CancelledError, asyncio.TimeoutError):
|
||||
pass
|
||||
|
||||
fut.add_done_callback(writer_finished_callback)
|
||||
return writer
|
||||
|
||||
|
||||
class BlobBuffer(AbstractBlob):
|
||||
"""
|
||||
An in-memory only blob
|
||||
"""
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None):
|
||||
self._verified_bytes: typing.Optional[BytesIO] = None
|
||||
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
if not self.is_readable():
|
||||
raise OSError("cannot open blob for reading")
|
||||
try:
|
||||
yield self._verified_bytes
|
||||
finally:
|
||||
if self._verified_bytes:
|
||||
self._verified_bytes.close()
|
||||
self._verified_bytes = None
|
||||
self.verified.clear()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
if self._verified_bytes:
|
||||
raise OSError("already have bytes for blob")
|
||||
self._verified_bytes = BytesIO(blob_bytes)
|
||||
|
||||
def delete(self):
|
||||
if self._verified_bytes:
|
||||
self._verified_bytes.close()
|
||||
self._verified_bytes = None
|
||||
return super().delete()
|
||||
|
||||
def __del__(self):
|
||||
super().__del__()
|
||||
if self._verified_bytes:
|
||||
self.delete()
|
||||
|
||||
|
||||
class BlobFile(AbstractBlob):
|
||||
"""
|
||||
A blob existing on the local file system
|
||||
"""
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None):
|
||||
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
|
||||
if not blob_directory or not os.path.isdir(blob_directory):
|
||||
raise OSError(f"invalid blob directory '{blob_directory}'")
|
||||
self.file_path = os.path.join(self.blob_directory, self.blob_hash)
|
||||
if self.file_exists:
|
||||
file_size = int(os.stat(self.file_path).st_size)
|
||||
if length and length != file_size:
|
||||
log.warning("expected %s to be %s bytes, file has %s", self.blob_hash, length, file_size)
|
||||
self.delete()
|
||||
else:
|
||||
self.length = file_size
|
||||
self.verified.set()
|
||||
|
||||
@property
|
||||
def file_exists(self):
|
||||
return os.path.isfile(self.file_path)
|
||||
|
||||
def is_writeable(self) -> bool:
|
||||
return super().is_writeable() and not os.path.isfile(self.file_path)
|
||||
|
||||
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
|
||||
peer_port: typing.Optional[str] = None) -> HashBlobWriter:
|
||||
if self.file_exists:
|
||||
raise OSError(f"File already exists '{self.file_path}'")
|
||||
return super().get_blob_writer(peer_address, peer_port)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
handle = open(self.file_path, 'rb')
|
||||
try:
|
||||
yield handle
|
||||
finally:
|
||||
handle.close()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
with open(self.file_path, 'wb') as f:
|
||||
f.write(blob_bytes)
|
||||
|
||||
def delete(self):
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
return super().delete()
|
||||
|
||||
@classmethod
|
||||
async def create_from_unencrypted(
|
||||
cls, loop: asyncio.BaseEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
|
||||
unencrypted: bytes, blob_num: int,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'],
|
||||
asyncio.Task]] = None) -> BlobInfo:
|
||||
if not blob_dir or not os.path.isdir(blob_dir):
|
||||
raise OSError(f"cannot create blob in directory: '{blob_dir}'")
|
||||
return await super().create_from_unencrypted(
|
||||
loop, blob_dir, key, iv, unencrypted, blob_num, blob_completed_callback
|
||||
)
|
||||
@@ -2,6 +2,13 @@ import typing
|
||||
|
||||
|
||||
class BlobInfo:
|
||||
__slots__ = [
|
||||
'blob_hash',
|
||||
'blob_num',
|
||||
'length',
|
||||
'iv',
|
||||
]
|
||||
|
||||
def __init__(self, blob_num: int, length: int, iv: str, blob_hash: typing.Optional[str] = None):
|
||||
self.blob_hash = blob_hash
|
||||
self.blob_num = blob_num
|
||||
|
||||
@@ -2,18 +2,19 @@ import os
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_file import BlobFile, is_valid_blobhash
|
||||
from lbrynet.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.dht.protocol.data_store import DictDataStore
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobFileManager:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, storage: SQLiteStorage,
|
||||
class BlobManager:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, storage: 'SQLiteStorage', config: 'Config',
|
||||
node_data_store: typing.Optional['DictDataStore'] = None):
|
||||
"""
|
||||
This class stores blobs on the hard disk
|
||||
@@ -27,16 +28,59 @@ class BlobFileManager:
|
||||
self._node_data_store = node_data_store
|
||||
self.completed_blob_hashes: typing.Set[str] = set() if not self._node_data_store\
|
||||
else self._node_data_store.completed_blobs
|
||||
self.blobs: typing.Dict[str, BlobFile] = {}
|
||||
self.blobs: typing.Dict[str, AbstractBlob] = {}
|
||||
self.config = config
|
||||
|
||||
def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None):
|
||||
if self.config.save_blobs:
|
||||
return BlobFile(
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
||||
)
|
||||
else:
|
||||
if is_valid_blobhash(blob_hash) and os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
return BlobFile(
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
||||
)
|
||||
return BlobBuffer(
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
||||
)
|
||||
|
||||
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
|
||||
if blob_hash in self.blobs:
|
||||
if self.config.save_blobs and isinstance(self.blobs[blob_hash], BlobBuffer):
|
||||
buffer = self.blobs.pop(blob_hash)
|
||||
if blob_hash in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.remove(blob_hash)
|
||||
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
|
||||
if buffer.is_readable():
|
||||
with buffer.reader_context() as reader:
|
||||
self.blobs[blob_hash].write_blob(reader.read())
|
||||
if length and self.blobs[blob_hash].length is None:
|
||||
self.blobs[blob_hash].set_length(length)
|
||||
else:
|
||||
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
|
||||
return self.blobs[blob_hash]
|
||||
|
||||
def is_blob_verified(self, blob_hash: str, length: typing.Optional[int] = None) -> bool:
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise ValueError(blob_hash)
|
||||
if blob_hash in self.blobs:
|
||||
return self.blobs[blob_hash].get_is_verified()
|
||||
if not os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
return False
|
||||
return self._get_blob(blob_hash, length).get_is_verified()
|
||||
|
||||
async def setup(self) -> bool:
|
||||
def get_files_in_blob_dir() -> typing.Set[str]:
|
||||
if not self.blob_dir:
|
||||
return set()
|
||||
return {
|
||||
item.name for item in os.scandir(self.blob_dir) if is_valid_blobhash(item.name)
|
||||
}
|
||||
|
||||
in_blobfiles_dir = await self.loop.run_in_executor(None, get_files_in_blob_dir)
|
||||
self.completed_blob_hashes.update(await self.storage.sync_missing_blobs(in_blobfiles_dir))
|
||||
to_add = await self.storage.sync_missing_blobs(in_blobfiles_dir)
|
||||
if to_add:
|
||||
self.completed_blob_hashes.update(to_add)
|
||||
return True
|
||||
|
||||
def stop(self):
|
||||
@@ -45,37 +89,31 @@ class BlobFileManager:
|
||||
blob.close()
|
||||
self.completed_blob_hashes.clear()
|
||||
|
||||
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
|
||||
if blob_hash in self.blobs:
|
||||
if length and self.blobs[blob_hash].length is None:
|
||||
self.blobs[blob_hash].set_length(length)
|
||||
else:
|
||||
self.blobs[blob_hash] = BlobFile(self.loop, self.blob_dir, blob_hash, length, self.blob_completed)
|
||||
return self.blobs[blob_hash]
|
||||
|
||||
def get_stream_descriptor(self, sd_hash):
|
||||
return StreamDescriptor.from_stream_descriptor_blob(self.loop, self.blob_dir, self.get_blob(sd_hash))
|
||||
|
||||
async def blob_completed(self, blob: BlobFile):
|
||||
def blob_completed(self, blob: AbstractBlob) -> asyncio.Task:
|
||||
if blob.blob_hash is None:
|
||||
raise Exception("Blob hash is None")
|
||||
if not blob.length:
|
||||
raise Exception("Blob has a length of 0")
|
||||
if blob.blob_hash not in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.add(blob.blob_hash)
|
||||
await self.storage.add_completed_blob(blob.blob_hash, blob.length)
|
||||
if isinstance(blob, BlobFile):
|
||||
if blob.blob_hash not in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.add(blob.blob_hash)
|
||||
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=True))
|
||||
else:
|
||||
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=False))
|
||||
|
||||
def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]:
|
||||
"""Returns of the blobhashes_to_check, which are valid"""
|
||||
blobs = [self.get_blob(b) for b in blob_hashes]
|
||||
return [blob.blob_hash for blob in blobs if blob.get_is_verified()]
|
||||
return [blob_hash for blob_hash in blob_hashes if self.is_blob_verified(blob_hash)]
|
||||
|
||||
def delete_blob(self, blob_hash: str):
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise Exception("invalid blob hash to delete")
|
||||
|
||||
if blob_hash not in self.blobs:
|
||||
if os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
if self.blob_dir and os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
os.remove(os.path.join(self.blob_dir, blob_hash))
|
||||
else:
|
||||
self.blobs.pop(blob_hash).delete()
|
||||
|
||||
@@ -44,16 +44,15 @@ class HashBlobWriter:
|
||||
self._hashsum.update(data)
|
||||
self.len_so_far += len(data)
|
||||
if self.len_so_far > expected_length:
|
||||
self.close_handle()
|
||||
self.finished.set_exception(InvalidDataError(
|
||||
f'Length so far is greater than the expected length. {self.len_so_far} to {expected_length}'
|
||||
))
|
||||
self.close_handle()
|
||||
return
|
||||
self.buffer.write(data)
|
||||
if self.len_so_far == expected_length:
|
||||
blob_hash = self.calculate_blob_hash()
|
||||
if blob_hash != self.expected_blob_hash:
|
||||
self.close_handle()
|
||||
self.finished.set_exception(InvalidBlobHashError(
|
||||
f"blob hash is {blob_hash} vs expected {self.expected_blob_hash}"
|
||||
))
|
||||
@@ -62,6 +61,8 @@ class HashBlobWriter:
|
||||
self.close_handle()
|
||||
|
||||
def close_handle(self):
|
||||
if not self.finished.done():
|
||||
self.finished.cancel()
|
||||
if self.buffer is not None:
|
||||
self.buffer.close()
|
||||
self.buffer = None
|
||||
@@ -4,8 +4,9 @@ import typing
|
||||
import binascii
|
||||
from lbrynet.error import InvalidBlobHashError, InvalidDataError
|
||||
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest
|
||||
from lbrynet.utils import cache_concurrent
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from lbrynet.blob.blob_file import AbstractBlob
|
||||
from lbrynet.blob.writer import HashBlobWriter
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -17,15 +18,18 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
self.peer_port: typing.Optional[int] = None
|
||||
self.peer_address: typing.Optional[str] = None
|
||||
self.peer_timeout = peer_timeout
|
||||
self.transport: asyncio.Transport = None
|
||||
self.transport: typing.Optional[asyncio.Transport] = None
|
||||
|
||||
self.writer: 'HashBlobWriter' = None
|
||||
self.blob: 'BlobFile' = None
|
||||
self.writer: typing.Optional['HashBlobWriter'] = None
|
||||
self.blob: typing.Optional['AbstractBlob'] = None
|
||||
|
||||
self._blob_bytes_received = 0
|
||||
self._response_fut: asyncio.Future = None
|
||||
self._response_fut: typing.Optional[asyncio.Future] = None
|
||||
self.buf = b''
|
||||
|
||||
# this is here to handle the race when the downloader is closed right as response_fut gets a result
|
||||
self.closed = asyncio.Event(loop=self.loop)
|
||||
|
||||
def data_received(self, data: bytes):
|
||||
log.debug("%s:%d -- got %s bytes -- %s bytes on buffer -- %s blob bytes received",
|
||||
self.peer_address, self.peer_port, len(data), len(self.buf), self._blob_bytes_received)
|
||||
@@ -35,6 +39,9 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return
|
||||
if not self._response_fut:
|
||||
log.warning("Protocol received data before expected, probable race on keep alive. Closing transport.")
|
||||
return self.close()
|
||||
if self._blob_bytes_received and not self.writer.closed():
|
||||
return self._write(data)
|
||||
|
||||
@@ -63,8 +70,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
# write blob bytes if we're writing a blob and have blob bytes to write
|
||||
self._write(response.blob_data)
|
||||
|
||||
|
||||
def _write(self, data):
|
||||
def _write(self, data: bytes):
|
||||
if len(data) > (self.blob.get_length() - self._blob_bytes_received):
|
||||
data = data[:(self.blob.get_length() - self._blob_bytes_received)]
|
||||
log.warning("got more than asked from %s:%d, probable sendfile bug", self.peer_address, self.peer_port)
|
||||
@@ -87,6 +93,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
:return: download success (bool), keep connection (bool)
|
||||
"""
|
||||
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
|
||||
blob_hash = self.blob.blob_hash
|
||||
try:
|
||||
msg = request.serialize()
|
||||
log.debug("send request to %s:%i -> %s", self.peer_address, self.peer_port, msg.decode())
|
||||
@@ -95,6 +102,10 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
availability_response = response.get_availability_response()
|
||||
price_response = response.get_price_response()
|
||||
blob_response = response.get_blob_response()
|
||||
if self.closed.is_set():
|
||||
msg = f"cancelled blob request for {blob_hash} immediately after we got a response"
|
||||
log.warning(msg)
|
||||
raise asyncio.CancelledError(msg)
|
||||
if (not blob_response or blob_response.error) and\
|
||||
(not availability_response or not availability_response.available_blobs):
|
||||
log.warning("%s not in availability response from %s:%i", self.blob.blob_hash, self.peer_address,
|
||||
@@ -133,6 +144,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
return self._blob_bytes_received, self.close()
|
||||
|
||||
def close(self):
|
||||
self.closed.set()
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
if self.writer and not self.writer.closed():
|
||||
@@ -145,18 +157,21 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
self.transport = None
|
||||
self.buf = b''
|
||||
|
||||
async def download_blob(self, blob: 'BlobFile') -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
if blob.get_is_verified() or blob.file_exists or blob.blob_write_lock.locked():
|
||||
async def download_blob(self, blob: 'AbstractBlob') -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
self.closed.clear()
|
||||
blob_hash = blob.blob_hash
|
||||
if blob.get_is_verified() or not blob.is_writeable():
|
||||
return 0, self.transport
|
||||
try:
|
||||
self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0
|
||||
self._blob_bytes_received = 0
|
||||
self.blob, self.writer = blob, blob.get_blob_writer(self.peer_address, self.peer_port)
|
||||
self._response_fut = asyncio.Future(loop=self.loop)
|
||||
return await self._download_blob()
|
||||
except OSError as e:
|
||||
log.error("race happened downloading from %s:%i", self.peer_address, self.peer_port)
|
||||
# i'm not sure how to fix this race condition - jack
|
||||
log.exception(e)
|
||||
return self._blob_bytes_received, self.transport
|
||||
log.warning("race happened downloading %s from %s:%i", blob_hash, self.peer_address, self.peer_port)
|
||||
# return self._blob_bytes_received, self.transport
|
||||
raise
|
||||
except asyncio.TimeoutError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
@@ -165,6 +180,10 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
except asyncio.CancelledError:
|
||||
self.close()
|
||||
raise
|
||||
finally:
|
||||
if self.writer and not self.writer.closed():
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
|
||||
def connection_made(self, transport: asyncio.Transport):
|
||||
self.transport = transport
|
||||
@@ -177,9 +196,10 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
self.close()
|
||||
|
||||
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', address: str, tcp_port: int,
|
||||
@cache_concurrent
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'AbstractBlob', address: str, tcp_port: int,
|
||||
peer_connect_timeout: float, blob_download_timeout: float,
|
||||
connected_transport: asyncio.Transport = None)\
|
||||
connected_transport: asyncio.Transport = None, connection_id: int = 0)\
|
||||
-> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
"""
|
||||
Returns [<downloaded blob>, <keep connection>]
|
||||
@@ -196,7 +216,7 @@ async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', address: s
|
||||
if not connected_transport:
|
||||
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
|
||||
peer_connect_timeout, loop=loop)
|
||||
if blob.get_is_verified() or blob.file_exists:
|
||||
if blob.get_is_verified() or not blob.is_writeable():
|
||||
# file exists but not verified means someone is writing right now, give it time, come back later
|
||||
return 0, connected_transport
|
||||
return await protocol.download_blob(blob)
|
||||
|
||||
@@ -1,21 +1,23 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet.utils import drain_tasks
|
||||
from lbrynet.utils import cache_concurrent
|
||||
from lbrynet.blob_exchange.client import request_blob
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
from lbrynet.blob.blob_file import AbstractBlob
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobDownloader:
|
||||
BAN_TIME = 10.0 # fixme: when connection manager gets implemented, move it out from here
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager',
|
||||
BAN_FACTOR = 2.0 # fixme: when connection manager gets implemented, move it out from here
|
||||
BAN_CAP = 30.0*60.0 # maximum time a node remains banned before being retried
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
peer_queue: asyncio.Queue):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
@@ -24,21 +26,16 @@ class BlobDownloader:
|
||||
self.active_connections: typing.Dict['KademliaPeer', asyncio.Task] = {} # active request_blob calls
|
||||
self.ignored: typing.Dict['KademliaPeer', int] = {}
|
||||
self.scores: typing.Dict['KademliaPeer', int] = {}
|
||||
self.failures: typing.Dict['KademliaPeer', int] = {}
|
||||
self.connections: typing.Dict['KademliaPeer', asyncio.Transport] = {}
|
||||
self.time_since_last_blob = loop.time()
|
||||
self.is_running = asyncio.Event(loop=self.loop)
|
||||
|
||||
def should_race_continue(self, blob: 'BlobFile'):
|
||||
def should_race_continue(self, blob: 'AbstractBlob'):
|
||||
if len(self.active_connections) >= self.config.max_connections_per_download:
|
||||
return False
|
||||
# if a peer won 3 or more blob races and is active as a downloader, stop the race so bandwidth improves
|
||||
# the safe net side is that any failure will reset the peer score, triggering the race back
|
||||
# TODO: this is a good idea for low bandwidth, but doesnt play nice on high bandwidth
|
||||
# for peer, task in self.active_connections.items():
|
||||
# if self.scores.get(peer, 0) >= 0 and self.rounds_won.get(peer, 0) >= 3 and not task.done():
|
||||
# return False
|
||||
return not (blob.get_is_verified() or blob.file_exists)
|
||||
return not (blob.get_is_verified() or not blob.is_writeable())
|
||||
|
||||
async def request_blob_from_peer(self, blob: 'BlobFile', peer: 'KademliaPeer'):
|
||||
async def request_blob_from_peer(self, blob: 'AbstractBlob', peer: 'KademliaPeer', connection_id: int = 0):
|
||||
if blob.get_is_verified():
|
||||
return
|
||||
self.scores[peer] = self.scores.get(peer, 0) - 1 # starts losing score, to account for cancelled ones
|
||||
@@ -46,102 +43,80 @@ class BlobDownloader:
|
||||
start = self.loop.time()
|
||||
bytes_received, transport = await request_blob(
|
||||
self.loop, blob, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
|
||||
self.config.blob_download_timeout, connected_transport=transport
|
||||
self.config.blob_download_timeout, connected_transport=transport, connection_id=connection_id
|
||||
)
|
||||
if bytes_received == blob.get_length():
|
||||
self.time_since_last_blob = self.loop.time()
|
||||
if not transport and peer not in self.ignored:
|
||||
self.ignored[peer] = self.loop.time()
|
||||
log.debug("drop peer %s:%i", peer.address, peer.tcp_port)
|
||||
self.failures[peer] = self.failures.get(peer, 0) + 1
|
||||
if peer in self.connections:
|
||||
del self.connections[peer]
|
||||
elif transport:
|
||||
log.debug("keep peer %s:%i", peer.address, peer.tcp_port)
|
||||
self.failures[peer] = 0
|
||||
self.connections[peer] = transport
|
||||
rough_speed = (bytes_received / (self.loop.time() - start)) if bytes_received else 0
|
||||
self.scores[peer] = rough_speed
|
||||
elapsed = self.loop.time() - start
|
||||
self.scores[peer] = bytes_received / elapsed if bytes_received and elapsed else 0
|
||||
|
||||
async def new_peer_or_finished(self, blob: 'BlobFile'):
|
||||
async def get_and_re_add_peers():
|
||||
try:
|
||||
new_peers = await asyncio.wait_for(self.peer_queue.get(), timeout=1.0)
|
||||
self.peer_queue.put_nowait(new_peers)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
tasks = [self.loop.create_task(get_and_re_add_peers()), self.loop.create_task(blob.verified.wait())]
|
||||
active_tasks = list(self.active_connections.values())
|
||||
try:
|
||||
await asyncio.wait(tasks + active_tasks, loop=self.loop, return_when='FIRST_COMPLETED')
|
||||
finally:
|
||||
drain_tasks(tasks)
|
||||
async def new_peer_or_finished(self):
|
||||
active_tasks = list(self.active_connections.values()) + [asyncio.sleep(1)]
|
||||
await asyncio.wait(active_tasks, loop=self.loop, return_when='FIRST_COMPLETED')
|
||||
|
||||
def cleanup_active(self):
|
||||
to_remove = [peer for (peer, task) in self.active_connections.items() if task.done()]
|
||||
for peer in to_remove:
|
||||
del self.active_connections[peer]
|
||||
self.clearbanned()
|
||||
|
||||
def clearbanned(self):
|
||||
now = self.loop.time()
|
||||
if now - self.time_since_last_blob > 60.0:
|
||||
return
|
||||
forgiven = [banned_peer for banned_peer, when in self.ignored.items() if now - when > self.BAN_TIME]
|
||||
self.peer_queue.put_nowait(forgiven)
|
||||
for banned_peer in forgiven:
|
||||
self.ignored.pop(banned_peer)
|
||||
self.ignored = dict((
|
||||
(peer, when) for (peer, when) in self.ignored.items()
|
||||
if (now - when) < min(self.BAN_CAP, (self.failures.get(peer, 0) ** self.BAN_FACTOR))
|
||||
))
|
||||
|
||||
async def download_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
|
||||
@cache_concurrent
|
||||
async def download_blob(self, blob_hash: str, length: typing.Optional[int] = None,
|
||||
connection_id: int = 0) -> 'AbstractBlob':
|
||||
blob = self.blob_manager.get_blob(blob_hash, length)
|
||||
if blob.get_is_verified():
|
||||
return blob
|
||||
self.is_running.set()
|
||||
try:
|
||||
while not blob.get_is_verified():
|
||||
batch: typing.List['KademliaPeer'] = []
|
||||
while not blob.get_is_verified() and self.is_running.is_set():
|
||||
batch: typing.Set['KademliaPeer'] = set()
|
||||
while not self.peer_queue.empty():
|
||||
batch.extend(self.peer_queue.get_nowait())
|
||||
batch.sort(key=lambda peer: self.scores.get(peer, 0), reverse=True)
|
||||
batch.update(self.peer_queue.get_nowait())
|
||||
if batch:
|
||||
self.peer_queue.put_nowait(list(batch))
|
||||
log.debug(
|
||||
"running, %d peers, %d ignored, %d active",
|
||||
len(batch), len(self.ignored), len(self.active_connections)
|
||||
)
|
||||
for peer in batch:
|
||||
for peer in sorted(batch, key=lambda peer: self.scores.get(peer, 0), reverse=True):
|
||||
if not self.should_race_continue(blob):
|
||||
break
|
||||
if peer not in self.active_connections and peer not in self.ignored:
|
||||
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
|
||||
t = self.loop.create_task(self.request_blob_from_peer(blob, peer))
|
||||
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id))
|
||||
self.active_connections[peer] = t
|
||||
await self.new_peer_or_finished(blob)
|
||||
await self.new_peer_or_finished()
|
||||
self.cleanup_active()
|
||||
if batch:
|
||||
self.peer_queue.put_nowait(set(batch).difference(self.ignored))
|
||||
else:
|
||||
self.clearbanned()
|
||||
while self.active_connections:
|
||||
peer, task = self.active_connections.popitem()
|
||||
if task and not task.done():
|
||||
task.cancel()
|
||||
blob.close()
|
||||
log.debug("downloaded %s", blob_hash[:8])
|
||||
return blob
|
||||
except asyncio.CancelledError:
|
||||
while self.active_connections:
|
||||
peer, task = self.active_connections.popitem()
|
||||
if task and not task.done():
|
||||
task.cancel()
|
||||
raise
|
||||
except (OSError, Exception) as e:
|
||||
log.exception(e)
|
||||
raise e
|
||||
finally:
|
||||
blob.close()
|
||||
|
||||
def close(self):
|
||||
self.scores.clear()
|
||||
self.ignored.clear()
|
||||
self.is_running.clear()
|
||||
for transport in self.connections.values():
|
||||
transport.close()
|
||||
|
||||
|
||||
async def download_blob(loop, config: 'Config', blob_manager: 'BlobFileManager', node: 'Node',
|
||||
blob_hash: str) -> 'BlobFile':
|
||||
async def download_blob(loop, config: 'Config', blob_manager: 'BlobManager', node: 'Node',
|
||||
blob_hash: str) -> 'AbstractBlob':
|
||||
search_queue = asyncio.Queue(loop=loop, maxsize=config.max_connections_per_download)
|
||||
search_queue.put_nowait(blob_hash)
|
||||
peer_queue, accumulate_task = node.accumulate_peers(search_queue)
|
||||
|
||||
@@ -8,13 +8,13 @@ from lbrynet.blob_exchange.serialization import BlobAvailabilityResponse, BlobPr
|
||||
BlobPaymentAddressResponse
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobServerProtocol(asyncio.Protocol):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', lbrycrd_address: str):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str):
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
@@ -94,7 +94,7 @@ class BlobServerProtocol(asyncio.Protocol):
|
||||
|
||||
|
||||
class BlobServer:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', lbrycrd_address: str):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str):
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
|
||||
+16
-6
@@ -21,10 +21,6 @@ CURRENCIES = {
|
||||
'LBC': {'type': 'crypto'},
|
||||
'USD': {'type': 'fiat'},
|
||||
}
|
||||
SLACK_WEBHOOK = (
|
||||
'nUE0pUZ6Yl9bo29epl5moTSwnl5wo20ip2IlqzywMKZiIQSFZR5'
|
||||
'AHx4mY0VmF0WQZ1ESEP9kMHZlp1WzJwWOoKN3ImR1M2yUAaMyqGZ='
|
||||
)
|
||||
HEADERS_FILE_SHA256_CHECKSUM = (
|
||||
366295, 'b0c8197153a33ccbc52fb81a279588b6015b68b7726f73f6a2b81f7e25bfe4b9'
|
||||
)
|
||||
@@ -473,13 +469,23 @@ class Config(CLIConfig):
|
||||
tcp_port = Integer("TCP port to listen for incoming blob requests", 3333, previous_names=['peer_port'])
|
||||
network_interface = String("Interface to use for the DHT and blob exchange", '0.0.0.0')
|
||||
|
||||
# routing table
|
||||
split_buckets_under_index = Integer(
|
||||
"Routing table bucket index below which we always split the bucket if given a new key to add to it and "
|
||||
"the bucket is full. As this value is raised the depth of the routing table (and number of peers in it) "
|
||||
"will increase. This setting is used by seed nodes, you probably don't want to change it during normal "
|
||||
"use.", 1
|
||||
)
|
||||
|
||||
# protocol timeouts
|
||||
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
|
||||
blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
|
||||
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 2.0)
|
||||
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0)
|
||||
node_rpc_timeout = Float("Timeout when making a DHT request", constants.rpc_timeout)
|
||||
|
||||
# blob announcement and download
|
||||
save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True)
|
||||
|
||||
announce_head_and_sd_only = Toggle(
|
||||
"Announce only the descriptor and first (rather than all) data blob for a stream to the DHT", True,
|
||||
previous_names=['announce_head_blobs_only']
|
||||
@@ -489,7 +495,7 @@ class Config(CLIConfig):
|
||||
previous_names=['concurrent_announcers']
|
||||
)
|
||||
max_connections_per_download = Integer(
|
||||
"Maximum number of peers to connect to while downloading a blob", 8,
|
||||
"Maximum number of peers to connect to while downloading a blob", 4,
|
||||
previous_names=['max_connections_per_stream']
|
||||
)
|
||||
fixed_peer_delay = Float(
|
||||
@@ -524,12 +530,16 @@ class Config(CLIConfig):
|
||||
('lbrynet4.lbry.io', 4444) # ASIA
|
||||
])
|
||||
|
||||
# comment server uses a single string because requests are made to the /api resource
|
||||
comment_server = String("Server to store user-generated metadata.", "http://comments.lbry.com:2903/api")
|
||||
|
||||
# blockchain
|
||||
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
|
||||
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
|
||||
cache_time = Integer("Time to cache resolved claims", 150) # TODO: use this
|
||||
|
||||
# daemon
|
||||
save_files = Toggle("Save downloaded files when calling `get` by default", True)
|
||||
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
|
||||
share_usage_data = Toggle(
|
||||
"Whether to share usage stats and diagnostic info with LBRY.", True,
|
||||
|
||||
@@ -13,55 +13,44 @@ class BlobAnnouncer:
|
||||
self.loop = loop
|
||||
self.node = node
|
||||
self.storage = storage
|
||||
self.pending_call: asyncio.Handle = None
|
||||
self.announce_task: asyncio.Task = None
|
||||
self.running = False
|
||||
self.announce_queue: typing.List[str] = []
|
||||
|
||||
async def _announce(self, batch_size: typing.Optional[int] = 10):
|
||||
if not batch_size:
|
||||
return
|
||||
if not self.node.joined.is_set():
|
||||
await self.node.joined.wait()
|
||||
blob_hashes = await self.storage.get_blobs_to_announce()
|
||||
if blob_hashes:
|
||||
self.announce_queue.extend(blob_hashes)
|
||||
log.info("%i blobs to announce", len(blob_hashes))
|
||||
batch = []
|
||||
while len(self.announce_queue):
|
||||
cnt = 0
|
||||
announced = []
|
||||
while self.announce_queue and cnt < batch_size:
|
||||
blob_hash = self.announce_queue.pop()
|
||||
announced.append(blob_hash)
|
||||
batch.append(self.node.announce_blob(blob_hash))
|
||||
cnt += 1
|
||||
to_await = []
|
||||
while batch:
|
||||
to_await.append(batch.pop())
|
||||
if to_await:
|
||||
await asyncio.gather(*tuple(to_await), loop=self.loop)
|
||||
await self.storage.update_last_announced_blobs(announced)
|
||||
log.info("announced %i blobs", len(announced))
|
||||
if self.running:
|
||||
self.pending_call = self.loop.call_later(60, self.announce, batch_size)
|
||||
async def _submit_announcement(self, blob_hash):
|
||||
try:
|
||||
peers = len(await self.node.announce_blob(blob_hash))
|
||||
if peers > 4:
|
||||
return blob_hash
|
||||
else:
|
||||
log.warning("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError):
|
||||
raise err
|
||||
log.warning("error announcing %s: %s", blob_hash[:8], str(err))
|
||||
|
||||
def announce(self, batch_size: typing.Optional[int] = 10):
|
||||
self.announce_task = self.loop.create_task(self._announce(batch_size))
|
||||
|
||||
async def _announce(self, batch_size: typing.Optional[int] = 10):
|
||||
while batch_size:
|
||||
if not self.node.joined.is_set():
|
||||
await self.node.joined.wait()
|
||||
self.announce_queue.extend(await self.storage.get_blobs_to_announce())
|
||||
log.debug("announcer task wake up, %d blobs to announce", len(self.announce_queue))
|
||||
while len(self.announce_queue):
|
||||
log.info("%i blobs to announce", len(self.announce_queue))
|
||||
announced = await asyncio.gather(*[
|
||||
self._submit_announcement(
|
||||
self.announce_queue.pop()) for _ in range(batch_size) if self.announce_queue
|
||||
], loop=self.loop)
|
||||
announced = list(filter(None, announced))
|
||||
if announced:
|
||||
await self.storage.update_last_announced_blobs(announced)
|
||||
log.info("announced %i blobs", len(announced))
|
||||
await asyncio.sleep(60)
|
||||
|
||||
def start(self, batch_size: typing.Optional[int] = 10):
|
||||
if self.running:
|
||||
raise Exception("already running")
|
||||
self.running = True
|
||||
self.announce(batch_size)
|
||||
assert not self.announce_task or self.announce_task.done(), "already running"
|
||||
self.announce_task = self.loop.create_task(self._announce(batch_size))
|
||||
|
||||
def stop(self):
|
||||
self.running = False
|
||||
if self.pending_call:
|
||||
if not self.pending_call.cancelled():
|
||||
self.pending_call.cancel()
|
||||
self.pending_call = None
|
||||
if self.announce_task:
|
||||
if not (self.announce_task.done() or self.announce_task.cancelled()):
|
||||
self.announce_task.cancel()
|
||||
self.announce_task = None
|
||||
if self.announce_task and not self.announce_task.done():
|
||||
self.announce_task.cancel()
|
||||
@@ -6,6 +6,7 @@ hash_length = hash_class().digest_size
|
||||
hash_bits = hash_length * 8
|
||||
alpha = 5
|
||||
k = 8
|
||||
split_buckets_under_index = 1
|
||||
replacement_cache_size = 8
|
||||
rpc_timeout = 5.0
|
||||
rpc_attempts = 5
|
||||
@@ -15,6 +16,7 @@ refresh_interval = 3600 # 1 hour
|
||||
replicate_interval = refresh_interval
|
||||
data_expiration = 86400 # 24 hours
|
||||
token_secret_refresh_interval = 300 # 5 minutes
|
||||
maybe_ping_delay = 300 # 5 minutes
|
||||
check_refresh_interval = refresh_interval / 5
|
||||
max_datagram_size = 8192 # 8 KB
|
||||
rpc_id_length = 20
|
||||
|
||||
+16
-25
@@ -20,10 +20,12 @@ log = logging.getLogger(__name__)
|
||||
|
||||
class Node:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int,
|
||||
internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: typing.Optional[float] = 5.0):
|
||||
internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: float = constants.rpc_timeout,
|
||||
split_buckets_under_index: int = constants.split_buckets_under_index):
|
||||
self.loop = loop
|
||||
self.internal_udp_port = internal_udp_port
|
||||
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port, rpc_timeout)
|
||||
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port, rpc_timeout,
|
||||
split_buckets_under_index)
|
||||
self.listening_port: asyncio.DatagramTransport = None
|
||||
self.joined = asyncio.Event(loop=self.loop)
|
||||
self._join_task: asyncio.Task = None
|
||||
@@ -128,7 +130,7 @@ class Node:
|
||||
if known_node_urls:
|
||||
for host, port in known_node_urls:
|
||||
address = await resolve_host(host, port, proto='udp')
|
||||
if (address, port) not in known_node_addresses:
|
||||
if (address, port) not in known_node_addresses and address != self.protocol.external_ip:
|
||||
known_node_addresses.append((address, port))
|
||||
url_to_addr[address] = host
|
||||
|
||||
@@ -188,10 +190,7 @@ class Node:
|
||||
|
||||
async def _add_hashes_from_queue():
|
||||
while True:
|
||||
try:
|
||||
blob_hash = await hash_queue.get()
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
blob_hash = await hash_queue.get()
|
||||
peer_generator.add_generator(
|
||||
self.get_iterative_value_finder(
|
||||
binascii.unhexlify(blob_hash.encode()), bottom_out_limit=bottom_out_limit,
|
||||
@@ -202,11 +201,6 @@ class Node:
|
||||
try:
|
||||
async with peer_generator as junction:
|
||||
yield junction
|
||||
await peer_generator.finished.wait()
|
||||
except asyncio.CancelledError:
|
||||
if add_hashes_task and not (add_hashes_task.done() or add_hashes_task.cancelled()):
|
||||
add_hashes_task.cancel()
|
||||
raise
|
||||
finally:
|
||||
if add_hashes_task and not (add_hashes_task.done() or add_hashes_task.cancelled()):
|
||||
add_hashes_task.cancel()
|
||||
@@ -234,19 +228,16 @@ class Node:
|
||||
|
||||
async def _accumulate_search_junction(self, search_queue: asyncio.Queue,
|
||||
result_queue: asyncio.Queue):
|
||||
try:
|
||||
async with self.stream_peer_search_junction(search_queue) as search_junction: # pylint: disable=E1701
|
||||
async for peers in search_junction:
|
||||
if peers:
|
||||
result_queue.put_nowait([
|
||||
peer for peer in peers
|
||||
if not (
|
||||
peer.address == self.protocol.external_ip
|
||||
and peer.tcp_port == self.protocol.peer_port
|
||||
)
|
||||
])
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
async with self.stream_peer_search_junction(search_queue) as search_junction: # pylint: disable=E1701
|
||||
async for peers in search_junction:
|
||||
if peers:
|
||||
result_queue.put_nowait([
|
||||
peer for peer in peers
|
||||
if not (
|
||||
peer.address == self.protocol.external_ip
|
||||
and peer.tcp_port == self.protocol.peer_port
|
||||
)
|
||||
])
|
||||
|
||||
def accumulate_peers(self, search_queue: asyncio.Queue,
|
||||
peer_queue: typing.Optional[asyncio.Queue] = None) -> typing.Tuple[
|
||||
|
||||
@@ -3,6 +3,8 @@ import asyncio
|
||||
import logging
|
||||
import ipaddress
|
||||
from binascii import hexlify
|
||||
from functools import lru_cache
|
||||
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
|
||||
|
||||
@@ -76,6 +78,7 @@ class PeerManager:
|
||||
self._node_id_mapping[(address, udp_port)] = node_id
|
||||
self._node_id_reverse_mapping[node_id] = (address, udp_port)
|
||||
|
||||
@lru_cache(maxsize=400)
|
||||
def get_kademlia_peer(self, node_id: bytes, address: str, udp_port: int) -> 'KademliaPeer':
|
||||
return KademliaPeer(self._loop, address, node_id, udp_port)
|
||||
|
||||
@@ -132,6 +135,15 @@ class PeerManager:
|
||||
|
||||
|
||||
class KademliaPeer:
|
||||
__slots__ = [
|
||||
'loop',
|
||||
'_node_id',
|
||||
'address',
|
||||
'udp_port',
|
||||
'tcp_port',
|
||||
'protocol_version',
|
||||
]
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, address: str, node_id: typing.Optional[bytes] = None,
|
||||
udp_port: typing.Optional[int] = None, tcp_port: typing.Optional[int] = None):
|
||||
if node_id is not None:
|
||||
|
||||
@@ -13,11 +13,6 @@ def cancel_task(task: typing.Optional[asyncio.Task]):
|
||||
task.cancel()
|
||||
|
||||
|
||||
def cancel_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
|
||||
for task in tasks:
|
||||
cancel_task(task)
|
||||
|
||||
|
||||
def drain_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
|
||||
while tasks:
|
||||
cancel_task(tasks.pop())
|
||||
@@ -31,12 +26,11 @@ class AsyncGeneratorJunction:
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, queue: typing.Optional[asyncio.Queue] = None):
|
||||
self.loop = loop
|
||||
self.__iterator_queue = asyncio.Queue(loop=loop)
|
||||
self.result_queue = queue or asyncio.Queue(loop=loop)
|
||||
self.tasks: typing.List[asyncio.Task] = []
|
||||
self.running_iterators: typing.Dict[typing.AsyncGenerator, bool] = {}
|
||||
self.generator_queue: asyncio.Queue = asyncio.Queue(loop=self.loop)
|
||||
self.can_iterate = asyncio.Event(loop=self.loop)
|
||||
self.finished = asyncio.Event(loop=self.loop)
|
||||
|
||||
@property
|
||||
def running(self):
|
||||
@@ -47,15 +41,16 @@ class AsyncGeneratorJunction:
|
||||
try:
|
||||
async for item in iterator:
|
||||
self.result_queue.put_nowait(item)
|
||||
self.__iterator_queue.put_nowait(item)
|
||||
finally:
|
||||
self.running_iterators[iterator] = False
|
||||
if not self.running:
|
||||
self.__iterator_queue.put_nowait(StopAsyncIteration)
|
||||
|
||||
while True:
|
||||
async_gen: typing.Union[typing.AsyncGenerator, 'AsyncGeneratorType'] = await self.generator_queue.get()
|
||||
self.running_iterators[async_gen] = True
|
||||
self.tasks.append(self.loop.create_task(iterate(async_gen)))
|
||||
if not self.can_iterate.is_set():
|
||||
self.can_iterate.set()
|
||||
|
||||
def add_generator(self, async_gen: typing.Union[typing.AsyncGenerator, 'AsyncGeneratorType']):
|
||||
"""
|
||||
@@ -67,14 +62,10 @@ class AsyncGeneratorJunction:
|
||||
return self
|
||||
|
||||
async def __anext__(self):
|
||||
if not self.can_iterate.is_set():
|
||||
await self.can_iterate.wait()
|
||||
if not self.running:
|
||||
raise StopAsyncIteration()
|
||||
try:
|
||||
return await self.result_queue.get()
|
||||
finally:
|
||||
self.awaiting = None
|
||||
result = await self.__iterator_queue.get()
|
||||
if result is StopAsyncIteration:
|
||||
raise result
|
||||
return result
|
||||
|
||||
def aclose(self):
|
||||
async def _aclose():
|
||||
@@ -85,8 +76,6 @@ class AsyncGeneratorJunction:
|
||||
self.running_iterators[iterator] = False
|
||||
drain_tasks(self.tasks)
|
||||
raise StopAsyncIteration()
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return self.loop.create_task(_aclose())
|
||||
|
||||
async def __aenter__(self):
|
||||
|
||||
@@ -10,11 +10,13 @@ class Distance:
|
||||
|
||||
def __init__(self, key: bytes):
|
||||
if len(key) != constants.hash_length:
|
||||
raise ValueError("invalid key length: %i" % len(key))
|
||||
raise ValueError(f"invalid key length: {len(key)}")
|
||||
self.key = key
|
||||
self.val_key_one = int.from_bytes(key, 'big')
|
||||
|
||||
def __call__(self, key_two: bytes) -> int:
|
||||
if len(key_two) != constants.hash_length:
|
||||
raise ValueError(f"invalid length of key to compare: {len(key_two)}")
|
||||
val_key_two = int.from_bytes(key_two, 'big')
|
||||
return self.val_key_one ^ val_key_two
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ from itertools import chain
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.error import RemoteException
|
||||
from lbrynet.dht.error import RemoteException, TransportNotConnected
|
||||
from lbrynet.dht.protocol.distance import Distance
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -162,8 +162,6 @@ class IterativeFinder:
|
||||
async def _send_probe(self, peer: 'KademliaPeer'):
|
||||
try:
|
||||
response = await self.send_probe(peer)
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
except asyncio.TimeoutError:
|
||||
self.active.discard(peer)
|
||||
return
|
||||
@@ -171,6 +169,8 @@ class IterativeFinder:
|
||||
log.warning(str(err))
|
||||
self.active.discard(peer)
|
||||
return
|
||||
except TransportNotConnected:
|
||||
return self.aclose()
|
||||
except RemoteException:
|
||||
return
|
||||
return await self._handle_probe_result(peer, response)
|
||||
@@ -217,7 +217,7 @@ class IterativeFinder:
|
||||
await self._search_round()
|
||||
if self.running:
|
||||
self.delayed_calls.append(self.loop.call_later(delay, self._search))
|
||||
except (asyncio.CancelledError, StopAsyncIteration):
|
||||
except (asyncio.CancelledError, StopAsyncIteration, TransportNotConnected):
|
||||
if self.running:
|
||||
self.loop.call_soon(self.aclose)
|
||||
|
||||
|
||||
@@ -187,56 +187,46 @@ class PingQueue:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, protocol: 'KademliaProtocol'):
|
||||
self._loop = loop
|
||||
self._protocol = protocol
|
||||
self._enqueued_contacts: typing.List['KademliaPeer'] = []
|
||||
self._pending_contacts: typing.Dict['KademliaPeer', float] = {}
|
||||
self._process_task: asyncio.Task = None
|
||||
self._next_task: asyncio.Future = None
|
||||
self._next_timer: asyncio.TimerHandle = None
|
||||
self._running = False
|
||||
self._running_pings: typing.Set[asyncio.Task] = set()
|
||||
|
||||
@property
|
||||
def running(self):
|
||||
return self._running
|
||||
|
||||
def enqueue_maybe_ping(self, *peers: 'KademliaPeer', delay: typing.Optional[float] = None):
|
||||
delay = constants.check_refresh_interval if delay is None else delay
|
||||
def enqueue_maybe_ping(self, *peers: 'KademliaPeer', delay: float = constants.maybe_ping_delay):
|
||||
now = self._loop.time()
|
||||
for peer in peers:
|
||||
if delay and peer not in self._enqueued_contacts:
|
||||
self._pending_contacts[peer] = self._loop.time() + delay
|
||||
elif peer not in self._enqueued_contacts:
|
||||
self._enqueued_contacts.append(peer)
|
||||
if peer in self._pending_contacts:
|
||||
del self._pending_contacts[peer]
|
||||
if peer not in self._pending_contacts or now + delay < self._pending_contacts[peer]:
|
||||
self._pending_contacts[peer] = delay + now
|
||||
|
||||
async def _process(self):
|
||||
async def _ping(p: 'KademliaPeer'):
|
||||
def maybe_ping(self, peer: 'KademliaPeer'):
|
||||
async def ping_task():
|
||||
try:
|
||||
if self._protocol.peer_manager.peer_is_good(p):
|
||||
await self._protocol.add_peer(p)
|
||||
if self._protocol.peer_manager.peer_is_good(peer):
|
||||
if peer not in self._protocol.routing_table.get_peers():
|
||||
await self._protocol.add_peer(peer)
|
||||
return
|
||||
await self._protocol.get_rpc_peer(p).ping()
|
||||
await self._protocol.get_rpc_peer(peer).ping()
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
task = self._loop.create_task(ping_task())
|
||||
task.add_done_callback(lambda _: None if task not in self._running_pings else self._running_pings.remove(task))
|
||||
self._running_pings.add(task)
|
||||
|
||||
async def _process(self): # send up to 1 ping per second
|
||||
while True:
|
||||
tasks = []
|
||||
|
||||
if self._enqueued_contacts or self._pending_contacts:
|
||||
now = self._loop.time()
|
||||
scheduled = [k for k, d in self._pending_contacts.items() if now >= d]
|
||||
for k in scheduled:
|
||||
del self._pending_contacts[k]
|
||||
if k not in self._enqueued_contacts:
|
||||
self._enqueued_contacts.append(k)
|
||||
while self._enqueued_contacts:
|
||||
peer = self._enqueued_contacts.pop()
|
||||
tasks.append(self._loop.create_task(_ping(peer)))
|
||||
if tasks:
|
||||
await asyncio.wait(tasks, loop=self._loop)
|
||||
|
||||
f = self._loop.create_future()
|
||||
self._loop.call_later(1.0, lambda: None if f.done() else f.set_result(None))
|
||||
await f
|
||||
enqueued = list(self._pending_contacts.keys())
|
||||
now = self._loop.time()
|
||||
for peer in enqueued:
|
||||
if self._pending_contacts[peer] <= now:
|
||||
del self._pending_contacts[peer]
|
||||
self.maybe_ping(peer)
|
||||
break
|
||||
await asyncio.sleep(1, loop=self._loop)
|
||||
|
||||
def start(self):
|
||||
assert not self._running
|
||||
@@ -250,17 +240,14 @@ class PingQueue:
|
||||
if self._process_task:
|
||||
self._process_task.cancel()
|
||||
self._process_task = None
|
||||
if self._next_task:
|
||||
self._next_task.cancel()
|
||||
self._next_task = None
|
||||
if self._next_timer:
|
||||
self._next_timer.cancel()
|
||||
self._next_timer = None
|
||||
while self._running_pings:
|
||||
self._running_pings.pop().cancel()
|
||||
|
||||
|
||||
class KademliaProtocol(DatagramProtocol):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', node_id: bytes, external_ip: str,
|
||||
udp_port: int, peer_port: int, rpc_timeout: float = 5.0):
|
||||
udp_port: int, peer_port: int, rpc_timeout: float = constants.rpc_timeout,
|
||||
split_buckets_under_index: int = constants.split_buckets_under_index):
|
||||
self.peer_manager = peer_manager
|
||||
self.loop = loop
|
||||
self.node_id = node_id
|
||||
@@ -275,7 +262,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
self.transport: DatagramTransport = None
|
||||
self.old_token_secret = constants.generate_id()
|
||||
self.token_secret = constants.generate_id()
|
||||
self.routing_table = TreeRoutingTable(self.loop, self.peer_manager, self.node_id)
|
||||
self.routing_table = TreeRoutingTable(self.loop, self.peer_manager, self.node_id, split_buckets_under_index)
|
||||
self.data_store = DictDataStore(self.loop, self.peer_manager)
|
||||
self.ping_queue = PingQueue(self.loop, self)
|
||||
self.node_rpc = KademliaRPC(self, self.loop, self.peer_port)
|
||||
@@ -312,9 +299,14 @@ class KademliaProtocol(DatagramProtocol):
|
||||
return args, {}
|
||||
|
||||
async def _add_peer(self, peer: 'KademliaPeer'):
|
||||
for p in self.routing_table.get_peers():
|
||||
if (p.address, p.udp_port) == (peer.address, peer.udp_port) and p.node_id != peer.node_id:
|
||||
self.routing_table.remove_peer(p)
|
||||
self.routing_table.join_buckets()
|
||||
bucket_index = self.routing_table.kbucket_index(peer.node_id)
|
||||
if self.routing_table.buckets[bucket_index].add_peer(peer):
|
||||
return True
|
||||
|
||||
# The bucket is full; see if it can be split (by checking if its range includes the host node's node_id)
|
||||
if self.routing_table.should_split(bucket_index, peer.node_id):
|
||||
self.routing_table.split_bucket(bucket_index)
|
||||
@@ -408,12 +400,13 @@ class KademliaProtocol(DatagramProtocol):
|
||||
sender_contact, ResponseDatagram(RESPONSE_TYPE, message.rpc_id, self.node_id, result),
|
||||
)
|
||||
|
||||
async def handle_request_datagram(self, address, request_datagram: RequestDatagram):
|
||||
async def handle_request_datagram(self, address: typing.Tuple[str, int], request_datagram: RequestDatagram):
|
||||
# This is an RPC method request
|
||||
self.peer_manager.report_last_requested(address[0], address[1])
|
||||
self.peer_manager.update_contact_triple(request_datagram.node_id, address[0], address[1])
|
||||
# only add a requesting contact to the routing table if it has replied to one of our requests
|
||||
peer = self.peer_manager.get_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
try:
|
||||
peer = self.routing_table.get_peer(request_datagram.node_id)
|
||||
except IndexError:
|
||||
peer = self.peer_manager.get_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
try:
|
||||
await self._handle_rpc(peer, request_datagram)
|
||||
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
|
||||
@@ -421,6 +414,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
is_good = self.peer_manager.peer_is_good(peer)
|
||||
if is_good is None:
|
||||
self.ping_queue.enqueue_maybe_ping(peer)
|
||||
# only add a requesting contact to the routing table if it has replied to one of our requests
|
||||
elif is_good is True:
|
||||
await self.add_peer(peer)
|
||||
except ValueError as err:
|
||||
@@ -448,11 +442,10 @@ class KademliaProtocol(DatagramProtocol):
|
||||
peer, df, request = self.sent_messages[response_datagram.rpc_id]
|
||||
if peer.address != address[0]:
|
||||
df.set_exception(RemoteException(
|
||||
f"response from {address[0]}:{address[1]}, "
|
||||
f"expected {peer.address}:{peer.udp_port}")
|
||||
f"response from {address[0]}, expected {peer.address}")
|
||||
)
|
||||
return
|
||||
peer.set_id(response_datagram.node_id)
|
||||
|
||||
# We got a result from the RPC
|
||||
if peer.node_id == self.node_id:
|
||||
df.set_exception(RemoteException("node has our node id"))
|
||||
@@ -460,6 +453,8 @@ class KademliaProtocol(DatagramProtocol):
|
||||
elif response_datagram.node_id == self.node_id:
|
||||
df.set_exception(RemoteException("incoming message is from our node id"))
|
||||
return
|
||||
peer.set_id(response_datagram.node_id)
|
||||
peer.update_udp_port(address[1])
|
||||
self.peer_manager.report_last_replied(address[0], address[1])
|
||||
self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
|
||||
if not df.cancelled():
|
||||
@@ -529,10 +524,16 @@ class KademliaProtocol(DatagramProtocol):
|
||||
response = await asyncio.wait_for(response_fut, self.rpc_timeout)
|
||||
self.peer_manager.report_last_replied(peer.address, peer.udp_port)
|
||||
return response
|
||||
except asyncio.CancelledError:
|
||||
if not response_fut.done():
|
||||
response_fut.cancel()
|
||||
raise
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
if self.peer_manager.peer_is_good(peer) is False:
|
||||
self.routing_table.remove_peer(peer)
|
||||
async with self._split_lock:
|
||||
self.routing_table.remove_peer(peer)
|
||||
self.routing_table.join_buckets()
|
||||
raise
|
||||
|
||||
async def send_response(self, peer: 'KademliaPeer', response: ResponseDatagram):
|
||||
@@ -543,7 +544,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
|
||||
async def _send(self, peer: 'KademliaPeer', message: typing.Union[RequestDatagram, ResponseDatagram,
|
||||
ErrorDatagram]):
|
||||
if not self.transport:
|
||||
if not self.transport or self.transport.is_closing():
|
||||
raise TransportNotConnected()
|
||||
|
||||
data = message.bencode()
|
||||
|
||||
@@ -29,6 +29,7 @@ class KBucket:
|
||||
self.range_max = range_max
|
||||
self.peers: typing.List['KademliaPeer'] = []
|
||||
self._node_id = node_id
|
||||
self._distance_to_self = Distance(node_id)
|
||||
|
||||
def add_peer(self, peer: 'KademliaPeer') -> bool:
|
||||
""" Add contact to _contact list in the right order. This will move the
|
||||
@@ -48,7 +49,14 @@ class KBucket:
|
||||
self.peers.remove(peer)
|
||||
self.peers.append(peer)
|
||||
return True
|
||||
elif len(self.peers) < constants.k:
|
||||
else:
|
||||
for i in range(len(self.peers)):
|
||||
p = self.peers[i]
|
||||
if p.node_id == peer.node_id:
|
||||
self.peers.remove(p)
|
||||
self.peers.append(peer)
|
||||
return True
|
||||
if len(self.peers) < constants.k:
|
||||
self.peers.append(peer)
|
||||
return True
|
||||
else:
|
||||
@@ -130,7 +138,7 @@ class KBucket:
|
||||
if not.
|
||||
@rtype: bool
|
||||
"""
|
||||
return self.range_min <= int.from_bytes(key, 'big') < self.range_max
|
||||
return self.range_min <= self._distance_to_self(key) < self.range_max
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.peers)
|
||||
@@ -155,10 +163,12 @@ class TreeRoutingTable:
|
||||
that paper.
|
||||
"""
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', parent_node_id: bytes):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', parent_node_id: bytes,
|
||||
split_buckets_under_index: int = constants.split_buckets_under_index):
|
||||
self._loop = loop
|
||||
self._peer_manager = peer_manager
|
||||
self._parent_node_id = parent_node_id
|
||||
self._split_buckets_under_index = split_buckets_under_index
|
||||
self.buckets: typing.List[KBucket] = [
|
||||
KBucket(
|
||||
self._peer_manager, range_min=0, range_max=2 ** constants.hash_bits, node_id=self._parent_node_id
|
||||
@@ -170,7 +180,7 @@ class TreeRoutingTable:
|
||||
|
||||
def should_split(self, bucket_index: int, to_add: bytes) -> bool:
|
||||
# https://stackoverflow.com/questions/32129978/highly-unbalanced-kademlia-routing-table/32187456#32187456
|
||||
if self.buckets[bucket_index].key_in_range(self._parent_node_id):
|
||||
if bucket_index < self._split_buckets_under_index:
|
||||
return True
|
||||
contacts = self.get_peers()
|
||||
distance = Distance(self._parent_node_id)
|
||||
@@ -236,11 +246,15 @@ class TreeRoutingTable:
|
||||
|
||||
def random_id_in_bucket_range(self, bucket_index: int) -> bytes:
|
||||
random_id = int(random.randrange(self.buckets[bucket_index].range_min, self.buckets[bucket_index].range_max))
|
||||
return random_id.to_bytes(constants.hash_length, 'big')
|
||||
return Distance(
|
||||
self._parent_node_id
|
||||
)(random_id.to_bytes(constants.hash_length, 'big')).to_bytes(constants.hash_length, 'big')
|
||||
|
||||
def midpoint_id_in_bucket_range(self, bucket_index: int) -> bytes:
|
||||
half = int((self.buckets[bucket_index].range_max - self.buckets[bucket_index].range_min) // 2)
|
||||
return int(self.buckets[bucket_index].range_min + half).to_bytes(constants.hash_length, 'big')
|
||||
return Distance(self._parent_node_id)(
|
||||
int(self.buckets[bucket_index].range_min + half).to_bytes(constants.hash_length, 'big')
|
||||
).to_bytes(constants.hash_length, 'big')
|
||||
|
||||
def split_bucket(self, old_bucket_index: int) -> None:
|
||||
""" Splits the specified k-bucket into two new buckets which together
|
||||
@@ -267,6 +281,8 @@ class TreeRoutingTable:
|
||||
old_bucket.remove_peer(contact)
|
||||
|
||||
def join_buckets(self):
|
||||
if len(self.buckets) == 1:
|
||||
return
|
||||
to_pop = [i for i, bucket in enumerate(self.buckets) if not len(bucket)]
|
||||
if not to_pop:
|
||||
return
|
||||
|
||||
@@ -65,46 +65,36 @@ class RequestDatagram(KademliaDatagramBase):
|
||||
|
||||
@classmethod
|
||||
def make_ping(cls, from_node_id: bytes, rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
|
||||
if rpc_id and len(rpc_id) != constants.rpc_id_length:
|
||||
raise ValueError("invalid rpc id length")
|
||||
elif not rpc_id:
|
||||
rpc_id = constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(from_node_id) != constants.hash_bits // 8:
|
||||
raise ValueError("invalid node id")
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
|
||||
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'ping')
|
||||
|
||||
@classmethod
|
||||
def make_store(cls, from_node_id: bytes, blob_hash: bytes, token: bytes, port: int,
|
||||
rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
|
||||
if rpc_id and len(rpc_id) != constants.rpc_id_length:
|
||||
raise ValueError("invalid rpc id length")
|
||||
if not rpc_id:
|
||||
rpc_id = constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(from_node_id) != constants.hash_bits // 8:
|
||||
raise ValueError("invalid node id")
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(blob_hash) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid blob hash length: {len(blob_hash)}")
|
||||
if not 0 < port < 65536:
|
||||
raise ValueError(f"invalid port: {port}")
|
||||
if len(token) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid token length: {len(token)}")
|
||||
store_args = [blob_hash, token, port, from_node_id, 0]
|
||||
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'store', store_args)
|
||||
|
||||
@classmethod
|
||||
def make_find_node(cls, from_node_id: bytes, key: bytes,
|
||||
rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
|
||||
if rpc_id and len(rpc_id) != constants.rpc_id_length:
|
||||
raise ValueError("invalid rpc id length")
|
||||
if not rpc_id:
|
||||
rpc_id = constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(from_node_id) != constants.hash_bits // 8:
|
||||
raise ValueError("invalid node id")
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(key) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid key length: {len(key)}")
|
||||
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'findNode', [key])
|
||||
|
||||
@classmethod
|
||||
def make_find_value(cls, from_node_id: bytes, key: bytes,
|
||||
rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
|
||||
if rpc_id and len(rpc_id) != constants.rpc_id_length:
|
||||
raise ValueError("invalid rpc id length")
|
||||
if not rpc_id:
|
||||
rpc_id = constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(from_node_id) != constants.hash_bits // 8:
|
||||
raise ValueError("invalid node id")
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(key) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid key length: {len(key)}")
|
||||
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'findValue', [key])
|
||||
|
||||
|
||||
@@ -148,8 +138,6 @@ def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDa
|
||||
}
|
||||
|
||||
primitive: typing.Dict = bdecode(datagram)
|
||||
if not isinstance(primitive, dict):
|
||||
raise ValueError("invalid datagram type")
|
||||
if primitive[0] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
|
||||
datagram_type = primitive[0] # pylint: disable=unsubscriptable-object
|
||||
else:
|
||||
@@ -163,14 +151,19 @@ def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDa
|
||||
)
|
||||
|
||||
|
||||
def make_compact_ip(address: str):
|
||||
return reduce(lambda buff, x: buff + bytearray([int(x)]), address.split('.'), bytearray())
|
||||
def make_compact_ip(address: str) -> bytearray:
|
||||
compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), address.split('.'), bytearray())
|
||||
if len(compact_ip) != 4:
|
||||
raise ValueError(f"invalid IPv4 length")
|
||||
return compact_ip
|
||||
|
||||
|
||||
def make_compact_address(node_id: bytes, address: str, port: int) -> bytearray:
|
||||
compact_ip = make_compact_ip(address)
|
||||
if not 0 <= port <= 65536:
|
||||
if not 0 < port < 65536:
|
||||
raise ValueError(f'Invalid port: {port}')
|
||||
if len(node_id) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid node node_id length")
|
||||
return compact_ip + port.to_bytes(2, 'big') + node_id
|
||||
|
||||
|
||||
@@ -178,4 +171,8 @@ def decode_compact_address(compact_address: bytes) -> typing.Tuple[bytes, str, i
|
||||
address = "{}.{}.{}.{}".format(*compact_address[:4])
|
||||
port = int.from_bytes(compact_address[4:6], 'big')
|
||||
node_id = compact_address[6:]
|
||||
if not 0 < port < 65536:
|
||||
raise ValueError(f'Invalid port: {port}')
|
||||
if len(node_id) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid node node_id length")
|
||||
return node_id, address, port
|
||||
+7
-2
@@ -28,11 +28,16 @@ class DownloadTimeoutError(Exception):
|
||||
|
||||
class DownloadDataTimeout(Exception):
|
||||
def __init__(self, download):
|
||||
super().__init__('Failed to download data blobs for sd hash '
|
||||
'{} within timeout'.format(download))
|
||||
super().__init__(f'Failed to download data blobs for sd hash {download} within timeout')
|
||||
self.download = download
|
||||
|
||||
|
||||
class ResolveTimeout(Exception):
|
||||
def __init__(self, uri):
|
||||
super().__init__(f'Failed to resolve "{uri}" within the timeout')
|
||||
self.uri = uri
|
||||
|
||||
|
||||
class RequestCanceledError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
+16
-6
@@ -1,5 +1,6 @@
|
||||
import os
|
||||
import sys
|
||||
import signal
|
||||
import pathlib
|
||||
import json
|
||||
import asyncio
|
||||
@@ -24,7 +25,7 @@ def display(data):
|
||||
print(json.dumps(data, indent=2))
|
||||
|
||||
|
||||
async def execute_command(conf, method, params):
|
||||
async def execute_command(conf, method, params, callback=display):
|
||||
async with aiohttp.ClientSession() as session:
|
||||
try:
|
||||
message = {'method': method, 'params': params}
|
||||
@@ -32,12 +33,9 @@ async def execute_command(conf, method, params):
|
||||
try:
|
||||
data = await resp.json()
|
||||
if 'result' in data:
|
||||
display(data['result'])
|
||||
return callback(data['result'])
|
||||
elif 'error' in data:
|
||||
if 'message' in data['error']:
|
||||
display(data['error']['message'])
|
||||
else:
|
||||
display(data['error'])
|
||||
return callback(data['error'])
|
||||
except Exception as e:
|
||||
log.exception('Could not process response from server:', exc_info=e)
|
||||
except aiohttp.ClientConnectionError:
|
||||
@@ -272,8 +270,20 @@ def main(argv=None):
|
||||
log.addHandler(loggly_handler)
|
||||
|
||||
daemon = Daemon(conf)
|
||||
started = False
|
||||
def __exit():
|
||||
if started:
|
||||
daemon.stop_event.set()
|
||||
else:
|
||||
raise GracefulExit()
|
||||
try:
|
||||
loop.add_signal_handler(signal.SIGINT, __exit)
|
||||
loop.add_signal_handler(signal.SIGTERM, __exit)
|
||||
except NotImplementedError:
|
||||
pass # Not implemented on Windows
|
||||
try:
|
||||
loop.run_until_complete(daemon.start())
|
||||
started = True
|
||||
loop.run_until_complete(daemon.stop_event.wait())
|
||||
except (GracefulExit, KeyboardInterrupt):
|
||||
pass
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import os
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import logging
|
||||
import math
|
||||
import binascii
|
||||
@@ -13,19 +12,18 @@ from aioupnp import __version__ as aioupnp_version
|
||||
from aioupnp.upnp import UPnP
|
||||
from aioupnp.fault import UPnPError
|
||||
|
||||
import lbrynet.schema
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import HEADERS_FILE_SHA256_CHECKSUM
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.blob_announcer import BlobAnnouncer
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
from lbrynet.blob_exchange.server import BlobServer
|
||||
from lbrynet.stream.stream_manager import StreamManager
|
||||
from lbrynet.extras.daemon.Component import Component
|
||||
from lbrynet.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.extras.wallet import LbryWalletManager
|
||||
from lbrynet.extras.wallet import Network
|
||||
from lbrynet.wallet import LbryWalletManager
|
||||
from lbrynet.wallet import Network
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -52,17 +50,6 @@ async def gather_dict(tasks: dict):
|
||||
)))
|
||||
|
||||
|
||||
async def get_external_ip(): # used if upnp is disabled or non-functioning
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get("https://api.lbry.io/ip") as resp:
|
||||
response = await resp.json()
|
||||
if response['success']:
|
||||
return response['data']['ip']
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
|
||||
class DatabaseComponent(Component):
|
||||
component_name = DATABASE_COMPONENT
|
||||
|
||||
@@ -76,7 +63,7 @@ class DatabaseComponent(Component):
|
||||
|
||||
@staticmethod
|
||||
def get_current_db_revision():
|
||||
return 9
|
||||
return 11
|
||||
|
||||
@property
|
||||
def revision_filename(self):
|
||||
@@ -149,7 +136,7 @@ class HeadersComponent(Component):
|
||||
async def fetch_headers_from_s3(self):
|
||||
local_header_size = self.local_header_file_size()
|
||||
resume_header = {"Range": f"bytes={local_header_size}-"}
|
||||
async with aiohttp.request('get', HEADERS_URL, headers=resume_header) as response:
|
||||
async with utils.aiohttp_request('get', HEADERS_URL, headers=resume_header) as response:
|
||||
if response.status == 406 or response.content_length < HEADER_SIZE: # our file is bigger
|
||||
log.warning("s3 is more out of date than we are")
|
||||
return
|
||||
@@ -264,23 +251,20 @@ class WalletComponent(Component):
|
||||
|
||||
async def get_status(self):
|
||||
if self.wallet_manager and self.running:
|
||||
local_height = self.wallet_manager.network.get_local_height()
|
||||
remote_height = self.wallet_manager.network.get_server_height()
|
||||
local_height = self.wallet_manager.ledger.headers.height
|
||||
remote_height = await self.wallet_manager.ledger.network.get_server_height()
|
||||
best_hash = self.wallet_manager.get_best_blockhash()
|
||||
return {
|
||||
'blocks': max(local_height, 0),
|
||||
'blocks_behind': max(remote_height - local_height, 0),
|
||||
'best_blockhash': best_hash,
|
||||
'is_encrypted': self.wallet_manager.wallet.use_encryption,
|
||||
'is_encrypted': self.wallet_manager.use_encryption,
|
||||
'is_locked': not self.wallet_manager.is_wallet_unlocked,
|
||||
}
|
||||
|
||||
async def start(self):
|
||||
log.info("Starting torba wallet")
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
lbrynet.schema.BLOCKCHAIN_NAME = self.conf.blockchain_name
|
||||
self.wallet_manager = await LbryWalletManager.from_lbrynet_config(self.conf, storage)
|
||||
self.wallet_manager.old_db = storage
|
||||
self.wallet_manager = await LbryWalletManager.from_lbrynet_config(self.conf)
|
||||
await self.wallet_manager.start()
|
||||
|
||||
async def stop(self):
|
||||
@@ -294,10 +278,10 @@ class BlobComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.blob_manager: BlobFileManager = None
|
||||
self.blob_manager: BlobManager = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[BlobFileManager]:
|
||||
def component(self) -> typing.Optional[BlobManager]:
|
||||
return self.blob_manager
|
||||
|
||||
async def start(self):
|
||||
@@ -307,8 +291,10 @@ class BlobComponent(Component):
|
||||
dht_node: Node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
if dht_node:
|
||||
data_store = dht_node.protocol.data_store
|
||||
self.blob_manager = BlobFileManager(asyncio.get_event_loop(), os.path.join(self.conf.data_dir, "blobfiles"),
|
||||
storage, data_store)
|
||||
blob_dir = os.path.join(self.conf.data_dir, 'blobfiles')
|
||||
if not os.path.isdir(blob_dir):
|
||||
os.mkdir(blob_dir)
|
||||
self.blob_manager = BlobManager(asyncio.get_event_loop(), blob_dir, storage, self.conf, data_store)
|
||||
return await self.blob_manager.setup()
|
||||
|
||||
async def stop(self):
|
||||
@@ -360,7 +346,7 @@ class DHTComponent(Component):
|
||||
external_ip = self.upnp_component.external_ip
|
||||
if not external_ip:
|
||||
log.warning("UPnP component failed to get external ip")
|
||||
external_ip = await get_external_ip()
|
||||
external_ip = await utils.get_external_ip()
|
||||
if not external_ip:
|
||||
log.warning("failed to get external ip")
|
||||
|
||||
@@ -372,7 +358,8 @@ class DHTComponent(Component):
|
||||
udp_port=self.external_udp_port,
|
||||
external_ip=external_ip,
|
||||
peer_port=self.external_peer_port,
|
||||
rpc_timeout=self.conf.node_rpc_timeout
|
||||
rpc_timeout=self.conf.node_rpc_timeout,
|
||||
split_buckets_under_index=self.conf.split_buckets_under_index
|
||||
)
|
||||
self.dht_node.start(
|
||||
interface=self.conf.network_interface, known_node_urls=self.conf.known_dht_nodes
|
||||
@@ -442,7 +429,7 @@ class StreamManagerComponent(Component):
|
||||
log.info('Starting the file manager')
|
||||
loop = asyncio.get_event_loop()
|
||||
self.stream_manager = StreamManager(
|
||||
loop, self.conf, blob_manager, wallet, storage, node,
|
||||
loop, self.conf, blob_manager, wallet, storage, node, self.component_manager.analytics_manager
|
||||
)
|
||||
await self.stream_manager.start()
|
||||
log.info('Done setting up file manager')
|
||||
@@ -466,7 +453,7 @@ class PeerProtocolServerComponent(Component):
|
||||
async def start(self):
|
||||
log.info("start blob server")
|
||||
upnp = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
blob_manager: BlobFileManager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
blob_manager: BlobManager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
wallet: LbryWalletManager = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
peer_port = self.conf.tcp_port
|
||||
address = await wallet.get_unused_address()
|
||||
@@ -500,7 +487,7 @@ class UPnPComponent(Component):
|
||||
while True:
|
||||
if now:
|
||||
await self._maintain_redirects()
|
||||
await asyncio.sleep(360)
|
||||
await asyncio.sleep(360, loop=self.component_manager.loop)
|
||||
|
||||
async def _maintain_redirects(self):
|
||||
# setup the gateway if necessary
|
||||
@@ -524,7 +511,7 @@ class UPnPComponent(Component):
|
||||
|
||||
if external_ip == "0.0.0.0" or not external_ip:
|
||||
log.warning("unable to get external ip from UPnP, checking lbry.io fallback")
|
||||
external_ip = await get_external_ip()
|
||||
external_ip = await utils.get_external_ip()
|
||||
if self.external_ip and self.external_ip != external_ip:
|
||||
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
|
||||
self.external_ip = external_ip
|
||||
@@ -543,7 +530,7 @@ class UPnPComponent(Component):
|
||||
self.upnp_redirects.update(upnp_redirects)
|
||||
except (asyncio.TimeoutError, UPnPError):
|
||||
self.upnp = None
|
||||
return self._maintain_redirects()
|
||||
return
|
||||
elif self.upnp: # check existing redirects are still active
|
||||
found = set()
|
||||
mappings = await self.upnp.get_redirects()
|
||||
@@ -575,7 +562,7 @@ class UPnPComponent(Component):
|
||||
async def start(self):
|
||||
log.info("detecting external ip")
|
||||
if not self.use_upnp:
|
||||
self.external_ip = await get_external_ip()
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
return
|
||||
success = False
|
||||
await self._maintain_redirects()
|
||||
|
||||
+1642
-1044
File diff suppressed because it is too large.
Load diff
+110
-121
@@ -1,9 +1,8 @@
|
||||
import asyncio
|
||||
import collections
|
||||
import logging
|
||||
|
||||
import aiohttp
|
||||
|
||||
import typing
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras import system_info
|
||||
@@ -22,102 +21,137 @@ HEARTBEAT = 'Heartbeat'
|
||||
CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon
|
||||
NEW_CHANNEL = 'New Channel'
|
||||
CREDITS_SENT = 'Credits Sent'
|
||||
NEW_DOWNLOAD_STAT = 'Download'
|
||||
UPNP_SETUP = "UPnP Setup"
|
||||
|
||||
BLOB_BYTES_UPLOADED = 'Blob Bytes Uploaded'
|
||||
|
||||
|
||||
TIME_TO_FIRST_BYTES = "Time To First Bytes"
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Manager:
|
||||
def _event_properties(installation_id: str, session_id: str,
|
||||
event_properties: typing.Optional[typing.Dict]) -> typing.Dict:
|
||||
properties = {
|
||||
'lbry_id': installation_id,
|
||||
'session_id': session_id,
|
||||
}
|
||||
properties.update(event_properties or {})
|
||||
return properties
|
||||
|
||||
|
||||
def _download_properties(conf: Config, external_ip: str, resolve_duration: float,
|
||||
total_duration: typing.Optional[float], download_id: str, name: str,
|
||||
outpoint: str, active_peer_count: int, tried_peers_count: int,
|
||||
added_fixed_peers: bool, fixed_peer_delay: float, sd_hash: str,
|
||||
sd_download_duration: typing.Optional[float] = None,
|
||||
head_blob_hash: typing.Optional[str] = None,
|
||||
head_blob_length: typing.Optional[int] = None,
|
||||
head_blob_download_duration: typing.Optional[float] = None,
|
||||
error: typing.Optional[str] = None) -> typing.Dict:
|
||||
return {
|
||||
"external_ip": external_ip,
|
||||
"download_id": download_id,
|
||||
"total_duration": round(total_duration, 4),
|
||||
"resolve_duration": None if not resolve_duration else round(resolve_duration, 4),
|
||||
"error": error,
|
||||
'name': name,
|
||||
"outpoint": outpoint,
|
||||
|
||||
"node_rpc_timeout": conf.node_rpc_timeout,
|
||||
"peer_connect_timeout": conf.peer_connect_timeout,
|
||||
"blob_download_timeout": conf.blob_download_timeout,
|
||||
"use_fixed_peers": len(conf.reflector_servers) > 0,
|
||||
"fixed_peer_delay": fixed_peer_delay,
|
||||
"added_fixed_peers": added_fixed_peers,
|
||||
"active_peer_count": active_peer_count,
|
||||
"tried_peers_count": tried_peers_count,
|
||||
|
||||
"sd_blob_hash": sd_hash,
|
||||
"sd_blob_duration": None if not sd_download_duration else round(sd_download_duration, 4),
|
||||
|
||||
"head_blob_hash": head_blob_hash,
|
||||
"head_blob_length": head_blob_length,
|
||||
"head_blob_duration": None if not head_blob_download_duration else round(head_blob_download_duration, 4)
|
||||
}
|
||||
|
||||
|
||||
def _make_context(platform):
|
||||
# see https://segment.com/docs/spec/common/#context
|
||||
# they say they'll ignore fields outside the spec, but evidently they don't
|
||||
context = {
|
||||
'app': {
|
||||
'version': platform['lbrynet_version'],
|
||||
'build': platform['build'],
|
||||
},
|
||||
# TODO: expand os info to give linux/osx specific info
|
||||
'os': {
|
||||
'name': platform['os_system'],
|
||||
'version': platform['os_release']
|
||||
},
|
||||
}
|
||||
if 'desktop' in platform and 'distro' in platform:
|
||||
context['os']['desktop'] = platform['desktop']
|
||||
context['os']['distro'] = platform['distro']
|
||||
return context
|
||||
|
||||
|
||||
class AnalyticsManager:
|
||||
|
||||
def __init__(self, conf: Config, installation_id: str, session_id: str):
|
||||
self.conf = conf
|
||||
self.cookies = {}
|
||||
self.url = ANALYTICS_ENDPOINT
|
||||
self._write_key = utils.deobfuscate(ANALYTICS_TOKEN)
|
||||
self._enabled = conf.share_usage_data
|
||||
self._tracked_data = collections.defaultdict(list)
|
||||
self.context = self._make_context(system_info.get_platform(), 'torba')
|
||||
self.context = _make_context(system_info.get_platform())
|
||||
self.installation_id = installation_id
|
||||
self.session_id = session_id
|
||||
self.task: asyncio.Task = None
|
||||
self.external_ip: typing.Optional[str] = None
|
||||
|
||||
@property
|
||||
def is_started(self):
|
||||
return self.task is not None
|
||||
|
||||
def start(self):
|
||||
async def start(self):
|
||||
if self._enabled and self.task is None:
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
self.task = asyncio.create_task(self.run())
|
||||
|
||||
async def run(self):
|
||||
while True:
|
||||
await self._send_heartbeat()
|
||||
await asyncio.sleep(1800)
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
|
||||
def stop(self):
|
||||
if self.task is not None and not self.task.done():
|
||||
self.task.cancel()
|
||||
|
||||
async def _post(self, endpoint, data):
|
||||
# there is an issue with a timing condition with keep-alive
|
||||
# that is best explained here: https://github.com/mikem23/keepalive-race
|
||||
#
|
||||
# If you make a request, wait just the right amount of time,
|
||||
# then make another request, the requests module may opt to
|
||||
# reuse the connection, but by the time the server gets it the
|
||||
# timeout will have expired.
|
||||
#
|
||||
# by forcing the connection to close, we will disable the keep-alive.
|
||||
|
||||
assert endpoint[0] == '/'
|
||||
async def _post(self, data: typing.Dict):
|
||||
request_kwargs = {
|
||||
'method': 'POST',
|
||||
'url': self.url + endpoint,
|
||||
'url': self.url + '/track',
|
||||
'headers': {'Connection': 'Close'},
|
||||
'auth': aiohttp.BasicAuth(self._write_key, ''),
|
||||
'json': data,
|
||||
'cookies': self.cookies
|
||||
}
|
||||
try:
|
||||
async with aiohttp.request(**request_kwargs) as response:
|
||||
async with utils.aiohttp_request(**request_kwargs) as response:
|
||||
self.cookies.update(response.cookies)
|
||||
except Exception as e:
|
||||
log.exception('Encountered an exception while POSTing to %s: ', self.url + endpoint, exc_info=e)
|
||||
log.debug('Encountered an exception while POSTing to %s: ', self.url + '/track', exc_info=e)
|
||||
|
||||
async def track(self, event):
|
||||
async def track(self, event: typing.Dict):
|
||||
"""Send a single tracking event"""
|
||||
if self._enabled:
|
||||
log.debug('Sending track event: %s', event)
|
||||
await self._post('/track', event)
|
||||
|
||||
async def send_new_download_start(self, download_id, name, claim_dict):
|
||||
await self._send_new_download_stats("start", download_id, name, claim_dict)
|
||||
|
||||
async def send_new_download_success(self, download_id, name, claim_dict):
|
||||
await self._send_new_download_stats("success", download_id, name, claim_dict)
|
||||
|
||||
async def send_new_download_fail(self, download_id, name, claim_dict, e):
|
||||
await self._send_new_download_stats("failure", download_id, name, claim_dict, {
|
||||
'name': type(e).__name__ if hasattr(type(e), "__name__") else str(type(e)),
|
||||
'message': str(e),
|
||||
})
|
||||
|
||||
async def _send_new_download_stats(self, action, download_id, name, claim_dict, e=None):
|
||||
await self.track({
|
||||
'userId': 'lbry', # required, see https://segment.com/docs/sources/server/http/#track
|
||||
'event': NEW_DOWNLOAD_STAT,
|
||||
'properties': self._event_properties({
|
||||
'download_id': download_id,
|
||||
'name': name,
|
||||
'sd_hash': None if not claim_dict else claim_dict.source_hash.decode(),
|
||||
'action': action,
|
||||
'error': e,
|
||||
}),
|
||||
'context': self.context,
|
||||
'timestamp': utils.isonow(),
|
||||
})
|
||||
await self._post(event)
|
||||
|
||||
async def send_upnp_setup_success_fail(self, success, status):
|
||||
await self.track(
|
||||
@@ -136,20 +170,33 @@ class Manager:
|
||||
async def send_server_startup_error(self, message):
|
||||
await self.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
|
||||
|
||||
async def send_download_started(self, id_, name, claim_dict=None):
|
||||
async def send_time_to_first_bytes(self, resolve_duration: typing.Optional[float],
|
||||
total_duration: typing.Optional[float], download_id: str,
|
||||
name: str, outpoint: str, found_peers_count: int,
|
||||
tried_peers_count: int, added_fixed_peers: bool,
|
||||
fixed_peers_delay: float, sd_hash: str,
|
||||
sd_download_duration: typing.Optional[float] = None,
|
||||
head_blob_hash: typing.Optional[str] = None,
|
||||
head_blob_length: typing.Optional[int] = None,
|
||||
head_blob_duration: typing.Optional[int] = None,
|
||||
error: typing.Optional[str] = None):
|
||||
await self.track(self._event(TIME_TO_FIRST_BYTES, _download_properties(
|
||||
self.conf, self.external_ip, resolve_duration, total_duration, download_id, name, outpoint,
|
||||
found_peers_count, tried_peers_count, added_fixed_peers, fixed_peers_delay, sd_hash,
|
||||
sd_download_duration, head_blob_hash, head_blob_length, head_blob_duration, error
|
||||
)))
|
||||
|
||||
async def send_download_finished(self, download_id, name, sd_hash):
|
||||
await self.track(
|
||||
self._event(DOWNLOAD_STARTED, self._download_properties(id_, name, claim_dict))
|
||||
self._event(
|
||||
DOWNLOAD_FINISHED, {
|
||||
'download_id': download_id,
|
||||
'name': name,
|
||||
'stream_info': sd_hash
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
async def send_download_errored(self, err, id_, name, claim_dict, report):
|
||||
download_error_properties = self._download_error_properties(err, id_, name, claim_dict,
|
||||
report)
|
||||
await self.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
|
||||
|
||||
async def send_download_finished(self, id_, name, report, claim_dict=None):
|
||||
download_properties = self._download_properties(id_, name, claim_dict, report)
|
||||
await self.track(self._event(DOWNLOAD_FINISHED, download_properties))
|
||||
|
||||
async def send_claim_action(self, action):
|
||||
await self.track(self._event(CLAIM_ACTION, {'action': action}))
|
||||
|
||||
@@ -162,69 +209,11 @@ class Manager:
|
||||
async def _send_heartbeat(self):
|
||||
await self.track(self._event(HEARTBEAT))
|
||||
|
||||
def _event(self, event, event_properties=None):
|
||||
def _event(self, event, properties: typing.Optional[typing.Dict] = None):
|
||||
return {
|
||||
'userId': 'lbry',
|
||||
'event': event,
|
||||
'properties': self._event_properties(event_properties),
|
||||
'properties': _event_properties(self.installation_id, self.session_id, properties),
|
||||
'context': self.context,
|
||||
'timestamp': utils.isonow()
|
||||
}
|
||||
|
||||
def _metric_event(self, metric_name, value):
|
||||
return self._event(metric_name, {'value': value})
|
||||
|
||||
def _event_properties(self, event_properties=None):
|
||||
properties = {
|
||||
'lbry_id': self.installation_id,
|
||||
'session_id': self.session_id,
|
||||
}
|
||||
properties.update(event_properties or {})
|
||||
return properties
|
||||
|
||||
@staticmethod
|
||||
def _download_properties(id_, name, claim_dict=None, report=None):
|
||||
sd_hash = None if not claim_dict else claim_dict.source_hash.decode()
|
||||
p = {
|
||||
'download_id': id_,
|
||||
'name': name,
|
||||
'stream_info': sd_hash
|
||||
}
|
||||
if report:
|
||||
p['report'] = report
|
||||
return p
|
||||
|
||||
@staticmethod
|
||||
def _download_error_properties(error, id_, name, claim_dict, report):
|
||||
def error_name(err):
|
||||
if not hasattr(type(err), "__name__"):
|
||||
return str(type(err))
|
||||
return type(err).__name__
|
||||
return {
|
||||
'download_id': id_,
|
||||
'name': name,
|
||||
'stream_info': claim_dict.source_hash.decode(),
|
||||
'error': error_name(error),
|
||||
'reason': str(error),
|
||||
'report': report
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _make_context(platform, wallet):
|
||||
# see https://segment.com/docs/spec/common/#context
|
||||
# they say they'll ignore fields outside the spec, but evidently they don't
|
||||
context = {
|
||||
'app': {
|
||||
'version': platform['lbrynet_version'],
|
||||
'build': platform['build'],
|
||||
},
|
||||
# TODO: expand os info to give linux/osx specific info
|
||||
'os': {
|
||||
'name': platform['os_system'],
|
||||
'version': platform['os_release']
|
||||
},
|
||||
}
|
||||
if 'desktop' in platform and 'distro' in platform:
|
||||
context['os']['desktop'] = platform['desktop']
|
||||
context['os']['distro'] = platform['distro']
|
||||
return context
|
||||
@@ -1,27 +1,6 @@
|
||||
import aiohttp
|
||||
import logging
|
||||
from lbrynet.conf import Config
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
USER_AGENT = "AuthServiceProxy/0.1"
|
||||
TWISTED_SECURE_SESSION = "TWISTED_SECURE_SESSION"
|
||||
TWISTED_SESSION = "TWISTED_SESSION"
|
||||
LBRY_SECRET = "LBRY_SECRET"
|
||||
HTTP_TIMEOUT = 30
|
||||
from lbrynet.extras.cli import execute_command
|
||||
|
||||
|
||||
class JSONRPCException(Exception):
|
||||
def __init__(self, rpc_error):
|
||||
super().__init__()
|
||||
self.error = rpc_error
|
||||
|
||||
|
||||
async def daemon_rpc(conf: Config, method: str, *args, **kwargs):
|
||||
async with aiohttp.ClientSession() as session:
|
||||
message = {'method': method, 'params': [args, kwargs]}
|
||||
async with session.get(conf.api_connection_url, json=message) as resp:
|
||||
data = await resp.json()
|
||||
if 'result' in data:
|
||||
return data['result']
|
||||
elif 'error' in data:
|
||||
raise JSONRPCException(data['error'])
|
||||
def daemon_rpc(conf: Config, method: str, **kwargs):
|
||||
return execute_command(conf, method, kwargs, callback=lambda data: data)
|
||||
@@ -0,0 +1,31 @@
|
||||
import logging
|
||||
import aiohttp
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def rpc_body(method: str, rpc_id: any, **params) -> dict:
|
||||
return {'jsonrpc': '2.0', 'id': rpc_id, 'method': method, 'params': {**params}}
|
||||
|
||||
|
||||
async def jsonrpc_post(url: str, method: str, **params) -> any:
|
||||
clean = params.pop('clean', True)
|
||||
response = (await jsonrpc_batch(url, [rpc_body(method, 1, **params)]))[0]
|
||||
if clean:
|
||||
if 'error' in response:
|
||||
return response['error']
|
||||
return response['result']
|
||||
else:
|
||||
return response
|
||||
|
||||
|
||||
async def jsonrpc_batch(url: str, calls: list, batch_size: int = 50, clean: bool = False) -> list:
|
||||
headers = {'Content-Type': 'application/json'}
|
||||
complete = []
|
||||
batch_size = max(batch_size, 50)
|
||||
for i in range(0, len(calls), batch_size):
|
||||
async with aiohttp.request('POST', url, headers=headers, json=calls[i:i+batch_size]) as response:
|
||||
complete += await response.json()
|
||||
if clean:
|
||||
complete = [body['result'] if 'result' in body else None for body in complete]
|
||||
return complete
|
||||
@@ -1,10 +1,11 @@
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import time
|
||||
import logging
|
||||
import json
|
||||
from decimal import Decimal
|
||||
|
||||
from lbrynet.error import InvalidExchangeRateResponse, CurrencyConversionError
|
||||
from lbrynet.utils import aiohttp_request
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -52,8 +53,8 @@ class MarketFeed:
|
||||
def is_online(self):
|
||||
return self._online
|
||||
|
||||
async def _make_request(self):
|
||||
async with aiohttp.request('get', self.url, params=self.params) as response:
|
||||
async def _make_request(self) -> str:
|
||||
async with aiohttp_request('get', self.url, params=self.params) as response:
|
||||
return (await response.read()).decode()
|
||||
|
||||
def _handle_response(self, response):
|
||||
@@ -226,12 +227,12 @@ class ExchangeRateManager:
|
||||
for market in self.market_feeds:
|
||||
if (market.rate_is_initialized() and market.is_online() and
|
||||
market.rate.currency_pair == (from_currency, to_currency)):
|
||||
return amount * market.rate.spot
|
||||
return amount * Decimal(market.rate.spot)
|
||||
for market in self.market_feeds:
|
||||
if (market.rate_is_initialized() and market.is_online() and
|
||||
market.rate.currency_pair[0] == from_currency):
|
||||
return self.convert_currency(
|
||||
market.rate.currency_pair[1], to_currency, amount * market.rate.spot)
|
||||
market.rate.currency_pair[1], to_currency, amount * Decimal(market.rate.spot))
|
||||
raise CurrencyConversionError(
|
||||
f'Unable to convert {amount} from {from_currency} to {to_currency}')
|
||||
|
||||
|
||||
@@ -4,25 +4,117 @@ from binascii import hexlify
|
||||
from datetime import datetime
|
||||
from json import JSONEncoder
|
||||
from ecdsa import BadSignatureError
|
||||
from lbrynet.extras.wallet import MainNetLedger
|
||||
from lbrynet.extras.wallet.transaction import Transaction, Output
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc
|
||||
from lbrynet.schema.claim import Claim
|
||||
from lbrynet.wallet.ledger import MainNetLedger, Account
|
||||
from lbrynet.wallet.transaction import Transaction, Output
|
||||
from lbrynet.wallet.dewies import dewies_to_lbc
|
||||
from lbrynet.stream.managed_stream import ManagedStream
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def encode_txo_doc():
|
||||
return {
|
||||
'txid': "hash of transaction in hex",
|
||||
'nout': "position in the transaction",
|
||||
'height': "block where transaction was recorded",
|
||||
'amount': "value of the txo as a decimal",
|
||||
'address': "address of who can spend the txo",
|
||||
'confirmations': "number of confirmed blocks",
|
||||
'is_change': "payment to change address, only available when it can be determined",
|
||||
'is_mine': "payment to one of your accounts, only available when it can be determined",
|
||||
'type': "one of 'claim', 'support' or 'payment'",
|
||||
'name': "when type is 'claim' or 'support', this is the claim name",
|
||||
'claim_id': "when type is 'claim' or 'support', this is the claim id",
|
||||
'claim_op': "when type is 'claim', this determines if it is 'create' or 'update'",
|
||||
'value': "when type is 'claim' or 'support' with payload, this is the decoded protobuf payload",
|
||||
'value_type': "determines the type of the 'value' field: 'channel', 'stream', etc",
|
||||
'protobuf': "hex encoded raw protobuf version of 'value' field",
|
||||
'permanent_url': "when type is 'claim' or 'support', this is the long permanent claim URL",
|
||||
'signing_channel': "for signed claims only, metadata of signing channel",
|
||||
'is_channel_signature_valid': "for signed claims only, whether signature is valid",
|
||||
}
|
||||
|
||||
|
||||
def encode_tx_doc():
|
||||
return {
|
||||
'txid': "hash of transaction in hex",
|
||||
'height': "block where transaction was recorded",
|
||||
'inputs': [encode_txo_doc()],
|
||||
'outputs': [encode_txo_doc()],
|
||||
'total_input': "sum of inputs as a decimal",
|
||||
'total_output': "sum of outputs, sans fee, as a decimal",
|
||||
'total_fee': "fee amount",
|
||||
'hex': "entire transaction encoded in hex",
|
||||
}
|
||||
|
||||
|
||||
def encode_account_doc():
|
||||
return {
|
||||
'id': 'account_id',
|
||||
'is_default': 'this account is used by default',
|
||||
'ledger': 'name of crypto currency and network',
|
||||
'name': 'optional account name',
|
||||
'seed': 'human friendly words from which account can be recreated',
|
||||
'encrypted': 'if account is encrypted',
|
||||
'private_key': 'extended private key',
|
||||
'public_key': 'extended public key',
|
||||
'address_generator': 'settings for generating addresses',
|
||||
'modified_on': 'date of last modification to account settings'
|
||||
}
|
||||
|
||||
|
||||
def encode_file_doc():
|
||||
return {
|
||||
'completed': '(bool) true if download is completed',
|
||||
'file_name': '(str) name of file',
|
||||
'download_directory': '(str) download directory',
|
||||
'points_paid': '(float) credit paid to download file',
|
||||
'stopped': '(bool) true if download is stopped',
|
||||
'stream_hash': '(str) stream hash of file',
|
||||
'stream_name': '(str) stream name',
|
||||
'suggested_file_name': '(str) suggested file name',
|
||||
'sd_hash': '(str) sd hash of file',
|
||||
'download_path': '(str) download path of file',
|
||||
'mime_type': '(str) mime type of file',
|
||||
'key': '(str) key attached to file',
|
||||
'total_bytes_lower_bound': '(int) lower bound file size in bytes',
|
||||
'total_bytes': '(int) file upper bound size in bytes',
|
||||
'written_bytes': '(int) written size in bytes',
|
||||
'blobs_completed': '(int) number of fully downloaded blobs',
|
||||
'blobs_in_stream': '(int) total blobs on stream',
|
||||
'blobs_remaining': '(int) total blobs remaining to download',
|
||||
'status': '(str) downloader status',
|
||||
'claim_id': '(str) None if claim is not found else the claim id',
|
||||
'txid': '(str) None if claim is not found else the transaction id',
|
||||
'nout': '(int) None if claim is not found else the transaction output index',
|
||||
'outpoint': '(str) None if claim is not found else the tx and output',
|
||||
'metadata': '(dict) None if claim is not found else the claim metadata',
|
||||
'channel_claim_id': '(str) None if claim is not found or not signed',
|
||||
'channel_name': '(str) None if claim is not found or not signed',
|
||||
'claim_name': '(str) None if claim is not found else the claim name'
|
||||
}
|
||||
|
||||
|
||||
class JSONResponseEncoder(JSONEncoder):
|
||||
|
||||
def __init__(self, *args, ledger: MainNetLedger, **kwargs):
|
||||
def __init__(self, *args, ledger: MainNetLedger, include_protobuf=False, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.ledger = ledger
|
||||
self.include_protobuf = include_protobuf
|
||||
|
||||
def default(self, obj): # pylint: disable=method-hidden
|
||||
if isinstance(obj, Account):
|
||||
return self.encode_account(obj)
|
||||
if isinstance(obj, ManagedStream):
|
||||
return self.encode_file(obj)
|
||||
if isinstance(obj, Transaction):
|
||||
return self.encode_transaction(obj)
|
||||
if isinstance(obj, Output):
|
||||
return self.encode_output(obj)
|
||||
if isinstance(obj, Claim):
|
||||
return self.encode_claim(obj)
|
||||
if isinstance(obj, datetime):
|
||||
return obj.strftime("%Y%m%dT%H:%M:%S")
|
||||
if isinstance(obj, Decimal):
|
||||
@@ -43,62 +135,88 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
'hex': hexlify(tx.raw).decode(),
|
||||
}
|
||||
|
||||
def encode_output(self, txo):
|
||||
def encode_output(self, txo, check_signature=True):
|
||||
tx_height = txo.tx_ref.height
|
||||
best_height = self.ledger.headers.height
|
||||
output = {
|
||||
'txid': txo.tx_ref.id,
|
||||
'nout': txo.position,
|
||||
'height': tx_height,
|
||||
'amount': dewies_to_lbc(txo.amount),
|
||||
'address': txo.get_address(self.ledger),
|
||||
'height': tx_height,
|
||||
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height
|
||||
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
|
||||
'timestamp': self.ledger.headers[tx_height]['timestamp'] if tx_height > 0 else None
|
||||
}
|
||||
if txo.is_change is not None:
|
||||
output['is_change'] = txo.is_change
|
||||
if txo.is_my_account is not None:
|
||||
output['is_mine'] = txo.is_my_account
|
||||
|
||||
if txo.script.is_claim_name:
|
||||
output['type'] = 'claim'
|
||||
output['claim_op'] = 'create'
|
||||
elif txo.script.is_update_claim:
|
||||
output['type'] = 'claim'
|
||||
output['claim_op'] = 'update'
|
||||
elif txo.script.is_support_claim:
|
||||
output['type'] = 'support'
|
||||
else:
|
||||
output['type'] = 'payment'
|
||||
|
||||
if txo.script.is_claim_involved:
|
||||
output.update({
|
||||
'name': txo.claim_name,
|
||||
'claim_id': txo.claim_id,
|
||||
'permanent_url': txo.permanent_url,
|
||||
})
|
||||
|
||||
if txo.script.is_claim_name or txo.script.is_update_claim:
|
||||
claim = txo.claim
|
||||
output['value'] = claim.claim_dict
|
||||
if claim.has_signature:
|
||||
output['valid_signature'] = None
|
||||
if txo.channel is not None:
|
||||
output['channel_name'] = txo.channel.claim_name
|
||||
output['value'] = txo.claim
|
||||
output['value_type'] = txo.claim.claim_type
|
||||
if self.include_protobuf:
|
||||
output['protobuf'] = hexlify(txo.claim.to_bytes())
|
||||
if txo.channel is not None:
|
||||
output['signing_channel'] = {
|
||||
'name': txo.channel.claim_name,
|
||||
'claim_id': txo.channel.claim_id,
|
||||
'value': txo.channel.claim
|
||||
}
|
||||
if check_signature and txo.claim.is_signed:
|
||||
output['is_channel_signature_valid'] = False
|
||||
try:
|
||||
output['valid_signature'] = claim.validate_signature(
|
||||
txo.get_address(self.ledger), txo.channel.claim, name=txo.claim_name
|
||||
)
|
||||
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
|
||||
except BadSignatureError:
|
||||
output['valid_signature'] = False
|
||||
pass
|
||||
except ValueError:
|
||||
log.exception(
|
||||
'txo.id: %s, txo.channel.id:%s, output: %s',
|
||||
txo.id, txo.channel.id, output
|
||||
)
|
||||
output['valid_signature'] = False
|
||||
|
||||
if txo.script.is_claim_name:
|
||||
output['type'] = 'claim'
|
||||
elif txo.script.is_update_claim:
|
||||
output['type'] = 'update'
|
||||
elif txo.script.is_support_claim:
|
||||
output['type'] = 'support'
|
||||
else:
|
||||
output['type'] = 'basic'
|
||||
|
||||
return output
|
||||
|
||||
def encode_input(self, txi):
|
||||
return self.encode_output(txi.txo_ref.txo) if txi.txo_ref.txo is not None else {
|
||||
return self.encode_output(txi.txo_ref.txo, False) if txi.txo_ref.txo is not None else {
|
||||
'txid': txi.txo_ref.tx_ref.id,
|
||||
'nout': txi.txo_ref.position
|
||||
}
|
||||
|
||||
def encode_account(self, account):
|
||||
result = account.to_dict()
|
||||
result['id'] = account.id
|
||||
result.pop('certificates', None)
|
||||
result['is_default'] = self.ledger.accounts[0] == account
|
||||
return result
|
||||
|
||||
def encode_file(self, managed_stream):
|
||||
file = managed_stream.as_dict()
|
||||
tx_height = managed_stream.stream_claim_info.height
|
||||
best_height = self.ledger.headers.height
|
||||
file.update({
|
||||
'height': tx_height,
|
||||
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
|
||||
'timestamp': self.ledger.headers[tx_height]['timestamp'] if tx_height > 0 else None
|
||||
})
|
||||
return file
|
||||
|
||||
@staticmethod
|
||||
def encode_claim(claim):
|
||||
return getattr(claim, claim.claim_type).to_dict()
|
||||
@@ -1,5 +1,4 @@
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import json
|
||||
import logging.handlers
|
||||
import traceback
|
||||
@@ -51,7 +50,7 @@ class HTTPSLogglyHandler(logging.Handler):
|
||||
|
||||
async def _emit(self, record):
|
||||
payload = self.format(record)
|
||||
async with aiohttp.request('post', self.url, data=payload.encode(), cookies=self.cookies) as response:
|
||||
async with utils.aiohttp_request('post', self.url, data=payload.encode(), cookies=self.cookies) as response:
|
||||
self.cookies.update(response.cookies)
|
||||
|
||||
def emit(self, record):
|
||||
|
||||
@@ -22,6 +22,10 @@ def migrate_db(conf, start, end):
|
||||
from .migrate7to8 import do_migration
|
||||
elif current == 8:
|
||||
from .migrate8to9 import do_migration
|
||||
elif current == 9:
|
||||
from .migrate9to10 import do_migration
|
||||
elif current == 10:
|
||||
from .migrate10to11 import do_migration
|
||||
else:
|
||||
raise Exception("DB migration of version {} to {} is not available".format(current,
|
||||
current+1))
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
import sqlite3
|
||||
import os
|
||||
import binascii
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
current_columns = []
|
||||
for col_info in cursor.execute("pragma table_info('file');").fetchall():
|
||||
current_columns.append(col_info[1])
|
||||
if 'content_fee' in current_columns or 'saved_file' in current_columns:
|
||||
connection.close()
|
||||
print("already migrated")
|
||||
return
|
||||
|
||||
cursor.execute(
|
||||
"pragma foreign_keys=off;"
|
||||
)
|
||||
|
||||
cursor.execute("""
|
||||
create table if not exists new_file (
|
||||
stream_hash text primary key not null references stream,
|
||||
file_name text,
|
||||
download_directory text,
|
||||
blob_data_rate real not null,
|
||||
status text not null,
|
||||
saved_file integer not null,
|
||||
content_fee text
|
||||
);
|
||||
""")
|
||||
for (stream_hash, file_name, download_dir, data_rate, status) in cursor.execute("select * from file").fetchall():
|
||||
saved_file = 0
|
||||
if download_dir != '{stream}' and file_name != '{stream}':
|
||||
try:
|
||||
if os.path.isfile(os.path.join(binascii.unhexlify(download_dir).decode(),
|
||||
binascii.unhexlify(file_name).decode())):
|
||||
saved_file = 1
|
||||
else:
|
||||
download_dir, file_name = None, None
|
||||
except Exception:
|
||||
download_dir, file_name = None, None
|
||||
else:
|
||||
download_dir, file_name = None, None
|
||||
cursor.execute(
|
||||
"insert into new_file values (?, ?, ?, ?, ?, ?, NULL)",
|
||||
(stream_hash, file_name, download_dir, data_rate, status, saved_file)
|
||||
)
|
||||
cursor.execute("drop table file")
|
||||
cursor.execute("alter table new_file rename to file")
|
||||
connection.commit()
|
||||
connection.close()
|
||||
@@ -3,7 +3,7 @@ import os
|
||||
import json
|
||||
import logging
|
||||
from binascii import hexlify
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
from lbrynet.schema.claim import Claim
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -260,7 +260,7 @@ def do_migration(conf):
|
||||
(
|
||||
"%s:%i" % (claim_arg_tup[0], claim_arg_tup[1]), claim_arg_tup[2], claim_arg_tup[3],
|
||||
claim_arg_tup[7], claim_arg_tup[6], claim_arg_tup[8],
|
||||
smart_decode(claim_arg_tup[8]).certificate_id, claim_arg_tup[5], claim_arg_tup[4]
|
||||
Claim.from_bytes(claim_arg_tup[8]).signing_channel_id, claim_arg_tup[5], claim_arg_tup[4]
|
||||
)
|
||||
for sd_hash, claim_arg_tup in claim_queries.items() if claim_arg_tup
|
||||
] # sd_hash, (txid, nout, claim_id, name, sequence, address, height, amount, serialized)
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
query = "select stream_hash, sd_hash from main.stream"
|
||||
for stream_hash, sd_hash in cursor.execute(query).fetchall():
|
||||
head_blob_hash = cursor.execute(
|
||||
"select blob_hash from stream_blob where position = 0 and stream_hash = ?",
|
||||
(stream_hash,)
|
||||
).fetchone()
|
||||
if not head_blob_hash:
|
||||
continue
|
||||
cursor.execute("update blob set should_announce=1 where blob_hash in (?, ?)", (sd_hash, head_blob_hash[0],))
|
||||
connection.commit()
|
||||
connection.close()
|
||||
@@ -1,160 +0,0 @@
|
||||
import os
|
||||
|
||||
|
||||
types_map = {
|
||||
# http://www.iana.org/assignments/media-types
|
||||
'.a': 'application/octet-stream',
|
||||
'.ai': 'application/postscript',
|
||||
'.aif': 'audio/x-aiff',
|
||||
'.aifc': 'audio/x-aiff',
|
||||
'.aiff': 'audio/x-aiff',
|
||||
'.au': 'audio/basic',
|
||||
'.avi': 'video/x-msvideo',
|
||||
'.bat': 'text/plain',
|
||||
'.bcpio': 'application/x-bcpio',
|
||||
'.bin': 'application/octet-stream',
|
||||
'.bmp': 'image/bmp',
|
||||
'.c': 'text/plain',
|
||||
'.cdf': 'application/x-netcdf',
|
||||
'.cpio': 'application/x-cpio',
|
||||
'.csh': 'application/x-csh',
|
||||
'.css': 'text/css',
|
||||
'.csv': 'text/csv',
|
||||
'.dll': 'application/octet-stream',
|
||||
'.doc': 'application/msword',
|
||||
'.dot': 'application/msword',
|
||||
'.dvi': 'application/x-dvi',
|
||||
'.eml': 'message/rfc822',
|
||||
'.eps': 'application/postscript',
|
||||
'.epub': 'application/epub+zip',
|
||||
'.etx': 'text/x-setext',
|
||||
'.exe': 'application/octet-stream',
|
||||
'.gif': 'image/gif',
|
||||
'.gtar': 'application/x-gtar',
|
||||
'.h': 'text/plain',
|
||||
'.hdf': 'application/x-hdf',
|
||||
'.htm': 'text/html',
|
||||
'.html': 'text/html',
|
||||
'.ico': 'image/vnd.microsoft.icon',
|
||||
'.ief': 'image/ief',
|
||||
'.iges': 'model/iges',
|
||||
'.jpe': 'image/jpeg',
|
||||
'.jpeg': 'image/jpeg',
|
||||
'.jpg': 'image/jpeg',
|
||||
'.js': 'application/javascript',
|
||||
'.json': 'application/json',
|
||||
'.ksh': 'text/plain',
|
||||
'.latex': 'application/x-latex',
|
||||
'.m1v': 'video/mpeg',
|
||||
'.m3u': 'application/vnd.apple.mpegurl',
|
||||
'.m3u8': 'application/vnd.apple.mpegurl',
|
||||
'.man': 'application/x-troff-man',
|
||||
'.markdown': 'text/markdown',
|
||||
'.md': 'text/markdown',
|
||||
'.me': 'application/x-troff-me',
|
||||
'.mht': 'message/rfc822',
|
||||
'.mhtml': 'message/rfc822',
|
||||
'.mif': 'application/x-mif',
|
||||
'.mov': 'video/quicktime',
|
||||
'.movie': 'video/x-sgi-movie',
|
||||
'.mp2': 'audio/mpeg',
|
||||
'.mp3': 'audio/mpeg',
|
||||
'.mp4': 'video/mp4',
|
||||
'.mpa': 'video/mpeg',
|
||||
'.mpe': 'video/mpeg',
|
||||
'.mpeg': 'video/mpeg',
|
||||
'.mpg': 'video/mpeg',
|
||||
'.ms': 'application/x-troff-ms',
|
||||
'.nc': 'application/x-netcdf',
|
||||
'.nws': 'message/rfc822',
|
||||
'.o': 'application/octet-stream',
|
||||
'.obj': 'application/octet-stream',
|
||||
'.oda': 'application/oda',
|
||||
'.p12': 'application/x-pkcs12',
|
||||
'.p7c': 'application/pkcs7-mime',
|
||||
'.pbm': 'image/x-portable-bitmap',
|
||||
'.pdf': 'application/pdf',
|
||||
'.pfx': 'application/x-pkcs12',
|
||||
'.pgm': 'image/x-portable-graymap',
|
||||
'.pl': 'text/plain',
|
||||
'.png': 'image/png',
|
||||
'.pnm': 'image/x-portable-anymap',
|
||||
'.pot': 'application/vnd.ms-powerpoint',
|
||||
'.ppa': 'application/vnd.ms-powerpoint',
|
||||
'.ppm': 'image/x-portable-pixmap',
|
||||
'.pps': 'application/vnd.ms-powerpoint',
|
||||
'.ppt': 'application/vnd.ms-powerpoint',
|
||||
'.ps': 'application/postscript',
|
||||
'.pwz': 'application/vnd.ms-powerpoint',
|
||||
'.py': 'text/x-python',
|
||||
'.pyc': 'application/x-python-code',
|
||||
'.pyo': 'application/x-python-code',
|
||||
'.qt': 'video/quicktime',
|
||||
'.ra': 'audio/x-pn-realaudio',
|
||||
'.ram': 'application/x-pn-realaudio',
|
||||
'.ras': 'image/x-cmu-raster',
|
||||
'.rdf': 'application/xml',
|
||||
'.rgb': 'image/x-rgb',
|
||||
'.roff': 'application/x-troff',
|
||||
'.rtx': 'text/richtext',
|
||||
'.sgm': 'text/x-sgml',
|
||||
'.sgml': 'text/x-sgml',
|
||||
'.sh': 'application/x-sh',
|
||||
'.shar': 'application/x-shar',
|
||||
'.snd': 'audio/basic',
|
||||
'.so': 'application/octet-stream',
|
||||
'.src': 'application/x-wais-source',
|
||||
'.stl': 'model/stl',
|
||||
'.sv4cpio': 'application/x-sv4cpio',
|
||||
'.sv4crc': 'application/x-sv4crc',
|
||||
'.svg': 'image/svg+xml',
|
||||
'.swf': 'application/x-shockwave-flash',
|
||||
'.t': 'application/x-troff',
|
||||
'.tar': 'application/x-tar',
|
||||
'.tcl': 'application/x-tcl',
|
||||
'.tex': 'application/x-tex',
|
||||
'.texi': 'application/x-texinfo',
|
||||
'.texinfo': 'application/x-texinfo',
|
||||
'.tif': 'image/tiff',
|
||||
'.tiff': 'image/tiff',
|
||||
'.tr': 'application/x-troff',
|
||||
'.tsv': 'text/tab-separated-values',
|
||||
'.txt': 'text/plain',
|
||||
'.ustar': 'application/x-ustar',
|
||||
'.vcf': 'text/x-vcard',
|
||||
'.wav': 'audio/x-wav',
|
||||
'.webm': 'video/webm',
|
||||
'.wiz': 'application/msword',
|
||||
'.wsdl': 'application/xml',
|
||||
'.xbm': 'image/x-xbitmap',
|
||||
'.xlb': 'application/vnd.ms-excel',
|
||||
'.xls': 'application/vnd.ms-excel',
|
||||
'.xml': 'text/xml',
|
||||
'.xpdl': 'application/xml',
|
||||
'.xpm': 'image/x-xpixmap',
|
||||
'.xsl': 'application/xml',
|
||||
'.xwd': 'image/x-xwindowdump',
|
||||
'.zip': 'application/zip',
|
||||
|
||||
# These are non-standard types, commonly found in the wild.
|
||||
'.mid': 'audio/midi',
|
||||
'.midi': 'audio/midi',
|
||||
'.pct': 'image/pict',
|
||||
'.pic': 'image/pict',
|
||||
'.pict': 'image/pict',
|
||||
'.rtf': 'application/rtf',
|
||||
'.xul': 'text/xul',
|
||||
'.m4v': 'video/m4v',
|
||||
|
||||
# microsoft is special and has its own 'standard'
|
||||
# https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
|
||||
'.wmv': 'video/x-ms-wmv'
|
||||
}
|
||||
|
||||
|
||||
def guess_media_type(path):
|
||||
_, ext = os.path.splitext(path)
|
||||
return types_map.get(
|
||||
ext,
|
||||
'application/octet-stream' if not (ext and ext[1:].strip()) else f'application/x-ext-{ext[1:].strip().lower()}'
|
||||
)
|
||||
+176
-101
@@ -7,9 +7,9 @@ import binascii
|
||||
import time
|
||||
from torba.client.basedatabase import SQLiteMixin
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
from lbrynet.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
||||
from lbrynet.wallet.transaction import Transaction
|
||||
from lbrynet.schema.claim import Claim
|
||||
from lbrynet.dht.constants import data_expiration
|
||||
from lbrynet.blob.blob_info import BlobInfo
|
||||
|
||||
@@ -39,7 +39,9 @@ class StoredStreamClaim:
|
||||
self.claim_name = name
|
||||
self.amount = amount
|
||||
self.height = height
|
||||
self.claim: typing.Optional[ClaimDict] = None if not serialized else smart_decode(serialized)
|
||||
self.claim: typing.Optional[Claim] = None if not serialized else Claim.from_bytes(
|
||||
binascii.unhexlify(serialized)
|
||||
)
|
||||
self.claim_address = address
|
||||
self.claim_sequence = claim_sequence
|
||||
self.channel_claim_id = channel_claim_id
|
||||
@@ -53,10 +55,6 @@ class StoredStreamClaim:
|
||||
def nout(self) -> typing.Optional[int]:
|
||||
return None if not self.outpoint else int(self.outpoint.split(":")[1])
|
||||
|
||||
@property
|
||||
def metadata(self) -> typing.Optional[typing.Dict]:
|
||||
return None if not self.claim else self.claim.claim_dict['stream']['metadata']
|
||||
|
||||
def as_dict(self) -> typing.Dict:
|
||||
return {
|
||||
"name": self.claim_name,
|
||||
@@ -117,8 +115,8 @@ def _batched_select(transaction, query, parameters, batch_size=900):
|
||||
def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
|
||||
files = []
|
||||
signed_claims = {}
|
||||
for (rowid, stream_hash, file_name, download_dir, data_rate, status, _, sd_hash, stream_key,
|
||||
stream_name, suggested_file_name, *claim_args) in _batched_select(
|
||||
for (rowid, stream_hash, file_name, download_dir, data_rate, status, saved_file, raw_content_fee, _,
|
||||
sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in _batched_select(
|
||||
transaction, "select file.rowid, file.*, stream.*, c.* "
|
||||
"from file inner join stream on file.stream_hash=stream.stream_hash "
|
||||
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
|
||||
@@ -144,33 +142,28 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
|
||||
"key": stream_key,
|
||||
"stream_name": stream_name, # hex
|
||||
"suggested_file_name": suggested_file_name, # hex
|
||||
"claim": claim
|
||||
"claim": claim,
|
||||
"saved_file": bool(saved_file),
|
||||
"content_fee": None if not raw_content_fee else Transaction(
|
||||
binascii.unhexlify(raw_content_fee)
|
||||
)
|
||||
}
|
||||
)
|
||||
for claim_id in signed_claims.keys():
|
||||
channel_name = transaction.execute("select claim_name from claim where claim_id=?", (claim_id, )).fetchone()
|
||||
if channel_name:
|
||||
for claim in signed_claims[claim_id]:
|
||||
claim.channel_name = channel_name[0]
|
||||
for claim_name, claim_id in _batched_select(
|
||||
transaction, "select c.claim_name, c.claim_id from claim c where c.claim_id in {}",
|
||||
list(signed_claims.keys())):
|
||||
for claim in signed_claims[claim_id]:
|
||||
claim.channel_name = claim_name
|
||||
return files
|
||||
|
||||
|
||||
def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'):
|
||||
# add the head blob and set it to be announced
|
||||
transaction.execute(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
sd_blob.blob_hash, sd_blob.length, 0, 1, "pending", 0, 0,
|
||||
descriptor.blobs[0].blob_hash, descriptor.blobs[0].length, 0, 1, "pending", 0, 0
|
||||
)
|
||||
# add all blobs, except the last one, which is empty
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
[(blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
|
||||
for blob in (descriptor.blobs[:-1] if len(descriptor.blobs) > 1 else descriptor.blobs) + [sd_blob]]
|
||||
)
|
||||
# add the rest of the blobs with announcement off
|
||||
if len(descriptor.blobs) > 2:
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
[(blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
|
||||
for blob in descriptor.blobs[1:-1]]
|
||||
)
|
||||
# associate the blobs to the stream
|
||||
transaction.execute("insert or ignore into stream values (?, ?, ?, ?, ?)",
|
||||
(descriptor.stream_hash, sd_blob.blob_hash, descriptor.key,
|
||||
@@ -182,6 +175,11 @@ def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descripto
|
||||
[(descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
|
||||
for blob in descriptor.blobs]
|
||||
)
|
||||
# ensure should_announce is set regardless if insert was ignored
|
||||
transaction.execute(
|
||||
"update blob set should_announce=1 where blob_hash in (?, ?)",
|
||||
(sd_blob.blob_hash, descriptor.blobs[0].blob_hash,)
|
||||
)
|
||||
|
||||
|
||||
def delete_stream(transaction: sqlite3.Connection, descriptor: 'StreamDescriptor'):
|
||||
@@ -194,13 +192,21 @@ def delete_stream(transaction: sqlite3.Connection, descriptor: 'StreamDescriptor
|
||||
transaction.executemany("delete from blob where blob_hash=?", blob_hashes)
|
||||
|
||||
|
||||
def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: str, download_directory: str,
|
||||
data_payment_rate: float, status: str) -> int:
|
||||
def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: typing.Optional[str],
|
||||
download_directory: typing.Optional[str], data_payment_rate: float, status: str,
|
||||
content_fee: typing.Optional[Transaction]) -> int:
|
||||
if not file_name and not download_directory:
|
||||
encoded_file_name, encoded_download_dir = None, None
|
||||
else:
|
||||
encoded_file_name = binascii.hexlify(file_name.encode()).decode()
|
||||
encoded_download_dir = binascii.hexlify(download_directory.encode()).decode()
|
||||
transaction.execute(
|
||||
"insert or replace into file values (?, ?, ?, ?, ?)",
|
||||
(stream_hash, binascii.hexlify(file_name.encode()).decode(),
|
||||
binascii.hexlify(download_directory.encode()).decode(), data_payment_rate, status)
|
||||
"insert or replace into file values (?, ?, ?, ?, ?, ?, ?)",
|
||||
(stream_hash, encoded_file_name, encoded_download_dir, data_payment_rate, status,
|
||||
1 if (file_name and download_directory and os.path.isfile(os.path.join(download_directory, file_name))) else 0,
|
||||
None if not content_fee else binascii.hexlify(content_fee.raw).decode())
|
||||
)
|
||||
|
||||
return transaction.execute("select rowid from file where stream_hash=?", (stream_hash, )).fetchone()[0]
|
||||
|
||||
|
||||
@@ -249,10 +255,12 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
create table if not exists file (
|
||||
stream_hash text primary key not null references stream,
|
||||
file_name text not null,
|
||||
download_directory text not null,
|
||||
file_name text,
|
||||
download_directory text,
|
||||
blob_data_rate real not null,
|
||||
status text not null
|
||||
status text not null,
|
||||
saved_file integer not null,
|
||||
content_fee text
|
||||
);
|
||||
|
||||
create table if not exists content_claim (
|
||||
@@ -295,27 +303,28 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
# # # # # # # # # blob functions # # # # # # # # #
|
||||
|
||||
def add_completed_blob(self, blob_hash: str, length: int):
|
||||
def _add_blob(transaction: sqlite3.Connection):
|
||||
transaction.execute(
|
||||
async def add_blobs(self, *blob_hashes_and_lengths: typing.Tuple[str, int], finished=False):
|
||||
def _add_blobs(transaction: sqlite3.Connection):
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
(blob_hash, length, 0, 0, "pending", 0, 0)
|
||||
[
|
||||
(blob_hash, length, 0, 0, "pending" if not finished else "finished", 0, 0)
|
||||
for blob_hash, length in blob_hashes_and_lengths
|
||||
]
|
||||
)
|
||||
transaction.execute(
|
||||
"update blob set status='finished' where blob.blob_hash=?", (blob_hash, )
|
||||
)
|
||||
return self.db.run(_add_blob)
|
||||
if finished:
|
||||
transaction.executemany(
|
||||
"update blob set status='finished' where blob.blob_hash=?", [
|
||||
(blob_hash, ) for blob_hash, _ in blob_hashes_and_lengths
|
||||
]
|
||||
)
|
||||
return await self.db.run(_add_blobs)
|
||||
|
||||
def get_blob_status(self, blob_hash: str):
|
||||
return self.run_and_return_one_or_none(
|
||||
"select status from blob where blob_hash=?", blob_hash
|
||||
)
|
||||
|
||||
def add_known_blob(self, blob_hash: str, length: int):
|
||||
return self.db.execute(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)", (blob_hash, length, 0, 0, "pending", 0, 0)
|
||||
)
|
||||
|
||||
def should_announce(self, blob_hash: str):
|
||||
return self.run_and_return_one_or_none(
|
||||
"select should_announce from blob where blob_hash=?", blob_hash
|
||||
@@ -418,6 +427,26 @@ class SQLiteStorage(SQLiteMixin):
|
||||
}
|
||||
return self.db.run(_sync_blobs)
|
||||
|
||||
def sync_files_to_blobs(self):
|
||||
def _sync_blobs(transaction: sqlite3.Connection):
|
||||
transaction.executemany(
|
||||
"update file set status='stopped' where stream_hash=?",
|
||||
transaction.execute(
|
||||
"select distinct sb.stream_hash from stream_blob sb "
|
||||
"inner join blob b on b.blob_hash=sb.blob_hash and b.status=='pending'"
|
||||
).fetchall()
|
||||
)
|
||||
return self.db.run(_sync_blobs)
|
||||
|
||||
def set_files_as_streaming(self, stream_hashes: typing.List[str]):
|
||||
def _set_streaming(transaction: sqlite3.Connection):
|
||||
transaction.executemany(
|
||||
"update file set file_name=null, download_directory=null where stream_hash=?",
|
||||
[(stream_hash, ) for stream_hash in stream_hashes]
|
||||
)
|
||||
|
||||
return self.db.run(_set_streaming)
|
||||
|
||||
# # # # # # # # # stream functions # # # # # # # # #
|
||||
|
||||
async def stream_exists(self, sd_hash: str) -> bool:
|
||||
@@ -480,20 +509,53 @@ class SQLiteStorage(SQLiteMixin):
|
||||
"select stream_hash from stream where sd_hash = ?", sd_blob_hash
|
||||
)
|
||||
|
||||
def get_stream_info_for_sd_hash(self, sd_blob_hash):
|
||||
return self.run_and_return_one_or_none(
|
||||
"select stream_hash, stream_name, suggested_filename, stream_key from stream where sd_hash = ?",
|
||||
sd_blob_hash
|
||||
)
|
||||
|
||||
def delete_stream(self, descriptor: 'StreamDescriptor'):
|
||||
return self.db.run_with_foreign_keys_disabled(delete_stream, descriptor)
|
||||
|
||||
# # # # # # # # # file stuff # # # # # # # # #
|
||||
|
||||
def save_downloaded_file(self, stream_hash, file_name, download_directory,
|
||||
data_payment_rate) -> typing.Awaitable[int]:
|
||||
def save_downloaded_file(self, stream_hash: str, file_name: typing.Optional[str],
|
||||
download_directory: typing.Optional[str], data_payment_rate: float,
|
||||
content_fee: typing.Optional[Transaction] = None) -> typing.Awaitable[int]:
|
||||
return self.save_published_file(
|
||||
stream_hash, file_name, download_directory, data_payment_rate, status="running"
|
||||
stream_hash, file_name, download_directory, data_payment_rate, status="running",
|
||||
content_fee=content_fee
|
||||
)
|
||||
|
||||
def save_published_file(self, stream_hash: str, file_name: str, download_directory: str, data_payment_rate: float,
|
||||
status="finished") -> typing.Awaitable[int]:
|
||||
return self.db.run(store_file, stream_hash, file_name, download_directory, data_payment_rate, status)
|
||||
def save_published_file(self, stream_hash: str, file_name: typing.Optional[str],
|
||||
download_directory: typing.Optional[str], data_payment_rate: float,
|
||||
status: str = "finished",
|
||||
content_fee: typing.Optional[Transaction] = None) -> typing.Awaitable[int]:
|
||||
return self.db.run(store_file, stream_hash, file_name, download_directory, data_payment_rate, status,
|
||||
content_fee)
|
||||
|
||||
async def update_manually_removed_files_since_last_run(self):
|
||||
"""
|
||||
Update files that have been removed from the downloads directory since the last run
|
||||
"""
|
||||
def update_manually_removed_files(transaction: sqlite3.Connection):
|
||||
removed = []
|
||||
for (stream_hash, download_directory, file_name) in transaction.execute(
|
||||
"select stream_hash, download_directory, file_name from file where saved_file=1"
|
||||
).fetchall():
|
||||
if download_directory and file_name and os.path.isfile(
|
||||
os.path.join(binascii.unhexlify(download_directory).decode(),
|
||||
binascii.unhexlify(file_name).decode())):
|
||||
continue
|
||||
else:
|
||||
removed.append((stream_hash,))
|
||||
if removed:
|
||||
transaction.executemany(
|
||||
"update file set file_name=null, download_directory=null, saved_file=0 where stream_hash=?",
|
||||
removed
|
||||
)
|
||||
return await self.db.run(update_manually_removed_files)
|
||||
|
||||
def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]:
|
||||
return self.db.run(get_all_lbry_files)
|
||||
@@ -502,24 +564,45 @@ class SQLiteStorage(SQLiteMixin):
|
||||
log.debug("update file status %s -> %s", stream_hash, new_status)
|
||||
return self.db.execute("update file set status=? where stream_hash=?", (new_status, stream_hash))
|
||||
|
||||
def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: str, file_name: str):
|
||||
return self.db.execute("update file set download_directory=?, file_name=? where stream_hash=?", (
|
||||
binascii.hexlify(download_dir.encode()).decode(), binascii.hexlify(file_name.encode()).decode(),
|
||||
stream_hash
|
||||
async def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: typing.Optional[str],
|
||||
file_name: typing.Optional[str]):
|
||||
if not file_name or not download_dir:
|
||||
encoded_file_name, encoded_download_dir = None, None
|
||||
else:
|
||||
encoded_file_name = binascii.hexlify(file_name.encode()).decode()
|
||||
encoded_download_dir = binascii.hexlify(download_dir.encode()).decode()
|
||||
return await self.db.execute("update file set download_directory=?, file_name=? where stream_hash=?", (
|
||||
encoded_download_dir, encoded_file_name, stream_hash,
|
||||
))
|
||||
|
||||
async def recover_streams(self, descriptors_and_sds: typing.List[typing.Tuple['StreamDescriptor', 'BlobFile']],
|
||||
async def save_content_fee(self, stream_hash: str, content_fee: Transaction):
|
||||
return await self.db.execute("update file set content_fee=? where stream_hash=?", (
|
||||
binascii.hexlify(content_fee.raw), stream_hash,
|
||||
))
|
||||
|
||||
async def set_saved_file(self, stream_hash: str):
|
||||
return await self.db.execute("update file set saved_file=1 where stream_hash=?", (
|
||||
stream_hash,
|
||||
))
|
||||
|
||||
async def clear_saved_file(self, stream_hash: str):
|
||||
return await self.db.execute("update file set saved_file=0 where stream_hash=?", (
|
||||
stream_hash,
|
||||
))
|
||||
|
||||
async def recover_streams(self, descriptors_and_sds: typing.List[typing.Tuple['StreamDescriptor', 'BlobFile',
|
||||
typing.Optional[Transaction]]],
|
||||
download_directory: str):
|
||||
def _recover(transaction: sqlite3.Connection):
|
||||
stream_hashes = [d.stream_hash for d, s in descriptors_and_sds]
|
||||
for descriptor, sd_blob in descriptors_and_sds:
|
||||
stream_hashes = [x[0].stream_hash for x in descriptors_and_sds]
|
||||
for descriptor, sd_blob, content_fee in descriptors_and_sds:
|
||||
content_claim = transaction.execute(
|
||||
"select * from content_claim where stream_hash=?", (descriptor.stream_hash, )
|
||||
).fetchone()
|
||||
delete_stream(transaction, descriptor) # this will also delete the content claim
|
||||
store_stream(transaction, sd_blob, descriptor)
|
||||
store_file(transaction, descriptor.stream_hash, os.path.basename(descriptor.suggested_file_name),
|
||||
download_directory, 0.0, 'stopped')
|
||||
download_directory, 0.0, 'stopped', content_fee=content_fee)
|
||||
if content_claim:
|
||||
transaction.execute("insert or ignore into content_claim values (?, ?)", content_claim)
|
||||
transaction.executemany(
|
||||
@@ -538,16 +621,18 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
# # # # # # # # # support functions # # # # # # # # #
|
||||
|
||||
def save_supports(self, claim_id, supports):
|
||||
def save_supports(self, claim_id_to_supports: dict):
|
||||
# TODO: add 'address' to support items returned for a claim from lbrycrdd and lbryum-server
|
||||
def _save_support(transaction):
|
||||
transaction.execute("delete from support where claim_id=?", (claim_id,))
|
||||
for support in supports:
|
||||
transaction.execute(
|
||||
"insert into support values (?, ?, ?, ?)",
|
||||
("%s:%i" % (support['txid'], support['nout']), claim_id, lbc_to_dewies(support['amount']),
|
||||
support.get('address', ""))
|
||||
)
|
||||
bind = "({})".format(','.join(['?'] * len(claim_id_to_supports)))
|
||||
transaction.execute(f"delete from support where claim_id in {bind}", list(claim_id_to_supports.keys()))
|
||||
for claim_id, supports in claim_id_to_supports.items():
|
||||
for support in supports:
|
||||
transaction.execute(
|
||||
"insert into support values (?, ?, ?, ?)",
|
||||
("%s:%i" % (support['txid'], support['nout']), claim_id, lbc_to_dewies(support['amount']),
|
||||
support.get('address', ""))
|
||||
)
|
||||
return self.db.run(_save_support)
|
||||
|
||||
def get_supports(self, *claim_ids):
|
||||
@@ -575,7 +660,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
# # # # # # # # # claim functions # # # # # # # # #
|
||||
|
||||
async def save_claims(self, claim_infos):
|
||||
support_callbacks = []
|
||||
claim_id_to_supports = {}
|
||||
update_file_callbacks = []
|
||||
|
||||
def _save_claims(transaction):
|
||||
@@ -588,19 +673,12 @@ class SQLiteStorage(SQLiteMixin):
|
||||
height = claim_info['height']
|
||||
address = claim_info['address']
|
||||
sequence = claim_info['claim_sequence']
|
||||
certificate_id = claim_info['value'].signing_channel_id
|
||||
try:
|
||||
certificate_id = claim_info['value'].get('publisherSignature', {}).get('certificateId')
|
||||
except AttributeError:
|
||||
certificate_id = None
|
||||
try:
|
||||
if claim_info['value'].get('stream', {}).get('source', {}).get('sourceType') == "lbry_sd_hash":
|
||||
source_hash = claim_info['value'].get('stream', {}).get('source', {}).get('source')
|
||||
else:
|
||||
source_hash = None
|
||||
except AttributeError:
|
||||
source_hash = claim_info['value'].stream.source.sd_hash
|
||||
except (AttributeError, ValueError):
|
||||
source_hash = None
|
||||
serialized = claim_info.get('hex') or binascii.hexlify(
|
||||
smart_decode(claim_info['value']).serialized).decode()
|
||||
serialized = binascii.hexlify(claim_info['value'].to_bytes())
|
||||
transaction.execute(
|
||||
"insert or replace into claim values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(outpoint, claim_id, name, amount, height, serialized, certificate_id, address, sequence)
|
||||
@@ -608,7 +686,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
# if this response doesn't have support info don't overwrite the existing
|
||||
# support info
|
||||
if 'supports' in claim_info:
|
||||
support_callbacks.append((claim_id, claim_info['supports']))
|
||||
claim_id_to_supports[claim_id] = claim_info['supports']
|
||||
if not source_hash:
|
||||
continue
|
||||
stream_hash = transaction.execute(
|
||||
@@ -620,12 +698,12 @@ class SQLiteStorage(SQLiteMixin):
|
||||
stream_hash = stream_hash[0]
|
||||
known_outpoint = transaction.execute(
|
||||
"select claim_outpoint from content_claim where stream_hash=?", (stream_hash,)
|
||||
)
|
||||
).fetchone()
|
||||
known_claim_id = transaction.execute(
|
||||
"select claim_id from claim "
|
||||
"inner join content_claim c3 ON claim.claim_outpoint=c3.claim_outpoint "
|
||||
"where c3.stream_hash=?", (stream_hash,)
|
||||
)
|
||||
).fetchone()
|
||||
if not known_claim_id:
|
||||
content_claims_to_update.append((stream_hash, outpoint))
|
||||
elif known_outpoint != outpoint:
|
||||
@@ -638,23 +716,20 @@ class SQLiteStorage(SQLiteMixin):
|
||||
await self.db.run(_save_claims)
|
||||
if update_file_callbacks:
|
||||
await asyncio.wait(update_file_callbacks)
|
||||
if support_callbacks:
|
||||
await asyncio.wait([
|
||||
self.save_supports(*args) for args in support_callbacks
|
||||
])
|
||||
if claim_id_to_supports:
|
||||
await self.save_supports(claim_id_to_supports)
|
||||
|
||||
def save_claims_for_resolve(self, claim_infos):
|
||||
to_save = []
|
||||
to_save = {}
|
||||
for info in claim_infos:
|
||||
if 'value' in info:
|
||||
if info['value']:
|
||||
to_save.append(info)
|
||||
to_save[info['claim_id']] = info
|
||||
else:
|
||||
if 'certificate' in info and info['certificate']['value']:
|
||||
to_save.append(info['certificate'])
|
||||
if 'claim' in info and info['claim']['value']:
|
||||
to_save.append(info['claim'])
|
||||
return self.save_claims(to_save)
|
||||
for key in ('certificate', 'claim'):
|
||||
if info.get(key, {}).get('value'):
|
||||
to_save[info[key]['claim_id']] = info[key]
|
||||
return self.save_claims(to_save.values())
|
||||
|
||||
@staticmethod
|
||||
def _save_content_claim(transaction, claim_outpoint, stream_hash):
|
||||
@@ -664,7 +739,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
).fetchone()
|
||||
if not claim_info:
|
||||
raise Exception("claim not found")
|
||||
new_claim_id, claim = claim_info[0], ClaimDict.deserialize(binascii.unhexlify(claim_info[1]))
|
||||
new_claim_id, claim = claim_info[0], Claim.from_bytes(binascii.unhexlify(claim_info[1]))
|
||||
|
||||
# certificate claims should not be in the content_claim table
|
||||
if not claim.is_stream:
|
||||
@@ -677,7 +752,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
if not known_sd_hash:
|
||||
raise Exception("stream not found")
|
||||
# check the claim contains the same sd hash
|
||||
if known_sd_hash[0].encode() != claim.source_hash:
|
||||
if known_sd_hash[0] != claim.stream.source.sd_hash:
|
||||
raise Exception("stream mismatch")
|
||||
|
||||
# if there is a current claim associated to the file, check that the new claim is an update to it
|
||||
|
||||
@@ -2,7 +2,6 @@ import platform
|
||||
import os
|
||||
import logging.handlers
|
||||
|
||||
from lbrynet.schema import __version__ as schema_version
|
||||
from lbrynet import build_type, __version__ as lbrynet_version
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -16,7 +15,6 @@ def get_platform() -> dict:
|
||||
"os_release": platform.release(),
|
||||
"os_system": platform.system(),
|
||||
"lbrynet_version": lbrynet_version,
|
||||
"lbryschema_version": schema_version,
|
||||
"build": build_type.BUILD, # CI server sets this during build step
|
||||
}
|
||||
if p["os_system"] == "Linux":
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
__node_daemon__ = 'lbrycrdd'
|
||||
__node_cli__ = 'lbrycrd-cli'
|
||||
__node_bin__ = ''
|
||||
__node_url__ = (
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.12.2.1/lbrycrd-linux.zip'
|
||||
)
|
||||
__spvserver__ = 'lbrynet.extras.wallet.server.coin.LBCRegTest'
|
||||
|
||||
from lbrynet.extras.wallet.ledger import MainNetLedger, RegTestLedger
|
||||
from lbrynet.extras.wallet.manager import LbryWalletManager
|
||||
from lbrynet.extras.wallet.network import Network
|
||||
@@ -1,491 +0,0 @@
|
||||
import logging
|
||||
|
||||
from ecdsa import BadSignatureError
|
||||
from binascii import unhexlify, hexlify
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc
|
||||
from lbrynet.error import UnknownNameError, UnknownClaimID, UnknownURI, UnknownOutpoint
|
||||
from lbrynet.schema.address import is_address
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
from lbrynet.schema.error import DecodeError
|
||||
from lbrynet.schema.uri import parse_lbry_uri
|
||||
from lbrynet.extras.wallet.claim_proofs import verify_proof, InvalidProofError
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Resolver:
|
||||
|
||||
def __init__(self, claim_trie_root, height, transaction_class, hash160_to_address, network):
|
||||
self.claim_trie_root = claim_trie_root
|
||||
self.height = height
|
||||
self.transaction_class = transaction_class
|
||||
self.hash160_to_address = hash160_to_address
|
||||
self.network = network
|
||||
|
||||
async def _handle_resolutions(self, resolutions, requested_uris, page, page_size):
|
||||
results = {}
|
||||
for uri in requested_uris:
|
||||
resolution = (resolutions or {}).get(uri, {})
|
||||
if resolution:
|
||||
try:
|
||||
results[uri] = _handle_claim_result(
|
||||
await self._handle_resolve_uri_response(uri, resolution, page, page_size)
|
||||
)
|
||||
except (UnknownNameError, UnknownClaimID, UnknownURI) as err:
|
||||
results[uri] = {'error': str(err)}
|
||||
else:
|
||||
results[uri] = {'error': "URI lbry://{} cannot be resolved".format(uri.replace("lbry://", ""))}
|
||||
return results
|
||||
|
||||
async def _handle_resolve_uri_response(self, uri, resolution, page=0, page_size=10, raw=False):
|
||||
result = {}
|
||||
claim_trie_root = self.claim_trie_root
|
||||
parsed_uri = parse_lbry_uri(uri)
|
||||
certificate = None
|
||||
# parse an included certificate
|
||||
if 'certificate' in resolution:
|
||||
certificate_response = resolution['certificate']['result']
|
||||
certificate_resolution_type = resolution['certificate']['resolution_type']
|
||||
if certificate_resolution_type == "winning" and certificate_response:
|
||||
if 'height' in certificate_response:
|
||||
height = certificate_response['height']
|
||||
depth = self.height - height
|
||||
certificate_result = _verify_proof(parsed_uri.name,
|
||||
claim_trie_root,
|
||||
certificate_response,
|
||||
height, depth,
|
||||
transaction_class=self.transaction_class,
|
||||
hash160_to_address=self.hash160_to_address)
|
||||
result['certificate'] = await self.parse_and_validate_claim_result(certificate_result,
|
||||
raw=raw)
|
||||
elif certificate_resolution_type == "claim_id":
|
||||
result['certificate'] = await self.parse_and_validate_claim_result(certificate_response,
|
||||
raw=raw)
|
||||
elif certificate_resolution_type == "sequence":
|
||||
result['certificate'] = await self.parse_and_validate_claim_result(certificate_response,
|
||||
raw=raw)
|
||||
else:
|
||||
log.error("unknown response type: %s", certificate_resolution_type)
|
||||
|
||||
if 'certificate' in result:
|
||||
certificate = result['certificate']
|
||||
if 'unverified_claims_in_channel' in resolution:
|
||||
max_results = len(resolution['unverified_claims_in_channel'])
|
||||
result['claims_in_channel'] = max_results
|
||||
else:
|
||||
result['claims_in_channel'] = 0
|
||||
else:
|
||||
result['error'] = "claim not found"
|
||||
result['success'] = False
|
||||
result['uri'] = str(parsed_uri)
|
||||
|
||||
else:
|
||||
certificate = None
|
||||
|
||||
# if this was a resolution for a name, parse the result
|
||||
if 'claim' in resolution:
|
||||
claim_response = resolution['claim']['result']
|
||||
claim_resolution_type = resolution['claim']['resolution_type']
|
||||
if claim_resolution_type == "winning" and claim_response:
|
||||
if 'height' in claim_response:
|
||||
height = claim_response['height']
|
||||
depth = self.height - height
|
||||
claim_result = _verify_proof(parsed_uri.name,
|
||||
claim_trie_root,
|
||||
claim_response,
|
||||
height, depth,
|
||||
transaction_class=self.transaction_class,
|
||||
hash160_to_address=self.hash160_to_address)
|
||||
result['claim'] = await self.parse_and_validate_claim_result(claim_result,
|
||||
certificate,
|
||||
raw)
|
||||
elif claim_resolution_type == "claim_id":
|
||||
result['claim'] = await self.parse_and_validate_claim_result(claim_response,
|
||||
certificate,
|
||||
raw)
|
||||
elif claim_resolution_type == "sequence":
|
||||
result['claim'] = await self.parse_and_validate_claim_result(claim_response,
|
||||
certificate,
|
||||
raw)
|
||||
else:
|
||||
log.error("unknown response type: %s", claim_resolution_type)
|
||||
|
||||
# if this was a resolution for a name in a channel make sure there is only one valid
|
||||
# match
|
||||
elif 'unverified_claims_for_name' in resolution and 'certificate' in result:
|
||||
unverified_claims_for_name = resolution['unverified_claims_for_name']
|
||||
|
||||
channel_info = await self.get_channel_claims_page(unverified_claims_for_name,
|
||||
result['certificate'], page=1)
|
||||
claims_in_channel, upper_bound = channel_info
|
||||
|
||||
if not claims_in_channel:
|
||||
log.error("No valid claims for this name for this channel")
|
||||
elif len(claims_in_channel) > 1:
|
||||
log.warning("Multiple signed claims for the same name.")
|
||||
winner = pick_winner_from_channel_path_collision(claims_in_channel)
|
||||
if winner:
|
||||
result['claim'] = winner
|
||||
else:
|
||||
log.error("No valid claims for this name for this channel")
|
||||
else:
|
||||
result['claim'] = claims_in_channel[0]
|
||||
|
||||
# parse and validate claims in a channel iteratively into pages of results
|
||||
elif 'unverified_claims_in_channel' in resolution and 'certificate' in result:
|
||||
ids_to_check = resolution['unverified_claims_in_channel']
|
||||
channel_info = await self.get_channel_claims_page(ids_to_check, result['certificate'],
|
||||
page=page, page_size=page_size)
|
||||
claims_in_channel, upper_bound = channel_info
|
||||
|
||||
if claims_in_channel:
|
||||
result['claims_in_channel'] = claims_in_channel
|
||||
elif 'error' not in result:
|
||||
return {'error': 'claim not found', 'success': False, 'uri': str(parsed_uri)}
|
||||
|
||||
# invalid signatures can only return outside a channel
|
||||
if result.get('claim', {}).get('has_signature', False):
|
||||
if parsed_uri.path and not result['claim']['signature_is_valid']:
|
||||
return {'error': 'claim not found', 'success': False, 'uri': str(parsed_uri)}
|
||||
return result
|
||||
|
||||
async def get_certificate_and_validate_result(self, claim_result):
|
||||
if not claim_result or 'value' not in claim_result:
|
||||
return claim_result
|
||||
certificate = None
|
||||
certificate_id = smart_decode(claim_result['value']).certificate_id
|
||||
if certificate_id:
|
||||
certificate = await self.network.get_claims_by_ids(certificate_id)
|
||||
certificate = certificate.pop(certificate_id) if certificate else None
|
||||
return await self.parse_and_validate_claim_result(claim_result, certificate=certificate)
|
||||
|
||||
async def parse_and_validate_claim_result(self, claim_result, certificate=None, raw=False):
|
||||
if not claim_result or 'value' not in claim_result:
|
||||
return claim_result
|
||||
|
||||
claim_result['decoded_claim'] = False
|
||||
decoded = None
|
||||
|
||||
if not raw:
|
||||
claim_value = claim_result['value']
|
||||
try:
|
||||
decoded = smart_decode(claim_value)
|
||||
claim_result['value'] = decoded.claim_dict
|
||||
claim_result['decoded_claim'] = True
|
||||
except DecodeError:
|
||||
pass
|
||||
|
||||
if decoded:
|
||||
claim_result['has_signature'] = False
|
||||
if decoded.has_signature:
|
||||
if certificate is None:
|
||||
log.info("fetching certificate to check claim signature")
|
||||
certificate = await self.network.get_claims_by_ids(decoded.certificate_id)
|
||||
if not certificate:
|
||||
log.warning('Certificate %s not found', decoded.certificate_id)
|
||||
claim_result['has_signature'] = True
|
||||
claim_result['signature_is_valid'] = False
|
||||
validated, channel_name = validate_claim_signature_and_get_channel_name(
|
||||
decoded, certificate, claim_result['address'], claim_result['name'])
|
||||
claim_result['channel_name'] = channel_name
|
||||
if validated:
|
||||
claim_result['signature_is_valid'] = True
|
||||
|
||||
if 'height' in claim_result and claim_result['height'] is None:
|
||||
claim_result['height'] = -1
|
||||
|
||||
if 'amount' in claim_result:
|
||||
claim_result = format_amount_value(claim_result)
|
||||
|
||||
claim_result['permanent_url'] = _get_permanent_url(claim_result, decoded.certificate_id if decoded else None)
|
||||
|
||||
return claim_result
|
||||
|
||||
@staticmethod
|
||||
def prepare_claim_queries(start_position, query_size, channel_claim_infos):
|
||||
queries = [tuple()]
|
||||
names = {}
|
||||
# a table of index counts for the sorted claim ids, including ignored claims
|
||||
absolute_position_index = {}
|
||||
|
||||
block_sorted_infos = sorted(channel_claim_infos.items(), key=lambda x: int(x[1][1]))
|
||||
per_block_infos = {}
|
||||
for claim_id, (name, height) in block_sorted_infos:
|
||||
claims = per_block_infos.get(height, [])
|
||||
claims.append((claim_id, name))
|
||||
per_block_infos[height] = sorted(claims, key=lambda x: int(x[0], 16))
|
||||
|
||||
abs_position = 0
|
||||
|
||||
for height in sorted(per_block_infos.keys(), reverse=True):
|
||||
for claim_id, name in per_block_infos[height]:
|
||||
names[claim_id] = name
|
||||
absolute_position_index[claim_id] = abs_position
|
||||
if abs_position >= start_position:
|
||||
if len(queries[-1]) >= query_size:
|
||||
queries.append(tuple())
|
||||
queries[-1] += (claim_id,)
|
||||
abs_position += 1
|
||||
return queries, names, absolute_position_index
|
||||
|
||||
async def iter_channel_claims_pages(self, queries, claim_positions, claim_names, certificate,
|
||||
page_size=10):
|
||||
# lbryum server returns a dict of {claim_id: (name, claim_height)}
|
||||
# first, sort the claims by block height (and by claim id int value within a block).
|
||||
|
||||
# map the sorted claims into getclaimsbyids queries of query_size claim ids each
|
||||
|
||||
# send the batched queries to lbryum server and iteratively validate and parse
|
||||
# the results, yield a page of results at a time.
|
||||
|
||||
# these results can include those where `signature_is_valid` is False. if they are skipped,
|
||||
# page indexing becomes tricky, as the number of results isn't known until after having
|
||||
# processed them.
|
||||
# TODO: fix ^ in lbrynet.schema
|
||||
|
||||
async def iter_validate_channel_claims():
|
||||
formatted_claims = []
|
||||
for claim_ids in queries:
|
||||
batch_result = await self.network.get_claims_by_ids(*claim_ids)
|
||||
for claim_id in claim_ids:
|
||||
claim = batch_result[claim_id]
|
||||
if claim['name'] == claim_names[claim_id]:
|
||||
formatted_claim = await self.parse_and_validate_claim_result(claim, certificate)
|
||||
formatted_claim['absolute_channel_position'] = claim_positions[
|
||||
claim['claim_id']]
|
||||
formatted_claims.append(formatted_claim)
|
||||
else:
|
||||
log.warning("ignoring claim with name mismatch %s %s", claim['name'],
|
||||
claim['claim_id'])
|
||||
return formatted_claims
|
||||
|
||||
results = []
|
||||
|
||||
for claim in await iter_validate_channel_claims():
|
||||
results.append(claim)
|
||||
|
||||
# if there is a full page of results, yield it
|
||||
if len(results) and len(results) % page_size == 0:
|
||||
return results[-page_size:]
|
||||
|
||||
return results
|
||||
|
||||
async def get_channel_claims_page(self, channel_claim_infos, certificate, page, page_size=10):
|
||||
page = page or 0
|
||||
page_size = max(page_size, 1)
|
||||
if page_size > 500:
|
||||
raise Exception("page size above maximum allowed")
|
||||
start_position = (page - 1) * page_size
|
||||
queries, names, claim_positions = self.prepare_claim_queries(start_position, page_size,
|
||||
channel_claim_infos)
|
||||
page_generator = await self.iter_channel_claims_pages(queries, claim_positions, names,
|
||||
certificate, page_size=page_size)
|
||||
upper_bound = len(claim_positions)
|
||||
if not page:
|
||||
return None, upper_bound
|
||||
if start_position > upper_bound:
|
||||
raise IndexError("claim %i greater than max %i" % (start_position, upper_bound))
|
||||
return page_generator, upper_bound
|
||||
|
||||
|
||||
# Format amount to be decimal encoded string
|
||||
# Format value to be hex encoded string
|
||||
# TODO: refactor. Came from lbryum, there could be another part of torba doing it
|
||||
def format_amount_value(obj):
|
||||
if isinstance(obj, dict):
|
||||
for k, v in obj.items():
|
||||
if k in ('amount', 'effective_amount'):
|
||||
if not isinstance(obj[k], float):
|
||||
obj[k] = dewies_to_lbc(obj[k])
|
||||
elif k == 'supports' and isinstance(v, list):
|
||||
obj[k] = [{'txid': txid, 'nout': nout, 'amount': dewies_to_lbc(amount)}
|
||||
for (txid, nout, amount) in v]
|
||||
elif isinstance(v, (list, dict)):
|
||||
obj[k] = format_amount_value(v)
|
||||
elif isinstance(obj, list):
|
||||
obj = [format_amount_value(o) for o in obj]
|
||||
return obj
|
||||
|
||||
|
||||
def _get_permanent_url(claim_result, certificate_id):
|
||||
if certificate_id:
|
||||
return f"{claim_result['channel_name']}#{certificate_id}/{claim_result['name']}"
|
||||
else:
|
||||
return f"{claim_result['name']}#{claim_result['claim_id']}"
|
||||
|
||||
|
||||
def _verify_proof(name, claim_trie_root, result, height, depth, transaction_class, hash160_to_address):
|
||||
"""
|
||||
Verify proof for name claim
|
||||
"""
|
||||
|
||||
def _build_response(name, value, claim_id, txid, n, amount, effective_amount,
|
||||
claim_sequence, claim_address, supports):
|
||||
r = {
|
||||
'name': name,
|
||||
'value': hexlify(value),
|
||||
'claim_id': claim_id,
|
||||
'txid': txid,
|
||||
'nout': n,
|
||||
'amount': amount,
|
||||
'effective_amount': effective_amount,
|
||||
'height': height,
|
||||
'depth': depth,
|
||||
'claim_sequence': claim_sequence,
|
||||
'address': claim_address,
|
||||
'supports': supports
|
||||
}
|
||||
return r
|
||||
|
||||
def _parse_proof_result(name, result):
|
||||
support_amount = sum([amt for (stxid, snout, amt) in result['supports']])
|
||||
supports = result['supports']
|
||||
if 'txhash' in result['proof'] and 'nOut' in result['proof']:
|
||||
if 'transaction' in result:
|
||||
tx = transaction_class(raw=unhexlify(result['transaction']))
|
||||
nOut = result['proof']['nOut']
|
||||
if result['proof']['txhash'] == tx.id:
|
||||
if 0 <= nOut < len(tx.outputs):
|
||||
claim_output = tx.outputs[nOut]
|
||||
effective_amount = claim_output.amount + support_amount
|
||||
claim_address = hash160_to_address(claim_output.script.values['pubkey_hash'])
|
||||
claim_id = result['claim_id']
|
||||
claim_sequence = result['claim_sequence']
|
||||
claim_script = claim_output.script
|
||||
decoded_name = claim_script.values['claim_name'].decode()
|
||||
decoded_value = claim_script.values['claim']
|
||||
if decoded_name == name:
|
||||
return _build_response(name, decoded_value, claim_id,
|
||||
tx.id, nOut, claim_output.amount,
|
||||
effective_amount, claim_sequence,
|
||||
claim_address, supports)
|
||||
return {'error': 'name in proof did not match requested name'}
|
||||
outputs = len(tx['outputs'])
|
||||
return {'error': 'invalid nOut: %d (let(outputs): %d' % (nOut, outputs)}
|
||||
return {'error': "computed txid did not match given transaction: %s vs %s" %
|
||||
(tx.id, result['proof']['txhash'])
|
||||
}
|
||||
return {'error': "didn't receive a transaction with the proof"}
|
||||
return {'error': 'name is not claimed'}
|
||||
|
||||
if 'proof' in result:
|
||||
try:
|
||||
verify_proof(result['proof'], claim_trie_root, name)
|
||||
except InvalidProofError:
|
||||
return {'error': "Proof was invalid"}
|
||||
return _parse_proof_result(name, result)
|
||||
else:
|
||||
return {'error': "proof not in result"}
|
||||
|
||||
|
||||
def validate_claim_signature_and_get_channel_name(claim, certificate_claim,
|
||||
claim_address, name, decoded_certificate=None):
|
||||
if not certificate_claim:
|
||||
return False, None
|
||||
if 'value' not in certificate_claim:
|
||||
log.warning('Got an invalid claim while parsing certificates, please report: %s', certificate_claim)
|
||||
return False, None
|
||||
certificate = decoded_certificate or smart_decode(certificate_claim['value'])
|
||||
if not isinstance(certificate, ClaimDict):
|
||||
raise TypeError("Certificate is not a ClaimDict: %s" % str(type(certificate)))
|
||||
if _validate_signed_claim(claim, claim_address, name, certificate):
|
||||
return True, certificate_claim['name']
|
||||
return False, None
|
||||
|
||||
|
||||
def _validate_signed_claim(claim, claim_address, name, certificate):
|
||||
if not claim.has_signature:
|
||||
raise Exception("Claim is not signed")
|
||||
if not is_address(claim_address):
|
||||
raise Exception("Not given a valid claim address")
|
||||
try:
|
||||
if claim.validate_signature(claim_address, certificate.protobuf, name):
|
||||
return True
|
||||
except BadSignatureError:
|
||||
# print_msg("Signature for %s is invalid" % claim_id)
|
||||
return False
|
||||
except Exception as err:
|
||||
log.error("Signature for %s is invalid, reason: %s - %s", claim_address,
|
||||
str(type(err)), err)
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
# TODO: The following came from code handling lbryum results. Now that it's all in one place a refactor should unify it.
|
||||
def _decode_claim_result(claim):
|
||||
if 'has_signature' in claim and claim['has_signature']:
|
||||
if not claim['signature_is_valid']:
|
||||
log.warning("lbry://%s#%s has an invalid signature",
|
||||
claim['name'], claim['claim_id'])
|
||||
if 'value' not in claim:
|
||||
log.warning('Got an invalid claim while parsing, please report: %s', claim)
|
||||
claim['hex'] = None
|
||||
claim['value'] = None
|
||||
claim['error'] = "Failed to parse: missing value"
|
||||
return claim
|
||||
try:
|
||||
decoded = smart_decode(claim['value'])
|
||||
claim_dict = decoded.claim_dict
|
||||
claim['value'] = claim_dict
|
||||
claim['hex'] = hexlify(decoded.serialized)
|
||||
except DecodeError:
|
||||
claim['hex'] = claim['value']
|
||||
claim['value'] = None
|
||||
claim['error'] = "Failed to decode value"
|
||||
return claim
|
||||
|
||||
|
||||
def _handle_claim_result(results):
|
||||
if not results:
|
||||
#TODO: cannot determine what name we searched for here
|
||||
# we should fix lbryum commands that return None
|
||||
raise UnknownNameError("")
|
||||
|
||||
if 'error' in results:
|
||||
if results['error'] in ['name is not claimed', 'claim not found']:
|
||||
if 'claim_id' in results:
|
||||
raise UnknownClaimID(results['claim_id'])
|
||||
elif 'name' in results:
|
||||
raise UnknownNameError(results['name'])
|
||||
elif 'uri' in results:
|
||||
raise UnknownURI(results['uri'])
|
||||
elif 'outpoint' in results:
|
||||
raise UnknownOutpoint(results['outpoint'])
|
||||
raise Exception(results['error'])
|
||||
|
||||
# case where return value is {'certificate':{'txid', 'value',...},...}
|
||||
if 'certificate' in results:
|
||||
results['certificate'] = _decode_claim_result(results['certificate'])
|
||||
|
||||
# case where return value is {'claim':{'txid','value',...},...}
|
||||
if 'claim' in results:
|
||||
results['claim'] = _decode_claim_result(results['claim'])
|
||||
|
||||
# case where return value is {'txid','value',...}
|
||||
# returned by queries that are not name resolve related
|
||||
# (getclaimbyoutpoint, getclaimbyid, getclaimsfromtx)
|
||||
elif 'value' in results:
|
||||
results = _decode_claim_result(results)
|
||||
|
||||
# case where there is no 'certificate', 'value', or 'claim' key
|
||||
elif 'certificate' not in results:
|
||||
msg = f'result in unexpected format:{results}'
|
||||
assert False, msg
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def pick_winner_from_channel_path_collision(claims_in_channel):
|
||||
# we should be doing this by effective amount so we pick the controlling claim, however changing the resolved
|
||||
# claim triggers another issue where 2 claims cant be saved for the same file. This code picks the oldest, so it
|
||||
# stays the same. Using effective amount would change the resolved claims for a channel path on takeovers,
|
||||
# potentially triggering that.
|
||||
winner = None
|
||||
for claim in claims_in_channel:
|
||||
if not claim['signature_is_valid']:
|
||||
continue
|
||||
if winner is None:
|
||||
winner = claim
|
||||
elif claim['claim_sequence'] < winner['claim_sequence']:
|
||||
winner = claim
|
||||
return winner
|
||||
@@ -1,47 +0,0 @@
|
||||
from collections import namedtuple
|
||||
import msgpack
|
||||
from torba.server.util import cachedproperty
|
||||
# Classes representing data and their serializers, if any.
|
||||
|
||||
|
||||
class ClaimInfo(namedtuple("NameClaim", "name value txid nout amount address height cert_id")):
|
||||
'''Claim information as its stored on database'''
|
||||
|
||||
@classmethod
|
||||
def from_serialized(cls, serialized):
|
||||
return cls(*msgpack.loads(serialized))
|
||||
|
||||
@property
|
||||
def serialized(self):
|
||||
return msgpack.dumps(self)
|
||||
|
||||
|
||||
class NameClaim(namedtuple("NameClaim", "name value")):
|
||||
pass
|
||||
|
||||
|
||||
class ClaimUpdate(namedtuple("ClaimUpdate", "name claim_id value")):
|
||||
pass
|
||||
|
||||
|
||||
class ClaimSupport(namedtuple("ClaimSupport", "name claim_id")):
|
||||
pass
|
||||
|
||||
|
||||
class LBRYTx(namedtuple("Tx", "version inputs outputs locktime")):
|
||||
'''Transaction that can contain claim, update or support in its outputs.'''
|
||||
|
||||
@cachedproperty
|
||||
def is_coinbase(self):
|
||||
return self.inputs[0].is_coinbase
|
||||
|
||||
@cachedproperty
|
||||
def has_claims(self):
|
||||
for output in self.outputs:
|
||||
if output.claim:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class TxClaimOutput(namedtuple("TxClaimOutput", "value pk_script claim")):
|
||||
pass
|
||||
@@ -1,126 +0,0 @@
|
||||
import struct
|
||||
from torba.server.enum import Enumeration
|
||||
from lbrynet.extras.wallet.server.model import NameClaim, ClaimSupport, ClaimUpdate
|
||||
# TODO: Take this to lbryschema (it's also on lbryum and lbryum-server)
|
||||
|
||||
|
||||
opcodes = Enumeration("Opcodes", [
|
||||
("OP_0", 0), ("OP_PUSHDATA1", 76), "OP_PUSHDATA2", "OP_PUSHDATA4", "OP_1NEGATE", "OP_RESERVED",
|
||||
"OP_1", "OP_2", "OP_3", "OP_4", "OP_5", "OP_6", "OP_7",
|
||||
"OP_8", "OP_9", "OP_10", "OP_11", "OP_12", "OP_13", "OP_14", "OP_15", "OP_16",
|
||||
"OP_NOP", "OP_VER", "OP_IF", "OP_NOTIF", "OP_VERIF", "OP_VERNOTIF", "OP_ELSE", "OP_ENDIF",
|
||||
"OP_VERIFY",
|
||||
"OP_RETURN", "OP_TOALTSTACK", "OP_FROMALTSTACK", "OP_2DROP", "OP_2DUP", "OP_3DUP", "OP_2OVER",
|
||||
"OP_2ROT", "OP_2SWAP",
|
||||
"OP_IFDUP", "OP_DEPTH", "OP_DROP", "OP_DUP", "OP_NIP", "OP_OVER", "OP_PICK", "OP_ROLL",
|
||||
"OP_ROT",
|
||||
"OP_SWAP", "OP_TUCK", "OP_CAT", "OP_SUBSTR", "OP_LEFT", "OP_RIGHT", "OP_SIZE", "OP_INVERT",
|
||||
"OP_AND",
|
||||
"OP_OR", "OP_XOR", "OP_EQUAL", "OP_EQUALVERIFY", "OP_RESERVED1", "OP_RESERVED2", "OP_1ADD",
|
||||
"OP_1SUB", "OP_2MUL",
|
||||
"OP_2DIV", "OP_NEGATE", "OP_ABS", "OP_NOT", "OP_0NOTEQUAL", "OP_ADD", "OP_SUB", "OP_MUL",
|
||||
"OP_DIV",
|
||||
"OP_MOD", "OP_LSHIFT", "OP_RSHIFT", "OP_BOOLAND", "OP_BOOLOR",
|
||||
"OP_NUMEQUAL", "OP_NUMEQUALVERIFY", "OP_NUMNOTEQUAL", "OP_LESSTHAN",
|
||||
"OP_GREATERTHAN", "OP_LESSTHANOREQUAL", "OP_GREATERTHANOREQUAL", "OP_MIN", "OP_MAX",
|
||||
"OP_WITHIN", "OP_RIPEMD160", "OP_SHA1", "OP_SHA256", "OP_HASH160",
|
||||
"OP_HASH256", "OP_CODESEPARATOR", "OP_CHECKSIG", "OP_CHECKSIGVERIFY", "OP_CHECKMULTISIG",
|
||||
"OP_CHECKMULTISIGVERIFY", "OP_NOP1", "OP_NOP2", "OP_NOP3", "OP_NOP4", "OP_NOP5",
|
||||
"OP_CLAIM_NAME",
|
||||
"OP_SUPPORT_CLAIM", "OP_UPDATE_CLAIM",
|
||||
("OP_SINGLEBYTE_END", 0xF0),
|
||||
("OP_DOUBLEBYTE_BEGIN", 0xF000),
|
||||
"OP_PUBKEY", "OP_PUBKEYHASH",
|
||||
("OP_INVALIDOPCODE", 0xFFFF),
|
||||
])
|
||||
|
||||
|
||||
def script_GetOp(bytes):
|
||||
i = 0
|
||||
while i < len(bytes):
|
||||
vch = None
|
||||
opcode = bytes[i]
|
||||
i += 1
|
||||
if opcode <= opcodes.OP_PUSHDATA4:
|
||||
nSize = opcode
|
||||
if opcode == opcodes.OP_PUSHDATA1:
|
||||
nSize = bytes[i]
|
||||
i += 1
|
||||
elif opcode == opcodes.OP_PUSHDATA2:
|
||||
(nSize,) = struct.unpack_from('<H', bytes, i)
|
||||
i += 2
|
||||
elif opcode == opcodes.OP_PUSHDATA4:
|
||||
(nSize,) = struct.unpack_from('<I', bytes, i)
|
||||
i += 4
|
||||
if i + nSize > len(bytes):
|
||||
vch = "_INVALID_" + bytes[i:]
|
||||
i = len(bytes)
|
||||
else:
|
||||
vch = bytes[i:i + nSize]
|
||||
i += nSize
|
||||
yield (opcode, vch, i)
|
||||
|
||||
|
||||
def decode_claim_script(bytes_script):
|
||||
try:
|
||||
decoded_script = [x for x in script_GetOp(bytes_script)]
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return None
|
||||
if len(decoded_script) <= 6:
|
||||
return False
|
||||
op = 0
|
||||
claim_type = decoded_script[op][0]
|
||||
if claim_type == opcodes.OP_UPDATE_CLAIM:
|
||||
if len(decoded_script) <= 7:
|
||||
return False
|
||||
if claim_type not in [
|
||||
opcodes.OP_CLAIM_NAME,
|
||||
opcodes.OP_SUPPORT_CLAIM,
|
||||
opcodes.OP_UPDATE_CLAIM
|
||||
]:
|
||||
return False
|
||||
op += 1
|
||||
value = None
|
||||
claim_id = None
|
||||
claim = None
|
||||
if not 0 <= decoded_script[op][0] <= opcodes.OP_PUSHDATA4:
|
||||
return False
|
||||
name = decoded_script[op][1]
|
||||
op += 1
|
||||
if not 0 <= decoded_script[op][0] <= opcodes.OP_PUSHDATA4:
|
||||
return False
|
||||
if decoded_script[0][0] in [
|
||||
opcodes.OP_SUPPORT_CLAIM,
|
||||
opcodes.OP_UPDATE_CLAIM
|
||||
]:
|
||||
claim_id = decoded_script[op][1]
|
||||
if len(claim_id) != 20:
|
||||
return False
|
||||
else:
|
||||
value = decoded_script[op][1]
|
||||
op += 1
|
||||
if decoded_script[0][0] == opcodes.OP_UPDATE_CLAIM:
|
||||
value = decoded_script[op][1]
|
||||
op += 1
|
||||
if decoded_script[op][0] != opcodes.OP_2DROP:
|
||||
return False
|
||||
op += 1
|
||||
if decoded_script[op][0] != opcodes.OP_DROP and decoded_script[0][0] == opcodes.OP_CLAIM_NAME:
|
||||
return False
|
||||
elif decoded_script[op][0] != opcodes.OP_2DROP and decoded_script[0][0] == opcodes.OP_UPDATE_CLAIM:
|
||||
return False
|
||||
op += 1
|
||||
if decoded_script[0][0] == opcodes.OP_CLAIM_NAME:
|
||||
if name is None or value is None:
|
||||
return False
|
||||
claim = NameClaim(name, value)
|
||||
elif decoded_script[0][0] == opcodes.OP_UPDATE_CLAIM:
|
||||
if name is None or value is None or claim_id is None:
|
||||
return False
|
||||
claim = ClaimUpdate(name, claim_id, value)
|
||||
elif decoded_script[0][0] == opcodes.OP_SUPPORT_CLAIM:
|
||||
if name is None or claim_id is None:
|
||||
return False
|
||||
claim = ClaimSupport(name, claim_id)
|
||||
return claim, decoded_script[op:]
|
||||
@@ -1,21 +0,0 @@
|
||||
from torba.server.tx import Deserializer
|
||||
from lbrynet.extras.wallet.server.opcodes import decode_claim_script
|
||||
from lbrynet.extras.wallet.server.model import TxClaimOutput, LBRYTx
|
||||
|
||||
|
||||
class LBRYDeserializer(Deserializer):
|
||||
|
||||
def _read_output(self):
|
||||
value = self._read_le_int64()
|
||||
script = self._read_varbytes() # pk_script
|
||||
claim = decode_claim_script(script)
|
||||
claim = claim[0] if claim else None
|
||||
return TxClaimOutput(value, script, claim)
|
||||
|
||||
def read_tx(self):
|
||||
return LBRYTx(
|
||||
self._read_le_int32(), # version
|
||||
self._read_inputs(), # inputs
|
||||
self._read_outputs(), # outputs
|
||||
self._read_le_uint32() # locktime
|
||||
)
|
||||
@@ -1,6 +1,5 @@
|
||||
build:
|
||||
rm proto/* -rf
|
||||
cd proto && protoc --python_out=. -I ../../../../types/proto/ ../../../../types/proto/*.proto
|
||||
sed -e 's/^import\ \(.*\)_pb2\ /from . import\ \1_pb2\ /g' -i proto/*.py
|
||||
touch proto/__init__.py
|
||||
|
||||
rm types/v2/* -rf
|
||||
touch types/v2/__init__.py
|
||||
cd types/v2/ && protoc --python_out=. -I ../../../../../types/v2/proto/ ../../../../../types/v2/proto/*.proto
|
||||
sed -e 's/^import\ \(.*\)_pb2\ /from . import\ \1_pb2\ /g' -i types/v2/*.py
|
||||
@@ -1,13 +1 @@
|
||||
import os
|
||||
|
||||
__version__ = "0.0.16"
|
||||
|
||||
BLOCKCHAIN_NAME_ENVVAR = "LBRYSCHEMA_BLOCKCHAIN_NAME"
|
||||
if BLOCKCHAIN_NAME_ENVVAR in os.environ:
|
||||
if os.environ[BLOCKCHAIN_NAME_ENVVAR] in ['lbrycrd_main', 'lbrycrd_regtest',
|
||||
'lbrycrd_testnet']:
|
||||
BLOCKCHAIN_NAME = os.environ[BLOCKCHAIN_NAME_ENVVAR]
|
||||
else:
|
||||
raise OSError("invalid blockchain name: %s" % os.environ[BLOCKCHAIN_NAME_ENVVAR])
|
||||
else:
|
||||
BLOCKCHAIN_NAME = "lbrycrd_main"
|
||||
from .claim import Claim
|
||||
@@ -1,64 +0,0 @@
|
||||
import lbrynet.schema
|
||||
from lbrynet.schema.base import b58encode, b58decode, validate_b58_checksum
|
||||
from lbrynet.schema.hashing import double_sha256, hash160
|
||||
from lbrynet.schema.error import InvalidAddress
|
||||
from lbrynet.schema.schema import ADDRESS_LENGTH, ADDRESS_PREFIXES, PUBKEY_ADDRESS, SCRIPT_ADDRESS
|
||||
|
||||
|
||||
def validate_address_length(addr_bytes):
|
||||
if len(addr_bytes) != ADDRESS_LENGTH:
|
||||
raise InvalidAddress("Invalid address length: %i" % len(addr_bytes))
|
||||
|
||||
|
||||
def validate_address_prefix(addr_bytes):
|
||||
prefix = addr_bytes[0]
|
||||
if prefix not in ADDRESS_PREFIXES[lbrynet.schema.BLOCKCHAIN_NAME].values():
|
||||
raise InvalidAddress("Invalid address prefix: %.2X" % prefix)
|
||||
|
||||
|
||||
def validate_lbrycrd_address_bytes(addr_bytes):
|
||||
validate_address_length(addr_bytes)
|
||||
validate_address_prefix(addr_bytes)
|
||||
validate_b58_checksum(addr_bytes)
|
||||
return addr_bytes
|
||||
|
||||
|
||||
def decode_address(v):
|
||||
"""decode and validate a b58 address"""
|
||||
return validate_lbrycrd_address_bytes(b58decode(v))
|
||||
|
||||
|
||||
def encode_address(addr_bytes):
|
||||
"""validate and encode an address as b58"""
|
||||
v = validate_lbrycrd_address_bytes(addr_bytes)
|
||||
return b58encode(v)
|
||||
|
||||
|
||||
def hash_160_bytes_to_address(h160, addrtype=PUBKEY_ADDRESS):
|
||||
if addrtype == PUBKEY_ADDRESS:
|
||||
prefix = chr(ADDRESS_PREFIXES[lbrynet.schema.BLOCKCHAIN_NAME][PUBKEY_ADDRESS])
|
||||
elif addrtype == SCRIPT_ADDRESS:
|
||||
prefix = chr(ADDRESS_PREFIXES[lbrynet.schema.BLOCKCHAIN_NAME][SCRIPT_ADDRESS])
|
||||
else:
|
||||
raise Exception("Invalid address prefix")
|
||||
return b58encode(prefix + h160 + double_sha256(prefix + h160)[0:4])
|
||||
|
||||
|
||||
def public_key_to_address(public_key):
|
||||
return hash_160_bytes_to_address(hash160(public_key))
|
||||
|
||||
|
||||
def address_to_hash_160(addr):
|
||||
bytes = decode_address(addr)
|
||||
prefix, pubkey_bytes, addr_checksum = bytes[0], bytes[1:21], bytes[21:]
|
||||
if prefix == chr(ADDRESS_PREFIXES[lbrynet.schema.BLOCKCHAIN_NAME][PUBKEY_ADDRESS]):
|
||||
return PUBKEY_ADDRESS, pubkey_bytes
|
||||
return SCRIPT_ADDRESS, pubkey_bytes
|
||||
|
||||
|
||||
def is_address(addr):
|
||||
try:
|
||||
addr_bytes = decode_address(addr)
|
||||
return True
|
||||
except InvalidAddress:
|
||||
return False
|
||||
@@ -0,0 +1,521 @@
|
||||
import json
|
||||
import logging
|
||||
import os.path
|
||||
import hashlib
|
||||
from typing import Tuple, List
|
||||
from string import ascii_letters
|
||||
from decimal import Decimal, ROUND_UP
|
||||
from binascii import hexlify, unhexlify
|
||||
from google.protobuf.json_format import MessageToDict
|
||||
|
||||
from torba.client.hash import Base58
|
||||
from torba.client.constants import COIN
|
||||
|
||||
from lbrynet.schema.mime_types import guess_media_type
|
||||
from lbrynet.schema.base import Metadata, BaseMessageList
|
||||
from lbrynet.schema.types.v2.claim_pb2 import (
|
||||
Fee as FeeMessage,
|
||||
Location as LocationMessage,
|
||||
Language as LanguageMessage
|
||||
)
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def calculate_sha384_file_hash(file_path):
|
||||
sha384 = hashlib.sha384()
|
||||
with open(file_path, 'rb') as f:
|
||||
for chunk in iter(lambda: f.read(128 * sha384.block_size), b''):
|
||||
sha384.update(chunk)
|
||||
return sha384.digest()
|
||||
|
||||
|
||||
class Dimmensional(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def width(self) -> int:
|
||||
return self.message.width
|
||||
|
||||
@width.setter
|
||||
def width(self, width: int):
|
||||
self.message.width = width
|
||||
|
||||
@property
|
||||
def height(self) -> int:
|
||||
return self.message.height
|
||||
|
||||
@height.setter
|
||||
def height(self, height: int):
|
||||
self.message.height = height
|
||||
|
||||
@property
|
||||
def dimensions(self) -> Tuple[int, int]:
|
||||
return self.width, self.height
|
||||
|
||||
@dimensions.setter
|
||||
def dimensions(self, dimensions: Tuple[int, int]):
|
||||
self.message.width, self.message.height = dimensions
|
||||
|
||||
def _extract(self, file_metadata, field):
|
||||
try:
|
||||
setattr(self, field, file_metadata.getValues(field)[0])
|
||||
except:
|
||||
log.exception(f'Could not extract {field} from file metadata.')
|
||||
|
||||
def update(self, file_metadata=None, height=None, width=None):
|
||||
if height is not None:
|
||||
self.height = height
|
||||
elif file_metadata:
|
||||
self._extract(file_metadata, 'height')
|
||||
|
||||
if width is not None:
|
||||
self.width = width
|
||||
elif file_metadata:
|
||||
self._extract(file_metadata, 'width')
|
||||
|
||||
|
||||
class Playable(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def duration(self) -> int:
|
||||
return self.message.duration
|
||||
|
||||
@duration.setter
|
||||
def duration(self, duration: int):
|
||||
self.message.duration = duration
|
||||
|
||||
def update(self, file_metadata=None, duration=None):
|
||||
if duration is not None:
|
||||
self.duration = duration
|
||||
elif file_metadata:
|
||||
try:
|
||||
self.duration = file_metadata.getValues('duration')[0].seconds
|
||||
except:
|
||||
log.exception('Could not extract duration from file metadata.')
|
||||
|
||||
|
||||
class Image(Dimmensional):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
|
||||
class Audio(Playable):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
|
||||
class Video(Dimmensional, Playable):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def update(self, file_metadata=None, height=None, width=None, duration=None):
|
||||
Dimmensional.update(self, file_metadata, height, width)
|
||||
Playable.update(self, file_metadata, duration)
|
||||
|
||||
|
||||
class Source(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def update(self, file_path=None):
|
||||
if file_path is not None:
|
||||
self.name = os.path.basename(file_path)
|
||||
self.media_type, stream_type = guess_media_type(file_path)
|
||||
if not os.path.isfile(file_path):
|
||||
raise Exception(f"File does not exist: {file_path}")
|
||||
self.size = os.path.getsize(file_path)
|
||||
if self.size == 0:
|
||||
raise Exception(f"Cannot publish empty file: {file_path}")
|
||||
self.file_hash_bytes = calculate_sha384_file_hash(file_path)
|
||||
return stream_type
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self.message.name
|
||||
|
||||
@name.setter
|
||||
def name(self, name: str):
|
||||
self.message.name = name
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
return self.message.size
|
||||
|
||||
@size.setter
|
||||
def size(self, size: int):
|
||||
self.message.size = size
|
||||
|
||||
@property
|
||||
def media_type(self) -> str:
|
||||
return self.message.media_type
|
||||
|
||||
@media_type.setter
|
||||
def media_type(self, media_type: str):
|
||||
self.message.media_type = media_type
|
||||
|
||||
@property
|
||||
def file_hash(self) -> str:
|
||||
return hexlify(self.message.hash).decode()
|
||||
|
||||
@file_hash.setter
|
||||
def file_hash(self, file_hash: str):
|
||||
self.message.hash = unhexlify(file_hash.encode())
|
||||
|
||||
@property
|
||||
def file_hash_bytes(self) -> bytes:
|
||||
return self.message.hash
|
||||
|
||||
@file_hash_bytes.setter
|
||||
def file_hash_bytes(self, file_hash_bytes: bytes):
|
||||
self.message.hash = file_hash_bytes
|
||||
|
||||
@property
|
||||
def sd_hash(self) -> str:
|
||||
return hexlify(self.message.sd_hash).decode()
|
||||
|
||||
@sd_hash.setter
|
||||
def sd_hash(self, sd_hash: str):
|
||||
self.message.sd_hash = unhexlify(sd_hash.encode())
|
||||
|
||||
@property
|
||||
def sd_hash_bytes(self) -> bytes:
|
||||
return self.message.sd_hash
|
||||
|
||||
@sd_hash_bytes.setter
|
||||
def sd_hash_bytes(self, sd_hash: bytes):
|
||||
self.message.sd_hash = sd_hash
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
return self.message.url
|
||||
|
||||
@url.setter
|
||||
def url(self, url: str):
|
||||
self.message.url = url
|
||||
|
||||
|
||||
class Fee(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def update(self, address: str = None, currency: str = None, amount=None):
|
||||
if address is not None:
|
||||
self.address = address
|
||||
if currency is not None and amount is not None:
|
||||
currency = currency.lower()
|
||||
assert currency in ('lbc', 'btc', 'usd'), f'Unknown currency type: {currency}'
|
||||
setattr(self, currency, Decimal(amount))
|
||||
|
||||
@property
|
||||
def currency(self) -> str:
|
||||
return FeeMessage.Currency.Name(self.message.currency)
|
||||
|
||||
@property
|
||||
def address(self) -> str:
|
||||
return Base58.encode(self.message.address)
|
||||
|
||||
@address.setter
|
||||
def address(self, address: str):
|
||||
self.message.address = Base58.decode(address)
|
||||
|
||||
@property
|
||||
def address_bytes(self) -> bytes:
|
||||
return self.message.address
|
||||
|
||||
@address_bytes.setter
|
||||
def address_bytes(self, address: bytes):
|
||||
self.message.address = address
|
||||
|
||||
@property
|
||||
def amount(self) -> Decimal:
|
||||
if self.currency == 'LBC':
|
||||
return self.lbc
|
||||
if self.currency == 'BTC':
|
||||
return self.btc
|
||||
if self.currency == 'USD':
|
||||
return self.usd
|
||||
|
||||
DEWIES = Decimal(COIN)
|
||||
|
||||
@property
|
||||
def lbc(self) -> Decimal:
|
||||
if self.message.currency != FeeMessage.LBC:
|
||||
raise ValueError('LBC can only be returned for LBC fees.')
|
||||
return Decimal(self.message.amount / self.DEWIES)
|
||||
|
||||
@lbc.setter
|
||||
def lbc(self, amount: Decimal):
|
||||
self.dewies = int(amount * self.DEWIES)
|
||||
|
||||
@property
|
||||
def dewies(self) -> int:
|
||||
if self.message.currency != FeeMessage.LBC:
|
||||
raise ValueError('Dewies can only be returned for LBC fees.')
|
||||
return self.message.amount
|
||||
|
||||
@dewies.setter
|
||||
def dewies(self, amount: int):
|
||||
self.message.amount = amount
|
||||
self.message.currency = FeeMessage.LBC
|
||||
|
||||
SATOSHIES = Decimal(COIN)
|
||||
|
||||
@property
|
||||
def btc(self) -> Decimal:
|
||||
if self.message.currency != FeeMessage.BTC:
|
||||
raise ValueError('BTC can only be returned for BTC fees.')
|
||||
return Decimal(self.message.amount / self.SATOSHIES)
|
||||
|
||||
@btc.setter
|
||||
def btc(self, amount: Decimal):
|
||||
self.satoshis = int(amount * self.SATOSHIES)
|
||||
|
||||
@property
|
||||
def satoshis(self) -> int:
|
||||
if self.message.currency != FeeMessage.BTC:
|
||||
raise ValueError('Satoshies can only be returned for BTC fees.')
|
||||
return self.message.amount
|
||||
|
||||
@satoshis.setter
|
||||
def satoshis(self, amount: int):
|
||||
self.message.amount = amount
|
||||
self.message.currency = FeeMessage.BTC
|
||||
|
||||
PENNIES = Decimal('100.0')
|
||||
PENNY = Decimal('0.01')
|
||||
|
||||
@property
|
||||
def usd(self) -> Decimal:
|
||||
if self.message.currency != FeeMessage.USD:
|
||||
raise ValueError('USD can only be returned for USD fees.')
|
||||
return Decimal(self.message.amount / self.PENNIES)
|
||||
|
||||
@usd.setter
|
||||
def usd(self, amount: Decimal):
|
||||
self.pennies = int(amount.quantize(self.PENNY, ROUND_UP) * self.PENNIES)
|
||||
|
||||
@property
|
||||
def pennies(self) -> int:
|
||||
if self.message.currency != FeeMessage.USD:
|
||||
raise ValueError('Pennies can only be returned for USD fees.')
|
||||
return self.message.amount
|
||||
|
||||
@pennies.setter
|
||||
def pennies(self, amount: int):
|
||||
self.message.amount = amount
|
||||
self.message.currency = FeeMessage.USD
|
||||
|
||||
|
||||
class ClaimReference(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def claim_id(self) -> str:
|
||||
return hexlify(self.claim_hash[::-1]).decode()
|
||||
|
||||
@claim_id.setter
|
||||
def claim_id(self, claim_id: str):
|
||||
self.claim_hash = unhexlify(claim_id)[::-1]
|
||||
|
||||
@property
|
||||
def claim_hash(self) -> bytes:
|
||||
return self.message.claim_hash
|
||||
|
||||
@claim_hash.setter
|
||||
def claim_hash(self, claim_hash: bytes):
|
||||
self.message.claim_hash = claim_hash
|
||||
|
||||
|
||||
class ClaimList(BaseMessageList[ClaimReference]):
|
||||
|
||||
__slots__ = ()
|
||||
item_class = ClaimReference
|
||||
|
||||
@property
|
||||
def _message(self):
|
||||
return self.message.claim_references
|
||||
|
||||
def append(self, value):
|
||||
self.add().claim_id = value
|
||||
|
||||
@property
|
||||
def ids(self) -> List[str]:
|
||||
return [c.claim_id for c in self]
|
||||
|
||||
|
||||
class Language(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def langtag(self) -> str:
|
||||
langtag = []
|
||||
if self.language:
|
||||
langtag.append(self.language)
|
||||
if self.script:
|
||||
langtag.append(self.script)
|
||||
if self.region:
|
||||
langtag.append(self.region)
|
||||
return '-'.join(langtag)
|
||||
|
||||
@langtag.setter
|
||||
def langtag(self, langtag: str):
|
||||
parts = langtag.split('-')
|
||||
self.language = parts.pop(0)
|
||||
if parts and len(parts[0]) == 4:
|
||||
self.script = parts.pop(0)
|
||||
if parts and len(parts[0]) == 2:
|
||||
self.region = parts.pop(0)
|
||||
assert not parts, f"Failed to parse language tag: {langtag}"
|
||||
|
||||
@property
|
||||
def language(self) -> str:
|
||||
if self.message.language:
|
||||
return LanguageMessage.Language.Name(self.message.language)
|
||||
|
||||
@language.setter
|
||||
def language(self, language: str):
|
||||
self.message.language = LanguageMessage.Language.Value(language)
|
||||
|
||||
@property
|
||||
def script(self) -> str:
|
||||
if self.message.script:
|
||||
return LanguageMessage.Script.Name(self.message.script)
|
||||
|
||||
@script.setter
|
||||
def script(self, script: str):
|
||||
self.message.script = LanguageMessage.Script.Value(script)
|
||||
|
||||
@property
|
||||
def region(self) -> str:
|
||||
if self.message.region:
|
||||
return LocationMessage.Country.Name(self.message.region)
|
||||
|
||||
@region.setter
|
||||
def region(self, region: str):
|
||||
self.message.region = LocationMessage.Country.Value(region)
|
||||
|
||||
|
||||
class LanguageList(BaseMessageList[Language]):
|
||||
__slots__ = ()
|
||||
item_class = Language
|
||||
|
||||
def append(self, value: str):
|
||||
self.add().langtag = value
|
||||
|
||||
|
||||
class Location(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def from_value(self, value):
|
||||
if isinstance(value, str) and value.startswith('{'):
|
||||
value = json.loads(value)
|
||||
|
||||
if isinstance(value, dict):
|
||||
for key, val in value.items():
|
||||
setattr(self, key, val)
|
||||
|
||||
elif isinstance(value, str):
|
||||
parts = value.split(':')
|
||||
if len(parts) > 2 or (parts[0] and parts[0][0] in ascii_letters):
|
||||
country = parts and parts.pop(0)
|
||||
if country:
|
||||
self.country = country
|
||||
state = parts and parts.pop(0)
|
||||
if state:
|
||||
self.state = state
|
||||
city = parts and parts.pop(0)
|
||||
if city:
|
||||
self.city = city
|
||||
code = parts and parts.pop(0)
|
||||
if code:
|
||||
self.code = code
|
||||
latitude = parts and parts.pop(0)
|
||||
if latitude:
|
||||
self.latitude = latitude
|
||||
longitude = parts and parts.pop(0)
|
||||
if longitude:
|
||||
self.longitude = longitude
|
||||
|
||||
else:
|
||||
raise ValueError(f'Could not parse country value: {value}')
|
||||
|
||||
def to_dict(self):
|
||||
d = MessageToDict(self.message)
|
||||
if self.message.longitude:
|
||||
d['longitude'] = self.longitude
|
||||
if self.message.latitude:
|
||||
d['latitude'] = self.latitude
|
||||
return d
|
||||
|
||||
@property
|
||||
def country(self) -> str:
|
||||
if self.message.country:
|
||||
return LocationMessage.Country.Name(self.message.country)
|
||||
|
||||
@country.setter
|
||||
def country(self, country: str):
|
||||
self.message.country = LocationMessage.Country.Value(country)
|
||||
|
||||
@property
|
||||
def state(self) -> str:
|
||||
return self.message.state
|
||||
|
||||
@state.setter
|
||||
def state(self, state: str):
|
||||
self.message.state = state
|
||||
|
||||
@property
|
||||
def city(self) -> str:
|
||||
return self.message.city
|
||||
|
||||
@city.setter
|
||||
def city(self, city: str):
|
||||
self.message.city = city
|
||||
|
||||
@property
|
||||
def code(self) -> str:
|
||||
return self.message.code
|
||||
|
||||
@code.setter
|
||||
def code(self, code: str):
|
||||
self.message.code = code
|
||||
|
||||
GPS_PRECISION = Decimal('10000000')
|
||||
|
||||
@property
|
||||
def latitude(self) -> str:
|
||||
if self.message.latitude:
|
||||
return str(Decimal(self.message.latitude) / self.GPS_PRECISION)
|
||||
|
||||
@latitude.setter
|
||||
def latitude(self, latitude: str):
|
||||
latitude = Decimal(latitude)
|
||||
assert -90 <= latitude <= 90, "Latitude must be between -90 and 90 degrees."
|
||||
self.message.latitude = int(latitude * self.GPS_PRECISION)
|
||||
|
||||
@property
|
||||
def longitude(self) -> str:
|
||||
if self.message.longitude:
|
||||
return str(Decimal(self.message.longitude) / self.GPS_PRECISION)
|
||||
|
||||
@longitude.setter
|
||||
def longitude(self, longitude: str):
|
||||
longitude = Decimal(longitude)
|
||||
assert -180 <= longitude <= 180, "Longitude must be between -180 and 180 degrees."
|
||||
self.message.longitude = int(longitude * self.GPS_PRECISION)
|
||||
|
||||
|
||||
class LocationList(BaseMessageList[Location]):
|
||||
__slots__ = ()
|
||||
item_class = Location
|
||||
|
||||
def append(self, value):
|
||||
self.add().from_value(value)
|
||||
+103
-72
@@ -1,90 +1,121 @@
|
||||
from lbrynet.schema.schema import ADDRESS_CHECKSUM_LENGTH
|
||||
from lbrynet.schema.hashing import double_sha256
|
||||
from lbrynet.schema.error import InvalidAddress
|
||||
from binascii import hexlify, unhexlify
|
||||
from typing import List, Iterator, TypeVar, Generic
|
||||
|
||||
from google.protobuf.message import DecodeError
|
||||
from google.protobuf.json_format import MessageToDict
|
||||
|
||||
|
||||
alphabet = b'123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
|
||||
class Signable:
|
||||
|
||||
__slots__ = (
|
||||
'message', 'version', 'signature',
|
||||
'signature_type', 'unsigned_payload', 'signing_channel_hash'
|
||||
)
|
||||
|
||||
message_class = None
|
||||
|
||||
def __init__(self, message=None):
|
||||
self.message = message or self.message_class()
|
||||
self.version = 2
|
||||
self.signature = None
|
||||
self.signature_type = 'SECP256k1'
|
||||
self.unsigned_payload = None
|
||||
self.signing_channel_hash = None
|
||||
|
||||
def clear_signature(self):
|
||||
self.signature = None
|
||||
self.unsigned_payload = None
|
||||
self.signing_channel_hash = None
|
||||
|
||||
@property
|
||||
def signing_channel_id(self):
|
||||
return hexlify(self.signing_channel_hash[::-1]).decode() if self.signing_channel_hash else None
|
||||
|
||||
@signing_channel_id.setter
|
||||
def signing_channel_id(self, channel_id: str):
|
||||
self.signing_channel_hash = unhexlify(channel_id)[::-1]
|
||||
|
||||
@property
|
||||
def is_signed(self):
|
||||
return self.signature is not None
|
||||
|
||||
def to_dict(self):
|
||||
return MessageToDict(self.message)
|
||||
|
||||
def to_message_bytes(self) -> bytes:
|
||||
return self.message.SerializeToString()
|
||||
|
||||
def to_bytes(self) -> bytes:
|
||||
pieces = bytearray()
|
||||
if self.is_signed:
|
||||
pieces.append(1)
|
||||
pieces.extend(self.signing_channel_hash)
|
||||
pieces.extend(self.signature)
|
||||
else:
|
||||
pieces.append(0)
|
||||
pieces.extend(self.to_message_bytes())
|
||||
return bytes(pieces)
|
||||
|
||||
@classmethod
|
||||
def from_bytes(cls, data: bytes):
|
||||
signable = cls()
|
||||
if data[0] == 0:
|
||||
signable.message.ParseFromString(data[1:])
|
||||
elif data[0] == 1:
|
||||
signable.signing_channel_hash = data[1:21]
|
||||
signable.signature = data[21:85]
|
||||
signable.message.ParseFromString(data[85:])
|
||||
else:
|
||||
raise DecodeError('Could not determine message format version.')
|
||||
return signable
|
||||
|
||||
def __len__(self):
|
||||
return len(self.to_bytes())
|
||||
|
||||
def __bytes__(self):
|
||||
return self.to_bytes()
|
||||
|
||||
|
||||
def scrub_input(v):
|
||||
if isinstance(v, str) and not isinstance(v, bytes):
|
||||
v = v.encode('ascii')
|
||||
return v
|
||||
class Metadata:
|
||||
|
||||
__slots__ = 'message',
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
|
||||
|
||||
def b58encode_int(i, default_one=True):
|
||||
'''Encode an integer using Base58'''
|
||||
if not i and default_one:
|
||||
return alphabet[0:1]
|
||||
string = b""
|
||||
while i:
|
||||
i, idx = divmod(i, 58)
|
||||
string = alphabet[idx:idx+1] + string
|
||||
return string
|
||||
I = TypeVar('I')
|
||||
|
||||
|
||||
def b58encode(v):
|
||||
'''Encode a string using Base58'''
|
||||
class BaseMessageList(Metadata, Generic[I]):
|
||||
|
||||
v = scrub_input(v)
|
||||
__slots__ = ()
|
||||
|
||||
nPad = len(v)
|
||||
v = v.lstrip(b'\0')
|
||||
nPad -= len(v)
|
||||
item_class = None
|
||||
|
||||
p, acc = 1, 0
|
||||
for c in reversed(v):
|
||||
acc += p * c
|
||||
p = p << 8
|
||||
@property
|
||||
def _message(self):
|
||||
return self.message
|
||||
|
||||
result = b58encode_int(acc, default_one=False)
|
||||
def add(self) -> I:
|
||||
return self.item_class(self._message.add())
|
||||
|
||||
return alphabet[0:1] * nPad + result
|
||||
def extend(self, values: List[str]):
|
||||
for value in values:
|
||||
self.append(value)
|
||||
|
||||
def append(self, value: str):
|
||||
raise NotImplemented
|
||||
|
||||
def b58decode_int(v):
|
||||
'''Decode a Base58 encoded string as an integer'''
|
||||
def __len__(self):
|
||||
return len(self._message)
|
||||
|
||||
v = scrub_input(v)
|
||||
def __iter__(self) -> Iterator[I]:
|
||||
for item in self._message:
|
||||
yield self.item_class(item)
|
||||
|
||||
decimal = 0
|
||||
for char in v:
|
||||
decimal = decimal * 58 + alphabet.index(char)
|
||||
return decimal
|
||||
def __getitem__(self, item) -> I:
|
||||
return self.item_class(self._message[item])
|
||||
|
||||
|
||||
def b58decode(v):
|
||||
'''Decode a Base58 encoded string'''
|
||||
|
||||
v = scrub_input(v)
|
||||
|
||||
origlen = len(v)
|
||||
v = v.lstrip(alphabet[0:1])
|
||||
newlen = len(v)
|
||||
|
||||
acc = b58decode_int(v)
|
||||
|
||||
result = []
|
||||
while acc > 0:
|
||||
acc, mod = divmod(acc, 256)
|
||||
result.append(mod)
|
||||
|
||||
return b'\0' * (origlen - newlen) + bytes(reversed(result))
|
||||
|
||||
|
||||
def validate_b58_checksum(addr_bytes):
|
||||
addr_without_checksum = addr_bytes[:-ADDRESS_CHECKSUM_LENGTH]
|
||||
addr_checksum = addr_bytes[-ADDRESS_CHECKSUM_LENGTH:]
|
||||
if double_sha256(addr_without_checksum)[:ADDRESS_CHECKSUM_LENGTH] != addr_checksum:
|
||||
raise InvalidAddress("Invalid address checksum")
|
||||
|
||||
|
||||
def b58decode_strip_checksum(v):
|
||||
addr_bytes = b58decode(v)
|
||||
validate_b58_checksum(addr_bytes)
|
||||
return addr_bytes[:-ADDRESS_CHECKSUM_LENGTH]
|
||||
|
||||
|
||||
def b58encode_with_checksum(addr_bytes):
|
||||
addr_checksum = double_sha256(addr_bytes)[:ADDRESS_CHECKSUM_LENGTH]
|
||||
return b58encode(addr_bytes + addr_checksum)
|
||||
def __delitem__(self, key):
|
||||
del self._message[key]
|
||||
+362
-162
@@ -1,198 +1,398 @@
|
||||
import json
|
||||
import binascii
|
||||
from google.protobuf import json_format # pylint: disable=no-name-in-module
|
||||
from google.protobuf.message import DecodeError as DecodeError_pb # pylint: disable=no-name-in-module,import-error
|
||||
import logging
|
||||
from typing import List
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
from collections import OrderedDict
|
||||
from google.protobuf.json_format import MessageToDict
|
||||
from google.protobuf.message import DecodeError
|
||||
from hachoir.core.log import log as hachoir_log
|
||||
from hachoir.parser import createParser as binary_file_parser
|
||||
from hachoir.metadata import extractMetadata as binary_file_metadata
|
||||
|
||||
from lbrynet.schema.schema.claim import Claim
|
||||
from lbrynet.schema.proto import claim_pb2
|
||||
from lbrynet.schema.signature import Signature
|
||||
from lbrynet.schema.validator import get_validator
|
||||
from lbrynet.schema.signer import get_signer
|
||||
from lbrynet.schema.schema import NIST256p, CURVE_NAMES, CLAIM_TYPE_NAMES, SECP256k1
|
||||
from lbrynet.schema.encoding import decode_fields, decode_b64_fields, encode_fields
|
||||
from lbrynet.schema.error import DecodeError
|
||||
from lbrynet.schema.fee import Fee
|
||||
from lbrynet.schema import compat
|
||||
from lbrynet.schema.base import Signable
|
||||
from lbrynet.schema.mime_types import guess_media_type, guess_stream_type
|
||||
from lbrynet.schema.attrs import (
|
||||
Source, Playable, Dimmensional, Fee, Image, Video, Audio,
|
||||
LanguageList, LocationList, ClaimList, ClaimReference
|
||||
)
|
||||
from lbrynet.schema.types.v2.claim_pb2 import Claim as ClaimMessage
|
||||
|
||||
|
||||
class ClaimDict(OrderedDict):
|
||||
def __init__(self, claim_dict=None, detached_signature: Signature=None):
|
||||
if isinstance(claim_dict, claim_pb2.Claim):
|
||||
raise Exception("To initialize %s with a Claim protobuf use %s.load_protobuf" %
|
||||
(self.__class__.__name__, self.__class__.__name__))
|
||||
self.detached_signature = detached_signature
|
||||
OrderedDict.__init__(self, claim_dict or [])
|
||||
hachoir_log.use_print = False
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Claim(Signable):
|
||||
|
||||
STREAM = 'stream'
|
||||
CHANNEL = 'channel'
|
||||
COLLECTION = 'collection'
|
||||
REPOST = 'repost'
|
||||
|
||||
__slots__ = 'version',
|
||||
|
||||
message_class = ClaimMessage
|
||||
|
||||
def __init__(self, message=None):
|
||||
super().__init__(message)
|
||||
self.version = 2
|
||||
|
||||
@property
|
||||
def protobuf_dict(self):
|
||||
"""Claim dictionary using base64 to represent bytes"""
|
||||
def claim_type(self) -> str:
|
||||
return self.message.WhichOneof('type')
|
||||
|
||||
return json.loads(json_format.MessageToJson(self.protobuf, True))
|
||||
|
||||
@property
|
||||
def protobuf(self):
|
||||
"""Claim message object"""
|
||||
|
||||
return Claim.load(self)
|
||||
|
||||
@property
|
||||
def serialized(self):
|
||||
"""Serialized Claim protobuf"""
|
||||
if self.detached_signature:
|
||||
return self.detached_signature.serialized
|
||||
return self.protobuf.SerializeToString()
|
||||
|
||||
@property
|
||||
def serialized_no_signature(self):
|
||||
"""Serialized Claim protobuf without publisherSignature field"""
|
||||
claim = self.protobuf
|
||||
claim.ClearField("publisherSignature")
|
||||
return ClaimDict.load_protobuf(claim).serialized
|
||||
|
||||
@property
|
||||
def has_signature(self):
|
||||
return self.protobuf.HasField("publisherSignature") or (
|
||||
self.detached_signature and self.detached_signature.raw_signature
|
||||
)
|
||||
|
||||
@property
|
||||
def is_certificate(self):
|
||||
claim = self.protobuf
|
||||
return CLAIM_TYPE_NAMES[claim.claimType] == "certificate"
|
||||
def get_message(self, type_name):
|
||||
message = getattr(self.message, type_name)
|
||||
if self.claim_type is None:
|
||||
message.SetInParent()
|
||||
if self.claim_type != type_name:
|
||||
raise ValueError(f'Claim is not a {type_name}.')
|
||||
return message
|
||||
|
||||
@property
|
||||
def is_stream(self):
|
||||
claim = self.protobuf
|
||||
return CLAIM_TYPE_NAMES[claim.claimType] == "stream"
|
||||
return self.claim_type == self.STREAM
|
||||
|
||||
@property
|
||||
def source_hash(self):
|
||||
claim = self.protobuf
|
||||
if not CLAIM_TYPE_NAMES[claim.claimType] == "stream":
|
||||
return None
|
||||
return binascii.hexlify(claim.stream.source.source)
|
||||
def stream(self) -> 'Stream':
|
||||
return Stream(self)
|
||||
|
||||
@property
|
||||
def has_fee(self):
|
||||
claim = self.protobuf
|
||||
if not CLAIM_TYPE_NAMES[claim.claimType] == "stream":
|
||||
return None
|
||||
if claim.stream.metadata.HasField("fee"):
|
||||
return True
|
||||
return False
|
||||
def is_channel(self):
|
||||
return self.claim_type == self.CHANNEL
|
||||
|
||||
@property
|
||||
def source_fee(self):
|
||||
claim = self.protobuf
|
||||
if not CLAIM_TYPE_NAMES[claim.claimType] == "stream":
|
||||
return None
|
||||
if claim.stream.metadata.HasField("fee"):
|
||||
return Fee.load_protobuf(claim.stream.metadata.fee)
|
||||
return None
|
||||
def channel(self) -> 'Channel':
|
||||
return Channel(self)
|
||||
|
||||
@property
|
||||
def certificate_id(self) -> str:
|
||||
if self.protobuf.HasField("publisherSignature"):
|
||||
return binascii.hexlify(self.protobuf.publisherSignature.certificateId).decode()
|
||||
if self.detached_signature and self.detached_signature.certificate_id:
|
||||
return binascii.hexlify(self.detached_signature.certificate_id).decode()
|
||||
def is_repost(self):
|
||||
return self.claim_type == self.REPOST
|
||||
|
||||
@property
|
||||
def signature(self):
|
||||
if not self.has_signature:
|
||||
return None
|
||||
return binascii.hexlify(self.protobuf.publisherSignature.signature)
|
||||
def repost(self) -> 'Repost':
|
||||
return Repost(self)
|
||||
|
||||
@property
|
||||
def protobuf_len(self):
|
||||
"""Length of serialized string"""
|
||||
|
||||
return self.protobuf.ByteSize()
|
||||
def is_collection(self):
|
||||
return self.claim_type == self.COLLECTION
|
||||
|
||||
@property
|
||||
def json_len(self):
|
||||
"""Length of json encoded string"""
|
||||
|
||||
return len(json.dumps(self.claim_dict))
|
||||
|
||||
@property
|
||||
def claim_dict(self):
|
||||
"""Claim dictionary with bytes represented as hex and base58"""
|
||||
|
||||
return dict(encode_fields(self, self.detached_signature))
|
||||
def collection(self) -> 'Collection':
|
||||
return Collection(self)
|
||||
|
||||
@classmethod
|
||||
def load_protobuf_dict(cls, protobuf_dict, detached_signature=None):
|
||||
"""
|
||||
Load a ClaimDict from a dictionary with base64 encoded bytes
|
||||
(as returned by the protobuf json formatter)
|
||||
"""
|
||||
|
||||
return cls(decode_b64_fields(protobuf_dict), detached_signature=detached_signature)
|
||||
|
||||
@classmethod
|
||||
def load_protobuf(cls, protobuf_claim, detached_signature=None):
|
||||
"""Load ClaimDict from a protobuf Claim message"""
|
||||
return cls.load_protobuf_dict(json.loads(json_format.MessageToJson(protobuf_claim, True)), detached_signature)
|
||||
|
||||
@classmethod
|
||||
def load_dict(cls, claim_dict):
|
||||
"""Load ClaimDict from a dictionary with hex and base58 encoded bytes"""
|
||||
def from_bytes(cls, data: bytes) -> 'Claim':
|
||||
try:
|
||||
claim_dict, detached_signature = decode_fields(claim_dict)
|
||||
return cls.load_protobuf(cls(claim_dict).protobuf, detached_signature)
|
||||
except json_format.ParseError as err:
|
||||
raise DecodeError(str(err))
|
||||
return super().from_bytes(data)
|
||||
except DecodeError:
|
||||
claim = cls()
|
||||
if data[0] == ord('{'):
|
||||
claim.version = 0
|
||||
compat.from_old_json_schema(claim, data)
|
||||
elif data[0] not in (0, 1):
|
||||
claim.version = 1
|
||||
compat.from_types_v1(claim, data)
|
||||
else:
|
||||
raise
|
||||
return claim
|
||||
|
||||
@classmethod
|
||||
def deserialize(cls, serialized):
|
||||
"""Load a ClaimDict from a serialized protobuf string"""
|
||||
detached_signature = Signature.flagged_parse(serialized)
|
||||
|
||||
temp_claim = claim_pb2.Claim()
|
||||
try:
|
||||
temp_claim.ParseFromString(detached_signature.payload)
|
||||
except DecodeError_pb:
|
||||
raise DecodeError(DecodeError_pb)
|
||||
return cls.load_protobuf(temp_claim, detached_signature=detached_signature)
|
||||
class BaseClaim:
|
||||
|
||||
@classmethod
|
||||
def generate_certificate(cls, private_key, curve=SECP256k1):
|
||||
signer = get_signer(curve).load_pem(private_key)
|
||||
return cls.load_protobuf(signer.certificate)
|
||||
__slots__ = 'claim', 'message'
|
||||
|
||||
def sign(self, private_key, claim_address, cert_claim_id, curve=SECP256k1, name=None, force_detached=False):
|
||||
signer = get_signer(curve).load_pem(private_key)
|
||||
signed, signature = signer.sign_stream_claim(self, claim_address, cert_claim_id, name, force_detached)
|
||||
return ClaimDict.load_protobuf(signed, signature)
|
||||
claim_type = None
|
||||
object_fields = 'thumbnail',
|
||||
repeat_fields = 'tags', 'languages', 'locations'
|
||||
|
||||
def validate_signature(self, claim_address, certificate, name=None):
|
||||
if isinstance(certificate, ClaimDict):
|
||||
certificate = certificate.protobuf
|
||||
curve = CURVE_NAMES[certificate.certificate.keyType]
|
||||
validator = get_validator(curve).load_from_certificate(certificate, self.certificate_id)
|
||||
return validator.validate_claim_signature(self, claim_address, name)
|
||||
def __init__(self, claim: Claim = None):
|
||||
self.claim = claim or Claim()
|
||||
self.message = self.claim.get_message(self.claim_type)
|
||||
|
||||
def validate_private_key(self, private_key, certificate_id):
|
||||
certificate = self.protobuf
|
||||
if CLAIM_TYPE_NAMES[certificate.claimType] != "certificate":
|
||||
return
|
||||
curve = CURVE_NAMES[certificate.certificate.keyType]
|
||||
validator = get_validator(curve).load_from_certificate(certificate, certificate_id)
|
||||
signing_key = validator.signing_key_from_pem(private_key)
|
||||
return validator.validate_private_key(signing_key)
|
||||
def to_dict(self):
|
||||
claim = MessageToDict(self.claim.message, preserving_proto_field_name=True)
|
||||
claim.update(claim.pop(self.claim_type))
|
||||
if 'languages' in claim:
|
||||
claim['languages'] = self.langtags
|
||||
if 'locations' in claim:
|
||||
claim['locations'] = [l.to_dict() for l in self.locations]
|
||||
return claim
|
||||
|
||||
def get_validator(self, certificate_id):
|
||||
"""
|
||||
Get a lbrynet.schema.validator.Validator object for a certificate claim
|
||||
def update(self, **kwargs):
|
||||
for key in list(kwargs):
|
||||
for field in self.object_fields:
|
||||
if key.startswith(f'{field}_'):
|
||||
attr = getattr(self, field)
|
||||
setattr(attr, key[len(f'{field}_'):], kwargs.pop(key))
|
||||
continue
|
||||
|
||||
:param certificate_id: claim id of this certificate claim
|
||||
:return: None or lbrynet.schema.validator.Validator object
|
||||
"""
|
||||
for l in self.repeat_fields:
|
||||
field = getattr(self, l)
|
||||
if kwargs.pop(f'clear_{l}', False):
|
||||
del field[:]
|
||||
items = kwargs.pop(l, None)
|
||||
if items is not None:
|
||||
if isinstance(items, str):
|
||||
field.append(items)
|
||||
elif isinstance(items, list):
|
||||
field.extend(items)
|
||||
else:
|
||||
raise ValueError(f"Unknown {l} value: {items}")
|
||||
|
||||
claim = self.protobuf
|
||||
if CLAIM_TYPE_NAMES[claim.claimType] != "certificate":
|
||||
return
|
||||
curve = CURVE_NAMES[claim.certificate.keyType]
|
||||
return get_validator(curve).load_from_certificate(claim, certificate_id)
|
||||
for key, value in kwargs.items():
|
||||
setattr(self, key, value)
|
||||
|
||||
@property
|
||||
def title(self) -> str:
|
||||
return self.claim.message.title
|
||||
|
||||
@title.setter
|
||||
def title(self, title: str):
|
||||
self.claim.message.title = title
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
return self.claim.message.description
|
||||
|
||||
@description.setter
|
||||
def description(self, description: str):
|
||||
self.claim.message.description = description
|
||||
|
||||
@property
|
||||
def thumbnail(self) -> Source:
|
||||
return Source(self.claim.message.thumbnail)
|
||||
|
||||
@property
|
||||
def tags(self) -> List:
|
||||
return self.claim.message.tags
|
||||
|
||||
@property
|
||||
def languages(self) -> LanguageList:
|
||||
return LanguageList(self.claim.message.languages)
|
||||
|
||||
@property
|
||||
def langtags(self) -> List[str]:
|
||||
return [l.langtag for l in self.languages]
|
||||
|
||||
@property
|
||||
def locations(self) -> LocationList:
|
||||
return LocationList(self.claim.message.locations)
|
||||
|
||||
|
||||
class Stream(BaseClaim):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
claim_type = Claim.STREAM
|
||||
|
||||
object_fields = BaseClaim.object_fields + ('source',)
|
||||
|
||||
def to_dict(self):
|
||||
claim = super().to_dict()
|
||||
if 'source' in claim:
|
||||
if 'hash' in claim['source']:
|
||||
claim['source']['hash'] = self.source.file_hash
|
||||
if 'sd_hash' in claim['source']:
|
||||
claim['source']['sd_hash'] = self.source.sd_hash
|
||||
fee = claim.get('fee', {})
|
||||
if 'address' in fee:
|
||||
fee['address'] = self.fee.address
|
||||
if 'amount' in fee:
|
||||
fee['amount'] = str(self.fee.amount)
|
||||
stream_type = self.message.WhichOneof('type')
|
||||
if stream_type:
|
||||
claim['stream_type'] = stream_type
|
||||
return claim
|
||||
|
||||
def update(self, file_path=None, height=None, width=None, duration=None, **kwargs):
|
||||
if kwargs.pop('clear_fee', False):
|
||||
self.message.ClearField('fee')
|
||||
else:
|
||||
self.fee.update(
|
||||
kwargs.pop('fee_address', None),
|
||||
kwargs.pop('fee_currency', None),
|
||||
kwargs.pop('fee_amount', None)
|
||||
)
|
||||
|
||||
if 'sd_hash' in kwargs:
|
||||
self.source.sd_hash = kwargs.pop('sd_hash')
|
||||
if 'file_name' in kwargs:
|
||||
self.source.name = kwargs.pop('file_name')
|
||||
if 'file_hash' in kwargs:
|
||||
self.source.file_hash = kwargs.pop('file_hash')
|
||||
|
||||
stream_type = None
|
||||
if file_path is not None:
|
||||
stream_type = self.source.update(file_path=file_path)
|
||||
elif self.source.name:
|
||||
self.source.media_type, stream_type = guess_media_type(self.source.name)
|
||||
elif self.source.media_type:
|
||||
stream_type = guess_stream_type(self.source.media_type)
|
||||
|
||||
if 'file_size' in kwargs:
|
||||
self.source.size = kwargs.pop('file_size')
|
||||
|
||||
if stream_type in ('image', 'video', 'audio'):
|
||||
media = getattr(self, stream_type)
|
||||
media_args = {'file_metadata': None}
|
||||
if file_path is not None:
|
||||
try:
|
||||
media_args['file_metadata'] = binary_file_metadata(binary_file_parser(file_path))
|
||||
except:
|
||||
log.exception('Could not read file metadata.')
|
||||
if isinstance(media, Playable):
|
||||
media_args['duration'] = duration
|
||||
if isinstance(media, Dimmensional):
|
||||
media_args['height'] = height
|
||||
media_args['width'] = width
|
||||
media.update(**media_args)
|
||||
|
||||
super().update(**kwargs)
|
||||
|
||||
@property
|
||||
def author(self) -> str:
|
||||
return self.message.author
|
||||
|
||||
@author.setter
|
||||
def author(self, author: str):
|
||||
self.message.author = author
|
||||
|
||||
@property
|
||||
def license(self) -> str:
|
||||
return self.message.license
|
||||
|
||||
@license.setter
|
||||
def license(self, license: str):
|
||||
self.message.license = license
|
||||
|
||||
@property
|
||||
def license_url(self) -> str:
|
||||
return self.message.license_url
|
||||
|
||||
@license_url.setter
|
||||
def license_url(self, license_url: str):
|
||||
self.message.license_url = license_url
|
||||
|
||||
@property
|
||||
def release_time(self) -> int:
|
||||
return self.message.release_time
|
||||
|
||||
@release_time.setter
|
||||
def release_time(self, release_time: int):
|
||||
self.message.release_time = release_time
|
||||
|
||||
@property
|
||||
def fee(self) -> Fee:
|
||||
return Fee(self.message.fee)
|
||||
|
||||
@property
|
||||
def has_fee(self) -> bool:
|
||||
return self.message.HasField('fee')
|
||||
|
||||
@property
|
||||
def source(self) -> Source:
|
||||
return Source(self.message.source)
|
||||
|
||||
@property
|
||||
def stream_type(self) -> str:
|
||||
return self.message.WhichOneof('type')
|
||||
|
||||
@property
|
||||
def image(self) -> Image:
|
||||
return Image(self.message.image)
|
||||
|
||||
@property
|
||||
def video(self) -> Video:
|
||||
return Video(self.message.video)
|
||||
|
||||
@property
|
||||
def audio(self) -> Audio:
|
||||
return Audio(self.message.audio)
|
||||
|
||||
|
||||
class Channel(BaseClaim):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
claim_type = Claim.CHANNEL
|
||||
|
||||
object_fields = BaseClaim.object_fields + ('cover',)
|
||||
repeat_fields = BaseClaim.repeat_fields + ('featured',)
|
||||
|
||||
def to_dict(self):
|
||||
claim = super().to_dict()
|
||||
claim['public_key'] = self.public_key
|
||||
if 'featured' in claim:
|
||||
claim['featured'] = self.featured.ids
|
||||
return claim
|
||||
|
||||
@property
|
||||
def public_key(self) -> str:
|
||||
return hexlify(self.message.public_key).decode()
|
||||
|
||||
@public_key.setter
|
||||
def public_key(self, sd_public_key: str):
|
||||
self.message.public_key = unhexlify(sd_public_key.encode())
|
||||
|
||||
@property
|
||||
def public_key_bytes(self) -> bytes:
|
||||
return self.message.public_key
|
||||
|
||||
@public_key_bytes.setter
|
||||
def public_key_bytes(self, public_key: bytes):
|
||||
self.message.public_key = public_key
|
||||
|
||||
@property
|
||||
def email(self) -> str:
|
||||
return self.message.email
|
||||
|
||||
@email.setter
|
||||
def email(self, email: str):
|
||||
self.message.email = email
|
||||
|
||||
@property
|
||||
def website_url(self) -> str:
|
||||
return self.message.website_url
|
||||
|
||||
@website_url.setter
|
||||
def website_url(self, website_url: str):
|
||||
self.message.website_url = website_url
|
||||
|
||||
@property
|
||||
def cover(self) -> Source:
|
||||
return Source(self.message.cover)
|
||||
|
||||
@property
|
||||
def featured(self) -> ClaimList:
|
||||
return ClaimList(self.message.featured)
|
||||
|
||||
|
||||
class Repost(BaseClaim):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
claim_type = Claim.REPOST
|
||||
|
||||
@property
|
||||
def reference(self) -> ClaimReference:
|
||||
return ClaimReference(self.message)
|
||||
|
||||
|
||||
class Collection(BaseClaim):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
claim_type = Claim.COLLECTION
|
||||
|
||||
repeat_fields = BaseClaim.repeat_fields + ('claims',)
|
||||
|
||||
def to_dict(self):
|
||||
claim = super().to_dict()
|
||||
if 'claim_references' in claim:
|
||||
claim['claim_references'] = self.claims.ids
|
||||
return claim
|
||||
|
||||
@property
|
||||
def claims(self) -> ClaimList:
|
||||
return ClaimList(self.message)
|
||||
@@ -0,0 +1,91 @@
|
||||
import json
|
||||
from decimal import Decimal
|
||||
|
||||
from google.protobuf.message import DecodeError
|
||||
|
||||
from lbrynet.schema.types.v1.legacy_claim_pb2 import Claim as OldClaimMessage
|
||||
from lbrynet.schema.types.v1.certificate_pb2 import KeyType
|
||||
from lbrynet.schema.types.v1.fee_pb2 import Fee as FeeMessage
|
||||
|
||||
|
||||
def from_old_json_schema(claim, payload: bytes):
|
||||
try:
|
||||
value = json.loads(payload)
|
||||
except:
|
||||
raise DecodeError('Could not parse JSON.')
|
||||
stream = claim.stream
|
||||
stream.source.sd_hash = value['sources']['lbry_sd_hash']
|
||||
stream.source.media_type = value.get(
|
||||
'content_type', value.get('content-type', 'application/octet-stream'))
|
||||
stream.title = value.get('title', '')
|
||||
stream.description = value.get('description', '')
|
||||
if value.get('thumbnail', ''):
|
||||
stream.thumbnail.url = value.get('thumbnail', '')
|
||||
stream.author = value.get('author', '')
|
||||
stream.license = value.get('license', '')
|
||||
stream.license_url = value.get('license_url', '')
|
||||
language = value.get('language', '')
|
||||
if language:
|
||||
if language.lower() == 'english':
|
||||
language = 'en'
|
||||
try:
|
||||
stream.languages.append(language)
|
||||
except:
|
||||
pass
|
||||
if value.get('nsfw', False):
|
||||
stream.tags.append('mature')
|
||||
if "fee" in value:
|
||||
fee = value["fee"]
|
||||
currency = list(fee.keys())[0]
|
||||
if currency == 'LBC':
|
||||
stream.fee.lbc = Decimal(fee[currency]['amount'])
|
||||
elif currency == 'USD':
|
||||
stream.fee.usd = Decimal(fee[currency]['amount'])
|
||||
elif currency == 'BTC':
|
||||
stream.fee.btc = Decimal(fee[currency]['amount'])
|
||||
else:
|
||||
raise DecodeError(f'Unknown currency: {currency}')
|
||||
stream.fee.address = fee[currency]['address']
|
||||
return claim
|
||||
|
||||
|
||||
def from_types_v1(claim, payload: bytes):
|
||||
old = OldClaimMessage()
|
||||
old.ParseFromString(payload)
|
||||
if old.claimType == 2:
|
||||
channel = claim.channel
|
||||
channel.public_key_bytes = old.certificate.publicKey
|
||||
else:
|
||||
stream = claim.stream
|
||||
stream.title = old.stream.metadata.title
|
||||
stream.description = old.stream.metadata.description
|
||||
stream.author = old.stream.metadata.author
|
||||
stream.license = old.stream.metadata.license
|
||||
stream.license_url = old.stream.metadata.licenseUrl
|
||||
stream.thumbnail.url = old.stream.metadata.thumbnail
|
||||
if old.stream.metadata.HasField('language'):
|
||||
stream.languages.add().message.language = old.stream.metadata.language
|
||||
stream.source.media_type = old.stream.source.contentType
|
||||
stream.source.sd_hash_bytes = old.stream.source.source
|
||||
if old.stream.metadata.nsfw:
|
||||
stream.tags.append('mature')
|
||||
if old.stream.metadata.HasField('fee'):
|
||||
fee = old.stream.metadata.fee
|
||||
stream.fee.address_bytes = fee.address
|
||||
currency = FeeMessage.Currency.Name(fee.currency)
|
||||
if currency == 'LBC':
|
||||
stream.fee.lbc = Decimal(fee.amount)
|
||||
elif currency == 'USD':
|
||||
stream.fee.usd = Decimal(fee.amount)
|
||||
elif currency == 'BTC':
|
||||
stream.fee.btc = Decimal(fee.amount)
|
||||
else:
|
||||
raise DecodeError(f'Unsupported currency: {currency}')
|
||||
if old.HasField('publisherSignature'):
|
||||
sig = old.publisherSignature
|
||||
claim.signature = sig.signature
|
||||
claim.signature_type = KeyType.Name(sig.signatureType)
|
||||
claim.signing_channel_hash = sig.certificateId[::-1]
|
||||
old.ClearField("publisherSignature")
|
||||
claim.unsigned_payload = old.SerializeToString()
|
||||
return claim
|
||||
@@ -1,61 +0,0 @@
|
||||
import json
|
||||
import binascii
|
||||
from google.protobuf import json_format # pylint: disable=no-name-in-module
|
||||
|
||||
from lbrynet.schema.error import DecodeError, InvalidAddress
|
||||
from lbrynet.schema.legacy.migrate import migrate as schema_migrator
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
|
||||
|
||||
|
||||
def migrate_json_claim_value(decoded_json):
|
||||
try:
|
||||
if 'fee' in decoded_json:
|
||||
old_fee = decoded_json['fee']
|
||||
if not old_fee[list(old_fee.keys())[0]]['amount']:
|
||||
del decoded_json['fee']
|
||||
return migrate_json_claim_value(decoded_json)
|
||||
except (TypeError, AttributeError, InvalidAddress):
|
||||
raise DecodeError("Failed to decode claim")
|
||||
try:
|
||||
pb_migrated = schema_migrator(decoded_json)
|
||||
return pb_migrated
|
||||
except json_format.ParseError as parse_error:
|
||||
raise DecodeError("Failed to parse protobuf: %s" % parse_error)
|
||||
except Exception as err:
|
||||
raise DecodeError("Failed to migrate claim: %s" % err)
|
||||
|
||||
|
||||
def smart_decode(claim_value):
|
||||
"""
|
||||
Decode a claim value
|
||||
|
||||
Try decoding claim protobuf, if this fails try decoding json and migrating it.
|
||||
If unable to decode or migrate, raise DecodeError
|
||||
"""
|
||||
|
||||
# if already decoded, return
|
||||
if isinstance(claim_value, ClaimDict):
|
||||
return claim_value
|
||||
elif isinstance(claim_value, dict):
|
||||
return ClaimDict.load_dict(claim_value)
|
||||
|
||||
try:
|
||||
claim_value = binascii.unhexlify(claim_value)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
if claim_value[0] in ['{', ord('{')]:
|
||||
try:
|
||||
if isinstance(claim_value, bytes):
|
||||
claim_value = claim_value.decode()
|
||||
decoded_json = json.loads(claim_value)
|
||||
return migrate_json_claim_value(decoded_json)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
try:
|
||||
if isinstance(claim_value, str):
|
||||
claim_value = claim_value.encode()
|
||||
return ClaimDict.deserialize(claim_value)
|
||||
except (DecodeError, InvalidAddress, KeyError, TypeError):
|
||||
raise DecodeError(claim_value)
|
||||
@@ -1,90 +0,0 @@
|
||||
import base64, binascii
|
||||
from copy import deepcopy
|
||||
from lbrynet.schema.address import decode_address, encode_address
|
||||
from lbrynet.schema.schema import CLAIM_TYPES, CLAIM_TYPE, STREAM_TYPE, CERTIFICATE_TYPE
|
||||
from lbrynet.schema.schema import SIGNATURE
|
||||
from lbrynet.schema.error import DecodeError, InvalidAddress
|
||||
from lbrynet.schema.signature import Signature
|
||||
|
||||
|
||||
def encode_fields(claim_dictionary, detached_signature: Signature):
|
||||
"""Encode bytes to hex and b58 for return by ClaimDict"""
|
||||
claim_dictionary = deepcopy(claim_dictionary)
|
||||
claim_type = CLAIM_TYPES[claim_dictionary[CLAIM_TYPE]]
|
||||
claim_value = claim_dictionary[claim_type]
|
||||
if claim_type == CLAIM_TYPES[STREAM_TYPE]:
|
||||
claim_value['source']['source'] = binascii.hexlify(claim_value['source']['source']).decode()
|
||||
if 'fee' in claim_value['metadata']:
|
||||
try:
|
||||
address = encode_address(claim_value['metadata']['fee']['address'])
|
||||
except InvalidAddress as err:
|
||||
raise DecodeError("Invalid fee address: %s" % err)
|
||||
claim_value['metadata']['fee']['address'] = address
|
||||
elif claim_type == CLAIM_TYPES[CERTIFICATE_TYPE]:
|
||||
public_key = claim_value["publicKey"]
|
||||
claim_value["publicKey"] = binascii.hexlify(public_key).decode()
|
||||
if SIGNATURE in claim_dictionary:
|
||||
encoded_sig = binascii.hexlify(claim_dictionary[SIGNATURE]['signature']).decode()
|
||||
encoded_cert_id = binascii.hexlify(claim_dictionary[SIGNATURE]['certificateId']).decode()
|
||||
claim_dictionary[SIGNATURE]['signature'] = encoded_sig
|
||||
claim_dictionary[SIGNATURE]['certificateId'] = encoded_cert_id
|
||||
elif detached_signature and detached_signature.raw_signature:
|
||||
claim_dictionary[SIGNATURE] = {
|
||||
'detached_signature': binascii.hexlify(detached_signature.serialized).decode(),
|
||||
'certificateId': binascii.hexlify(detached_signature.certificate_id).decode()
|
||||
}
|
||||
|
||||
claim_dictionary[claim_type] = claim_value
|
||||
return claim_dictionary
|
||||
|
||||
|
||||
def decode_fields(claim_dictionary):
|
||||
"""Decode hex and b58 encoded bytes in dictionaries given to ClaimDict"""
|
||||
detached_signature = None
|
||||
claim_dictionary = deepcopy(claim_dictionary)
|
||||
claim_type = CLAIM_TYPES[claim_dictionary[CLAIM_TYPE]]
|
||||
claim_value = claim_dictionary[claim_type]
|
||||
if claim_type == CLAIM_TYPES[STREAM_TYPE]:
|
||||
claim_value['source']['source'] = binascii.unhexlify(claim_value['source']['source'])
|
||||
if 'fee' in claim_value['metadata']:
|
||||
try:
|
||||
address = decode_address(claim_value['metadata']['fee']['address'])
|
||||
except InvalidAddress as err:
|
||||
raise DecodeError("Invalid fee address: %s" % err)
|
||||
claim_value['metadata']['fee']['address'] = address
|
||||
elif claim_type == CLAIM_TYPES[CERTIFICATE_TYPE]:
|
||||
public_key = binascii.unhexlify(claim_value["publicKey"])
|
||||
claim_value["publicKey"] = public_key
|
||||
if SIGNATURE in claim_dictionary and not claim_dictionary[SIGNATURE].get('detached_signature'):
|
||||
decoded_sig = binascii.unhexlify(claim_dictionary[SIGNATURE]['signature'])
|
||||
decoded_cert_id = binascii.unhexlify(claim_dictionary[SIGNATURE]['certificateId'])
|
||||
claim_dictionary[SIGNATURE]['signature'] = decoded_sig
|
||||
claim_dictionary[SIGNATURE]['certificateId'] = decoded_cert_id
|
||||
elif claim_dictionary.get(SIGNATURE, {}).get('detached_signature'):
|
||||
hex_detached_signature = claim_dictionary[SIGNATURE]['detached_signature']
|
||||
detached_signature = Signature.flagged_parse(binascii.unhexlify(hex_detached_signature))
|
||||
del claim_dictionary[SIGNATURE]
|
||||
claim_dictionary[claim_type] = claim_value
|
||||
return claim_dictionary, detached_signature
|
||||
|
||||
|
||||
def decode_b64_fields(claim_dictionary):
|
||||
"""Decode b64 encoded bytes in protobuf generated dictionary to be given to ClaimDict"""
|
||||
claim_dictionary = deepcopy(claim_dictionary)
|
||||
claim_type = CLAIM_TYPES[claim_dictionary[CLAIM_TYPE]]
|
||||
claim_value = claim_dictionary[claim_type]
|
||||
if claim_type == CLAIM_TYPES[STREAM_TYPE]:
|
||||
claim_value['source']['source'] = base64.b64decode(claim_value['source']['source'])
|
||||
if 'fee' in claim_value['metadata']:
|
||||
address = base64.b64decode(claim_value['metadata']['fee']['address'])
|
||||
claim_value['metadata']['fee']['address'] = address
|
||||
elif claim_type == CLAIM_TYPES[CERTIFICATE_TYPE]:
|
||||
public_key = base64.b64decode(claim_value["publicKey"])
|
||||
claim_value["publicKey"] = public_key
|
||||
if SIGNATURE in claim_dictionary:
|
||||
encoded_sig = base64.b64decode(claim_dictionary[SIGNATURE]['signature'])
|
||||
encoded_cert_id = base64.b64decode(claim_dictionary[SIGNATURE]['certificateId'])
|
||||
claim_dictionary[SIGNATURE]['signature'] = encoded_sig
|
||||
claim_dictionary[SIGNATURE]['certificateId'] = encoded_cert_id
|
||||
claim_dictionary[claim_type] = claim_value
|
||||
return claim_dictionary
|
||||
@@ -1,22 +0,0 @@
|
||||
class UnknownSourceType(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidSourceHashLength(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class DecodeError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class URIParseError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class CertificateError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidAddress(Exception):
|
||||
pass
|
||||
@@ -1,79 +0,0 @@
|
||||
from collections import OrderedDict
|
||||
|
||||
from lbrynet.schema.address import encode_address, decode_address
|
||||
from lbrynet.schema.schema import CURRENCY_NAMES, CURRENCY_MAP
|
||||
from lbrynet.schema.schema.fee import Fee as FeeHelper
|
||||
from lbrynet.schema.proto import fee_pb2
|
||||
|
||||
|
||||
def migrate(fee):
|
||||
if len(list(fee.keys())) == 3 and 'currency' in fee and 'amount' in fee and 'address' in fee:
|
||||
return FeeHelper.load({
|
||||
"version": "_0_0_1",
|
||||
"currency": fee['currency'],
|
||||
"amount": fee['amount'],
|
||||
"address": decode_address(fee['address'])
|
||||
})
|
||||
if len(list(fee.keys())) > 1:
|
||||
raise Exception("Invalid fee")
|
||||
|
||||
currency = list(fee.keys())[0]
|
||||
amount = fee[currency]['amount']
|
||||
address = fee[currency]['address']
|
||||
|
||||
return FeeHelper.load({
|
||||
"version": "_0_0_1",
|
||||
"currency": currency,
|
||||
"amount": amount,
|
||||
"address": decode_address(address)
|
||||
})
|
||||
|
||||
|
||||
class Fee(OrderedDict):
|
||||
def __init__(self, fee):
|
||||
if (len(fee) == 4 and "version" in fee and "currency" in fee
|
||||
and "amount" in fee and "address" in fee):
|
||||
OrderedDict.__init__(self, fee)
|
||||
else:
|
||||
OrderedDict.__init__(self, Fee.load_protobuf(migrate(fee)))
|
||||
|
||||
@property
|
||||
def currency(self):
|
||||
return self['currency']
|
||||
|
||||
@property
|
||||
def address(self):
|
||||
return self['address']
|
||||
|
||||
@property
|
||||
def amount(self):
|
||||
return self['amount']
|
||||
|
||||
@property
|
||||
def version(self):
|
||||
return self['version']
|
||||
|
||||
@property
|
||||
def protobuf(self):
|
||||
pb = {
|
||||
"version": self.version,
|
||||
"currency": CURRENCY_MAP[self.currency],
|
||||
"address": decode_address(self.address),
|
||||
"amount": self.amount
|
||||
}
|
||||
return FeeHelper.load(pb)
|
||||
|
||||
@classmethod
|
||||
def load_protobuf(cls, pb):
|
||||
return cls({
|
||||
"version": pb.version,
|
||||
"currency": CURRENCY_NAMES[pb.currency],
|
||||
"address": encode_address(pb.address),
|
||||
"amount": pb.amount
|
||||
})
|
||||
|
||||
@classmethod
|
||||
def deserialize(cls, serialized):
|
||||
pb = fee_pb2.Fee()
|
||||
pb.ParseFromString(serialized)
|
||||
return cls.load_protobuf(pb)
|
||||
@@ -1,23 +0,0 @@
|
||||
import hashlib
|
||||
|
||||
|
||||
def sha256(x):
|
||||
if isinstance(x, str):
|
||||
x = x.encode('utf-8')
|
||||
return hashlib.sha256(x).digest()
|
||||
|
||||
|
||||
def double_sha256(x):
|
||||
return sha256(sha256(x))
|
||||
|
||||
|
||||
def ripemd160(x):
|
||||
if isinstance(x, str):
|
||||
x = x.encode('utf-8')
|
||||
md = hashlib.new('ripemd160')
|
||||
md.update(x)
|
||||
return md.digest()
|
||||
|
||||
|
||||
def hash160(x):
|
||||
return ripemd160(sha256(x))
|
||||
@@ -1,59 +0,0 @@
|
||||
import jsonschema
|
||||
|
||||
|
||||
class StructuredDict(dict):
|
||||
"""
|
||||
A dictionary that enforces a structure specified by a schema, and supports
|
||||
migration between different versions of the schema.
|
||||
"""
|
||||
|
||||
# To be specified in sub-classes, an array in the format
|
||||
# [(version, schema, migration), ...]
|
||||
_versions = []
|
||||
|
||||
# Used internally to allow schema lookups by version number
|
||||
_schemas = {}
|
||||
|
||||
version = None
|
||||
|
||||
def __init__(self, value, starting_version, migrate=True, target_version=None):
|
||||
dict.__init__(self, value)
|
||||
|
||||
self.version = starting_version
|
||||
self._schemas = dict([(version, schema) for (version, schema, _) in self._versions])
|
||||
|
||||
self.validate(starting_version)
|
||||
|
||||
if migrate:
|
||||
self.migrate(target_version)
|
||||
|
||||
def _upgrade_version_range(self, start_version, end_version):
|
||||
after_starting_version = False
|
||||
for version, schema, migration in self._versions:
|
||||
if not after_starting_version:
|
||||
if version == self.version:
|
||||
after_starting_version = True
|
||||
continue
|
||||
|
||||
yield version, schema, migration
|
||||
|
||||
if end_version and version == end_version:
|
||||
break
|
||||
|
||||
def validate(self, version):
|
||||
jsonschema.validate(self, self._schemas[version])
|
||||
|
||||
def migrate(self, target_version=None):
|
||||
if target_version:
|
||||
assert self._versions.index(target_version) > self._versions.index(self.version), \
|
||||
"Current version is above target version"
|
||||
|
||||
for version, schema, migration in self._upgrade_version_range(self.version, target_version):
|
||||
migration(self)
|
||||
try:
|
||||
self.validate(version)
|
||||
except jsonschema.ValidationError as e:
|
||||
raise jsonschema.ValidationError(
|
||||
"Could not migrate to version %s due to validation error: %s" %
|
||||
(version, e.message))
|
||||
self.version = version
|
||||
@@ -1,276 +0,0 @@
|
||||
VER_001 = {
|
||||
'$schema': 'http://json-schema.org/draft-04/schema#',
|
||||
'title': 'LBRY metadata schema 0.0.1',
|
||||
'definitions': {
|
||||
'fee_info': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'amount': {
|
||||
'type': 'number',
|
||||
'minimum': 0,
|
||||
'exclusiveMinimum': True,
|
||||
},
|
||||
'address': {
|
||||
'type': 'string'
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
'type': 'object',
|
||||
|
||||
'properties': {
|
||||
'ver': {
|
||||
'type': 'string',
|
||||
'default': '0.0.1'
|
||||
},
|
||||
'title': {
|
||||
'type': 'string'
|
||||
},
|
||||
'description': {
|
||||
'type': 'string'
|
||||
},
|
||||
'author': {
|
||||
'type': 'string'
|
||||
},
|
||||
'language': {
|
||||
'type': 'string'
|
||||
},
|
||||
'license': {
|
||||
'type': 'string'
|
||||
},
|
||||
'content-type': {
|
||||
'type': 'string'
|
||||
},
|
||||
'sources': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'lbry_sd_hash': {
|
||||
'type': 'string'
|
||||
},
|
||||
'btih': {
|
||||
'type': 'string'
|
||||
},
|
||||
'url': {
|
||||
'type': 'string'
|
||||
}
|
||||
},
|
||||
'additionalProperties': False
|
||||
},
|
||||
'thumbnail': {
|
||||
'type': 'string'
|
||||
},
|
||||
'preview': {
|
||||
'type': 'string'
|
||||
},
|
||||
'fee': {
|
||||
'properties': {
|
||||
'LBC': {'$ref': '#/definitions/fee_info'},
|
||||
'BTC': {'$ref': '#/definitions/fee_info'},
|
||||
'USD': {'$ref': '#/definitions/fee_info'}
|
||||
}
|
||||
},
|
||||
'contact': {
|
||||
'type': 'number'
|
||||
},
|
||||
'pubkey': {
|
||||
'type': 'string'
|
||||
},
|
||||
},
|
||||
'required': [
|
||||
'title', 'description', 'author', 'language', 'license', 'content-type', 'sources'],
|
||||
'additionalProperties': False
|
||||
}
|
||||
|
||||
|
||||
VER_002 = {
|
||||
'$schema': 'http://json-schema.org/draft-04/schema#',
|
||||
'title': 'LBRY metadata schema 0.0.2',
|
||||
'definitions': {
|
||||
'fee_info': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'amount': {
|
||||
'type': 'number',
|
||||
'minimum': 0,
|
||||
'exclusiveMinimum': True,
|
||||
},
|
||||
'address': {
|
||||
'type': 'string'
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
'type': 'object',
|
||||
|
||||
'properties': {
|
||||
'ver': {
|
||||
'type': 'string',
|
||||
'enum': ['0.0.2'],
|
||||
},
|
||||
'title': {
|
||||
'type': 'string'
|
||||
},
|
||||
'description': {
|
||||
'type': 'string'
|
||||
},
|
||||
'author': {
|
||||
'type': 'string'
|
||||
},
|
||||
'language': {
|
||||
'type': 'string'
|
||||
},
|
||||
'license': {
|
||||
'type': 'string'
|
||||
},
|
||||
'content-type': {
|
||||
'type': 'string'
|
||||
},
|
||||
'sources': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'lbry_sd_hash': {
|
||||
'type': 'string'
|
||||
},
|
||||
'btih': {
|
||||
'type': 'string'
|
||||
},
|
||||
'url': {
|
||||
'type': 'string'
|
||||
}
|
||||
},
|
||||
'additionalProperties': False
|
||||
},
|
||||
'thumbnail': {
|
||||
'type': 'string'
|
||||
},
|
||||
'preview': {
|
||||
'type': 'string'
|
||||
},
|
||||
'fee': {
|
||||
'properties': {
|
||||
'LBC': {'$ref': '#/definitions/fee_info'},
|
||||
'BTC': {'$ref': '#/definitions/fee_info'},
|
||||
'USD': {'$ref': '#/definitions/fee_info'}
|
||||
}
|
||||
},
|
||||
'contact': {
|
||||
'type': 'number'
|
||||
},
|
||||
'pubkey': {
|
||||
'type': 'string'
|
||||
},
|
||||
'license_url': {
|
||||
'type': 'string'
|
||||
},
|
||||
'nsfw': {
|
||||
'type': 'boolean',
|
||||
'default': False
|
||||
},
|
||||
|
||||
},
|
||||
'required': [
|
||||
'ver', 'title', 'description', 'author', 'language', 'license',
|
||||
'content-type', 'sources', 'nsfw'
|
||||
],
|
||||
'additionalProperties': False
|
||||
}
|
||||
|
||||
|
||||
VER_003 = {
|
||||
'$schema': 'http://json-schema.org/draft-04/schema#',
|
||||
'title': 'LBRY metadata schema 0.0.3',
|
||||
'definitions': {
|
||||
'fee_info': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'amount': {
|
||||
'type': 'number',
|
||||
'minimum': 0,
|
||||
'exclusiveMinimum': True,
|
||||
},
|
||||
'address': {
|
||||
'type': 'string'
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
'type': 'object',
|
||||
|
||||
'properties': {
|
||||
'ver': {
|
||||
'type': 'string',
|
||||
'enum': ['0.0.3'],
|
||||
},
|
||||
'title': {
|
||||
'type': 'string'
|
||||
},
|
||||
'description': {
|
||||
'type': 'string'
|
||||
},
|
||||
'author': {
|
||||
'type': 'string'
|
||||
},
|
||||
'language': {
|
||||
'type': 'string'
|
||||
},
|
||||
'license': {
|
||||
'type': 'string'
|
||||
},
|
||||
'content_type': {
|
||||
'type': 'string'
|
||||
},
|
||||
'sources': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'lbry_sd_hash': {
|
||||
'type': 'string'
|
||||
},
|
||||
'btih': {
|
||||
'type': 'string'
|
||||
},
|
||||
'url': {
|
||||
'type': 'string'
|
||||
}
|
||||
},
|
||||
'additionalProperties': False
|
||||
},
|
||||
'thumbnail': {
|
||||
'type': 'string'
|
||||
},
|
||||
'preview': {
|
||||
'type': 'string'
|
||||
},
|
||||
'fee': {
|
||||
'properties': {
|
||||
'LBC': {'$ref': '#/definitions/fee_info'},
|
||||
'BTC': {'$ref': '#/definitions/fee_info'},
|
||||
'USD': {'$ref': '#/definitions/fee_info'}
|
||||
}
|
||||
},
|
||||
'contact': {
|
||||
'type': 'number'
|
||||
},
|
||||
'pubkey': {
|
||||
'type': 'string'
|
||||
},
|
||||
'license_url': {
|
||||
'type': 'string'
|
||||
},
|
||||
'nsfw': {
|
||||
'type': 'boolean',
|
||||
'default': False
|
||||
},
|
||||
'sig': {
|
||||
'type': 'string'
|
||||
}
|
||||
},
|
||||
'required': [
|
||||
'ver', 'title', 'description', 'author', 'language', 'license',
|
||||
'content_type', 'sources', 'nsfw'
|
||||
],
|
||||
'additionalProperties': False,
|
||||
'dependencies': {
|
||||
'pubkey': ['sig'],
|
||||
'sig': ['pubkey']
|
||||
}
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
"""
|
||||
migrate claim json schema (0.0.1-3) to protobuf (0.1.0)
|
||||
"""
|
||||
|
||||
from lbrynet.schema.legacy import metadata_schemas
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from .StructuredDict import StructuredDict
|
||||
|
||||
|
||||
def migrate_001_to_002(metadata):
|
||||
metadata['ver'] = '0.0.2'
|
||||
metadata['nsfw'] = False
|
||||
|
||||
|
||||
def migrate_002_to_003(metadata):
|
||||
metadata['ver'] = '0.0.3'
|
||||
if 'content-type' in metadata:
|
||||
metadata['content_type'] = metadata['content-type']
|
||||
del metadata['content-type']
|
||||
|
||||
|
||||
class LegacyMetadata(StructuredDict):
|
||||
current_version = '0.0.3'
|
||||
|
||||
_versions = [
|
||||
('0.0.1', metadata_schemas.VER_001, None),
|
||||
('0.0.2', metadata_schemas.VER_002, migrate_001_to_002),
|
||||
('0.0.3', metadata_schemas.VER_003, migrate_002_to_003)
|
||||
]
|
||||
|
||||
def __init__(self, metadata, migrate=True, target_version=None):
|
||||
if not isinstance(metadata, dict):
|
||||
raise TypeError("{} is not a dictionary".format(metadata))
|
||||
starting_version = metadata.get('ver', '0.0.1')
|
||||
|
||||
StructuredDict.__init__(self, metadata, starting_version, migrate, target_version)
|
||||
|
||||
|
||||
def migrate_003_to_010(value):
|
||||
migrated_to_003 = LegacyMetadata(value)
|
||||
metadata = {
|
||||
"version": "_0_1_0"
|
||||
}
|
||||
for k in ["author", "description", "language", "license", "nsfw", "thumbnail", "title",
|
||||
"preview"]:
|
||||
if k in migrated_to_003:
|
||||
metadata.update({k: migrated_to_003[k]})
|
||||
|
||||
if 'license_url' in migrated_to_003:
|
||||
metadata['licenseUrl'] = migrated_to_003['license_url']
|
||||
|
||||
if "fee" in migrated_to_003:
|
||||
fee = migrated_to_003["fee"]
|
||||
currency = list(fee.keys())[0]
|
||||
amount = fee[currency]['amount']
|
||||
address = fee[currency]['address']
|
||||
metadata.update(dict(fee={"currency": currency, "version": "_0_0_1",
|
||||
"amount": amount, "address": address}))
|
||||
source = {
|
||||
"source": migrated_to_003['sources']['lbry_sd_hash'],
|
||||
"contentType": migrated_to_003['content_type'],
|
||||
"sourceType": "lbry_sd_hash",
|
||||
"version": "_0_0_1"
|
||||
}
|
||||
|
||||
migrated = {
|
||||
"version": "_0_0_1",
|
||||
"claimType": "streamType",
|
||||
"stream": {
|
||||
"version": "_0_0_1",
|
||||
"metadata": metadata,
|
||||
"source": source
|
||||
}
|
||||
}
|
||||
return ClaimDict.load_dict(migrated)
|
||||
|
||||
|
||||
def migrate(value):
|
||||
return migrate_003_to_010(value)
|
||||
@@ -0,0 +1,174 @@
|
||||
import os
|
||||
|
||||
types_map = {
|
||||
# http://www.iana.org/assignments/media-types
|
||||
# Type mapping for automated metadata extraction (video, audio, image, document, binary, model)
|
||||
'.a': ('application/octet-stream', 'binary'),
|
||||
'.ai': ('application/postscript', 'image'),
|
||||
'.aif': ('audio/x-aiff', 'audio'),
|
||||
'.aifc': ('audio/x-aiff', 'audio'),
|
||||
'.aiff': ('audio/x-aiff', 'audio'),
|
||||
'.au': ('audio/basic', 'audio'),
|
||||
'.avi': ('video/x-msvideo', 'video'),
|
||||
'.bat': ('text/plain', 'document'),
|
||||
'.bcpio': ('application/x-bcpio', 'binary'),
|
||||
'.bin': ('application/octet-stream', 'binary'),
|
||||
'.bmp': ('image/bmp', 'image'),
|
||||
'.c': ('text/plain', 'document'),
|
||||
'.cdf': ('application/x-netcdf', 'binary'),
|
||||
'.cpio': ('application/x-cpio', 'binary'),
|
||||
'.csh': ('application/x-csh', 'binary'),
|
||||
'.css': ('text/css', 'document'),
|
||||
'.csv': ('text/csv', 'document'),
|
||||
'.dll': ('application/octet-stream', 'binary'),
|
||||
'.doc': ('application/msword', 'document'),
|
||||
'.dot': ('application/msword', 'document'),
|
||||
'.dvi': ('application/x-dvi', 'binary'),
|
||||
'.eml': ('message/rfc822', 'document'),
|
||||
'.eps': ('application/postscript', 'document'),
|
||||
'.epub': ('application/epub+zip', 'document'),
|
||||
'.etx': ('text/x-setext', 'document'),
|
||||
'.exe': ('application/octet-stream', 'binary'),
|
||||
'.gif': ('image/gif', 'image'),
|
||||
'.gtar': ('application/x-gtar', 'binary'),
|
||||
'.h': ('text/plain', 'document'),
|
||||
'.hdf': ('application/x-hdf', 'binary'),
|
||||
'.htm': ('text/html', 'document'),
|
||||
'.html': ('text/html', 'document'),
|
||||
'.ico': ('image/vnd.microsoft.icon', 'image'),
|
||||
'.ief': ('image/ief', 'image'),
|
||||
'.iges': ('model/iges', 'model'),
|
||||
'.jpe': ('image/jpeg', 'image'),
|
||||
'.jpeg': ('image/jpeg', 'image'),
|
||||
'.jpg': ('image/jpeg', 'image'),
|
||||
'.js': ('application/javascript', 'document'),
|
||||
'.json': ('application/json', 'document'),
|
||||
'.ksh': ('text/plain', 'document'),
|
||||
'.latex': ('application/x-latex', 'binary'),
|
||||
'.m1v': ('video/mpeg', 'video'),
|
||||
'.m3u': ('application/vnd.apple.mpegurl', 'audio'),
|
||||
'.m3u8': ('application/vnd.apple.mpegurl', 'audio'),
|
||||
'.man': ('application/x-troff-man', 'document'),
|
||||
'.markdown': ('text/markdown', 'document'),
|
||||
'.md': ('text/markdown', 'document'),
|
||||
'.me': ('application/x-troff-me', 'binary'),
|
||||
'.mht': ('message/rfc822', 'document'),
|
||||
'.mhtml': ('message/rfc822', 'document'),
|
||||
'.mif': ('application/x-mif', 'binary'),
|
||||
'.mov': ('video/quicktime', 'video'),
|
||||
'.movie': ('video/x-sgi-movie', 'video'),
|
||||
'.mp2': ('audio/mpeg', 'audio'),
|
||||
'.mp3': ('audio/mpeg', 'audio'),
|
||||
'.mp4': ('video/mp4', 'video'),
|
||||
'.mpa': ('video/mpeg', 'video'),
|
||||
'.mpe': ('video/mpeg', 'video'),
|
||||
'.mpeg': ('video/mpeg', 'video'),
|
||||
'.mpg': ('video/mpeg', 'video'),
|
||||
'.ms': ('application/x-troff-ms', 'binary'),
|
||||
'.nc': ('application/x-netcdf', 'binary'),
|
||||
'.nws': ('message/rfc822', 'document'),
|
||||
'.o': ('application/octet-stream', 'binary'),
|
||||
'.obj': ('application/octet-stream', 'model'),
|
||||
'.oda': ('application/oda', 'binary'),
|
||||
'.p12': ('application/x-pkcs12', 'binary'),
|
||||
'.p7c': ('application/pkcs7-mime', 'binary'),
|
||||
'.pbm': ('image/x-portable-bitmap', 'image'),
|
||||
'.pdf': ('application/pdf', 'document'),
|
||||
'.pfx': ('application/x-pkcs12', 'binary'),
|
||||
'.pgm': ('image/x-portable-graymap', 'image'),
|
||||
'.pl': ('text/plain', 'document'),
|
||||
'.png': ('image/png', 'image'),
|
||||
'.pnm': ('image/x-portable-anymap', 'image'),
|
||||
'.pot': ('application/vnd.ms-powerpoint', 'document'),
|
||||
'.ppa': ('application/vnd.ms-powerpoint', 'document'),
|
||||
'.ppm': ('image/x-portable-pixmap', 'image'),
|
||||
'.pps': ('application/vnd.ms-powerpoint', 'document'),
|
||||
'.ppt': ('application/vnd.ms-powerpoint', 'document'),
|
||||
'.ps': ('application/postscript', 'document'),
|
||||
'.pwz': ('application/vnd.ms-powerpoint', 'document'),
|
||||
'.py': ('text/x-python', 'document'),
|
||||
'.pyc': ('application/x-python-code', 'binary'),
|
||||
'.pyo': ('application/x-python-code', 'binary'),
|
||||
'.qt': ('video/quicktime', 'video'),
|
||||
'.ra': ('audio/x-pn-realaudio', 'audio'),
|
||||
'.ram': ('application/x-pn-realaudio', 'audio'),
|
||||
'.ras': ('image/x-cmu-raster', 'image'),
|
||||
'.rdf': ('application/xml', 'binary'),
|
||||
'.rgb': ('image/x-rgb', 'image'),
|
||||
'.roff': ('application/x-troff', 'binary'),
|
||||
'.rtx': ('text/richtext', 'document'),
|
||||
'.sgm': ('text/x-sgml', 'document'),
|
||||
'.sgml': ('text/x-sgml', 'document'),
|
||||
'.sh': ('application/x-sh', 'document'),
|
||||
'.shar': ('application/x-shar', 'binary'),
|
||||
'.snd': ('audio/basic', 'audio'),
|
||||
'.so': ('application/octet-stream', 'binary'),
|
||||
'.src': ('application/x-wais-source', 'binary'),
|
||||
'.stl': ('model/stl', 'model'),
|
||||
'.sv4cpio': ('application/x-sv4cpio', 'binary'),
|
||||
'.sv4crc': ('application/x-sv4crc', 'binary'),
|
||||
'.svg': ('image/svg+xml', 'image'),
|
||||
'.swf': ('application/x-shockwave-flash', 'binary'),
|
||||
'.t': ('application/x-troff', 'binary'),
|
||||
'.tar': ('application/x-tar', 'binary'),
|
||||
'.tcl': ('application/x-tcl', 'binary'),
|
||||
'.tex': ('application/x-tex', 'binary'),
|
||||
'.texi': ('application/x-texinfo', 'binary'),
|
||||
'.texinfo': ('application/x-texinfo', 'binary'),
|
||||
'.tif': ('image/tiff', 'image'),
|
||||
'.tiff': ('image/tiff', 'image'),
|
||||
'.tr': ('application/x-troff', 'binary'),
|
||||
'.tsv': ('text/tab-separated-values', 'document'),
|
||||
'.txt': ('text/plain', 'document'),
|
||||
'.ustar': ('application/x-ustar', 'binary'),
|
||||
'.vcf': ('text/x-vcard', 'document'),
|
||||
'.wav': ('audio/x-wav', 'audio'),
|
||||
'.webm': ('video/webm', 'video'),
|
||||
'.wiz': ('application/msword', 'document'),
|
||||
'.wsdl': ('application/xml', 'document'),
|
||||
'.xbm': ('image/x-xbitmap', 'image'),
|
||||
'.xlb': ('application/vnd.ms-excel', 'document'),
|
||||
'.xls': ('application/vnd.ms-excel', 'document'),
|
||||
'.xml': ('text/xml', 'document'),
|
||||
'.xpdl': ('application/xml', 'document'),
|
||||
'.xpm': ('image/x-xpixmap', 'image'),
|
||||
'.xsl': ('application/xml', 'document'),
|
||||
'.xwd': ('image/x-xwindowdump', 'image'),
|
||||
'.zip': ('application/zip', 'binary'),
|
||||
|
||||
# These are non-standard types, commonly found in the wild.
|
||||
'.cbr': ('application/vnd.comicbook-rar', 'document'),
|
||||
'.cbz': ('application/vnd.comicbook+zip', 'document'),
|
||||
'.lbry': ('application/x-ext-lbry', 'document'),
|
||||
'.mid': ('audio/midi', 'audio'),
|
||||
'.midi': ('audio/midi', 'audio'),
|
||||
'.mobi': ('application/x-mobipocket-ebook', 'document'),
|
||||
'.pct': ('image/pict', 'image'),
|
||||
'.pic': ('image/pict', 'image'),
|
||||
'.pict': ('image/pict', 'image'),
|
||||
'.prc': ('application/x-mobipocket-ebook', 'document'),
|
||||
'.rtf': ('application/rtf', 'document'),
|
||||
'.xul': ('text/xul', 'document'),
|
||||
'.m4v': ('video/m4v', 'video'),
|
||||
|
||||
# microsoft is special and has its own 'standard'
|
||||
# https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
|
||||
'.wmv': ('video/x-ms-wmv', 'video')
|
||||
}
|
||||
|
||||
|
||||
def guess_media_type(path):
|
||||
_, ext = os.path.splitext(path)
|
||||
extension = ext.strip().lower()
|
||||
if extension[1:]:
|
||||
if extension in types_map:
|
||||
return types_map[extension]
|
||||
return f'application/x-ext-{extension[1:]}', 'binary'
|
||||
return 'application/octet-stream', 'binary'
|
||||
|
||||
|
||||
def guess_stream_type(media_type):
|
||||
for media, stream in types_map.values():
|
||||
if media == media_type:
|
||||
return stream
|
||||
return 'binary'
|
||||
@@ -1,18 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
enum KeyType {
|
||||
UNKNOWN_PUBLIC_KEY_TYPE = 0;
|
||||
NIST256p = 1;
|
||||
NIST384p = 2;
|
||||
SECP256k1 = 3;
|
||||
}
|
||||
|
||||
message Certificate {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
}
|
||||
required Version version = 1;
|
||||
required KeyType keyType = 2;
|
||||
required bytes publicKey = 4;
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
import "lbrynet/schema/proto/stream.proto";
|
||||
import "lbrynet/schema/proto/certificate.proto";
|
||||
import "lbrynet/schema/proto/signature.proto";
|
||||
|
||||
message Claim {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
}
|
||||
required Version version = 1;
|
||||
enum ClaimType {
|
||||
UNKNOWN_CLAIM_TYPE = 0;
|
||||
streamType = 1;
|
||||
certificateType = 2;
|
||||
}
|
||||
required ClaimType claimType = 2;
|
||||
optional Stream stream = 3;
|
||||
optional Certificate certificate = 4;
|
||||
optional Signature publisherSignature = 5;
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
message Fee {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
}
|
||||
enum Currency {
|
||||
UNKNOWN_CURRENCY = 0;
|
||||
LBC = 1;
|
||||
BTC = 2;
|
||||
USD = 3;
|
||||
}
|
||||
required Version version = 1;
|
||||
required Currency currency = 2;
|
||||
required bytes address = 3;
|
||||
required float amount = 4;
|
||||
}
|
||||
@@ -1,212 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
import "lbrynet/schema/proto/fee.proto";
|
||||
|
||||
message Metadata {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
_0_0_2 = 2;
|
||||
_0_0_3 = 3;
|
||||
_0_1_0 = 4;
|
||||
}
|
||||
enum Language {
|
||||
UNKNOWN_LANGUAGE = 0;
|
||||
en = 1;
|
||||
aa = 2;
|
||||
ab = 3;
|
||||
ae = 4;
|
||||
af = 5;
|
||||
ak = 6;
|
||||
am = 7;
|
||||
an = 8;
|
||||
ar = 9;
|
||||
as = 10;
|
||||
av = 11;
|
||||
ay = 12;
|
||||
az = 13;
|
||||
ba = 14;
|
||||
be = 15;
|
||||
bg = 16;
|
||||
bh = 17;
|
||||
bi = 18;
|
||||
bm = 19;
|
||||
bn = 20;
|
||||
bo = 21;
|
||||
br = 22;
|
||||
bs = 23;
|
||||
ca = 24;
|
||||
ce = 25;
|
||||
ch = 26;
|
||||
co = 27;
|
||||
cr = 28;
|
||||
cs = 29;
|
||||
cu = 30;
|
||||
cv = 31;
|
||||
cy = 32;
|
||||
da = 33;
|
||||
de = 34;
|
||||
dv = 35;
|
||||
dz = 36;
|
||||
ee = 37;
|
||||
el = 38;
|
||||
eo = 39;
|
||||
es = 40;
|
||||
et = 41;
|
||||
eu = 42;
|
||||
fa = 43;
|
||||
ff = 44;
|
||||
fi = 45;
|
||||
fj = 46;
|
||||
fo = 47;
|
||||
fr = 48;
|
||||
fy = 49;
|
||||
ga = 50;
|
||||
gd = 51;
|
||||
gl = 52;
|
||||
gn = 53;
|
||||
gu = 54;
|
||||
gv = 55;
|
||||
ha = 56;
|
||||
he = 57;
|
||||
hi = 58;
|
||||
ho = 59;
|
||||
hr = 60;
|
||||
ht = 61;
|
||||
hu = 62;
|
||||
hy = 63;
|
||||
hz = 64;
|
||||
ia = 65;
|
||||
id = 66;
|
||||
ie = 67;
|
||||
ig = 68;
|
||||
ii = 69;
|
||||
ik = 70;
|
||||
io = 71;
|
||||
is = 72;
|
||||
it = 73;
|
||||
iu = 74;
|
||||
ja = 75;
|
||||
jv = 76;
|
||||
ka = 77;
|
||||
kg = 78;
|
||||
ki = 79;
|
||||
kj = 80;
|
||||
kk = 81;
|
||||
kl = 82;
|
||||
km = 83;
|
||||
kn = 84;
|
||||
ko = 85;
|
||||
kr = 86;
|
||||
ks = 87;
|
||||
ku = 88;
|
||||
kv = 89;
|
||||
kw = 90;
|
||||
ky = 91;
|
||||
la = 92;
|
||||
lb = 93;
|
||||
lg = 94;
|
||||
li = 95;
|
||||
ln = 96;
|
||||
lo = 97;
|
||||
lt = 98;
|
||||
lu = 99;
|
||||
lv = 100;
|
||||
mg = 101;
|
||||
mh = 102;
|
||||
mi = 103;
|
||||
mk = 104;
|
||||
ml = 105;
|
||||
mn = 106;
|
||||
mr = 107;
|
||||
ms = 108;
|
||||
mt = 109;
|
||||
my = 110;
|
||||
na = 111;
|
||||
nb = 112;
|
||||
nd = 113;
|
||||
ne = 114;
|
||||
ng = 115;
|
||||
nl = 116;
|
||||
nn = 117;
|
||||
no = 118;
|
||||
nr = 119;
|
||||
nv = 120;
|
||||
ny = 121;
|
||||
oc = 122;
|
||||
oj = 123;
|
||||
om = 124;
|
||||
or = 125;
|
||||
os = 126;
|
||||
pa = 127;
|
||||
pi = 128;
|
||||
pl = 129;
|
||||
ps = 130;
|
||||
pt = 131;
|
||||
qu = 132;
|
||||
rm = 133;
|
||||
rn = 134;
|
||||
ro = 135;
|
||||
ru = 136;
|
||||
rw = 137;
|
||||
sa = 138;
|
||||
sc = 139;
|
||||
sd = 140;
|
||||
se = 141;
|
||||
sg = 142;
|
||||
si = 143;
|
||||
sk = 144;
|
||||
sl = 145;
|
||||
sm = 146;
|
||||
sn = 147;
|
||||
so = 148;
|
||||
sq = 149;
|
||||
sr = 150;
|
||||
ss = 151;
|
||||
st = 152;
|
||||
su = 153;
|
||||
sv = 154;
|
||||
sw = 155;
|
||||
ta = 156;
|
||||
te = 157;
|
||||
tg = 158;
|
||||
th = 159;
|
||||
ti = 160;
|
||||
tk = 161;
|
||||
tl = 162;
|
||||
tn = 163;
|
||||
to = 164;
|
||||
tr = 165;
|
||||
ts = 166;
|
||||
tt = 167;
|
||||
tw = 168;
|
||||
ty = 169;
|
||||
ug = 170;
|
||||
uk = 171;
|
||||
ur = 172;
|
||||
uz = 173;
|
||||
ve = 174;
|
||||
vi = 175;
|
||||
vo = 176;
|
||||
wa = 177;
|
||||
wo = 178;
|
||||
xh = 179;
|
||||
yi = 180;
|
||||
yo = 181;
|
||||
za = 182;
|
||||
zh = 183;
|
||||
zu = 184;
|
||||
}
|
||||
required Version version = 1;
|
||||
required Language language = 2;
|
||||
required string title = 3;
|
||||
required string description = 4;
|
||||
required string author = 5;
|
||||
required string license = 6;
|
||||
required bool nsfw = 7;
|
||||
|
||||
optional Fee fee = 8;
|
||||
optional string thumbnail = 9;
|
||||
optional string preview = 10;
|
||||
optional string licenseUrl = 11;
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
import "lbrynet/schema/proto/certificate.proto";
|
||||
|
||||
message Signature {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
}
|
||||
required Version version = 1;
|
||||
required KeyType signatureType = 2;
|
||||
required bytes signature = 3;
|
||||
required bytes certificateId = 4;
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
message Source {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
}
|
||||
required Version version = 1;
|
||||
enum SourceTypes {
|
||||
UNKNOWN_SOURCE_TYPE = 0;
|
||||
lbry_sd_hash = 1;
|
||||
}
|
||||
required SourceTypes sourceType = 2;
|
||||
required bytes source = 3;
|
||||
required string contentType = 4;
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
import "lbrynet/schema/proto/metadata.proto";
|
||||
import "lbrynet/schema/proto/source.proto";
|
||||
|
||||
|
||||
message Stream {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
}
|
||||
required Version version = 1;
|
||||
required Metadata metadata = 2;
|
||||
required Source source = 3;
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
V_0_0_1 = "_0_0_1"
|
||||
V_0_0_2 = "_0_0_2"
|
||||
V_0_0_3 = "_0_0_3"
|
||||
V_0_1_0 = "_0_1_0"
|
||||
|
||||
|
||||
VERSION_MAP = {
|
||||
V_0_0_1: 1,
|
||||
V_0_0_2: 2,
|
||||
V_0_0_3: 3,
|
||||
V_0_1_0: 4,
|
||||
}
|
||||
|
||||
VERSION_NAMES = {
|
||||
1: V_0_0_1,
|
||||
2: V_0_0_2,
|
||||
3: V_0_0_3,
|
||||
4: V_0_1_0
|
||||
}
|
||||
|
||||
LBC = "LBC"
|
||||
BTC = "BTC"
|
||||
USD = "USD"
|
||||
|
||||
CURRENCY_MAP = {
|
||||
LBC: 1,
|
||||
BTC: 2,
|
||||
USD: 3
|
||||
}
|
||||
|
||||
CURRENCY_NAMES = {
|
||||
1: LBC,
|
||||
2: BTC,
|
||||
3: USD
|
||||
}
|
||||
|
||||
ADDRESS_LENGTH = 25
|
||||
ADDRESS_CHECKSUM_LENGTH = 4
|
||||
|
||||
VERSION = "version"
|
||||
STREAM_TYPE = "streamType"
|
||||
CERTIFICATE_TYPE = "certificateType"
|
||||
CLAIM_TYPE = "claimType"
|
||||
SIGNATURE = "publisherSignature"
|
||||
|
||||
CLAIM_TYPES = {
|
||||
STREAM_TYPE: "stream",
|
||||
CERTIFICATE_TYPE: "certificate"
|
||||
}
|
||||
|
||||
CLAIM_TYPE_NAMES = {
|
||||
1: "stream",
|
||||
2: "certificate"
|
||||
}
|
||||
|
||||
LBRY_SD_HASH = "lbry_sd_hash"
|
||||
LBRY_SD_HASH_LENGTH = 48
|
||||
|
||||
SOURCE_TYPES = {
|
||||
LBRY_SD_HASH: 1
|
||||
}
|
||||
|
||||
NIST256p = "NIST256p"
|
||||
NIST384p = "NIST384p"
|
||||
SECP256k1 = "SECP256k1"
|
||||
|
||||
ECDSA_CURVES = {
|
||||
NIST256p: 1,
|
||||
NIST384p: 2,
|
||||
SECP256k1: 3
|
||||
}
|
||||
|
||||
CURVE_NAMES = {
|
||||
1: NIST256p,
|
||||
2: NIST384p,
|
||||
3: SECP256k1
|
||||
}
|
||||
|
||||
SHA256 = "sha256"
|
||||
SHA384 = "sha384"
|
||||
|
||||
|
||||
B58_CHARS = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
|
||||
assert len(B58_CHARS) == 58
|
||||
|
||||
PUBKEY_ADDRESS = 0
|
||||
SCRIPT_ADDRESS = 5
|
||||
|
||||
ADDRESS_PREFIXES = {
|
||||
"lbrycrd_main": {
|
||||
PUBKEY_ADDRESS: 85,
|
||||
SCRIPT_ADDRESS: 122
|
||||
},
|
||||
"lbrycrd_regtest": {
|
||||
PUBKEY_ADDRESS: 111,
|
||||
SCRIPT_ADDRESS: 196
|
||||
},
|
||||
"lbrycrd_testnet": {
|
||||
PUBKEY_ADDRESS: 111,
|
||||
SCRIPT_ADDRESS: 196
|
||||
},
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
from copy import deepcopy
|
||||
|
||||
from lbrynet.schema.proto import certificate_pb2 as cert_pb
|
||||
from lbrynet.schema.schema.schema import Schema
|
||||
from lbrynet.schema.schema import VERSION_MAP, V_0_0_1, ECDSA_CURVES, CURVE_NAMES
|
||||
from lbrynet.schema.validator import get_key_type_from_dem
|
||||
|
||||
|
||||
class _ECDSAKeyHelper(object):
|
||||
def __init__(self, key):
|
||||
self._key = key
|
||||
|
||||
@property
|
||||
def der(self):
|
||||
return self._key.to_der()
|
||||
|
||||
@property
|
||||
def curve_name(self):
|
||||
return self._key.curve.name
|
||||
|
||||
|
||||
class Certificate(Schema):
|
||||
@classmethod
|
||||
def load(cls, message):
|
||||
_key = deepcopy(message)
|
||||
_message_pb = cert_pb.Certificate()
|
||||
if isinstance(_key, dict):
|
||||
_message_pb.publicKey = _key.pop("publicKey")
|
||||
_message_pb.version = VERSION_MAP[_key.pop("version")]
|
||||
_message_pb.keyType = ECDSA_CURVES[_key.pop("keyType")]
|
||||
else:
|
||||
_message_pb.version = _key.version
|
||||
_message_pb.keyType = _key.keyType
|
||||
_message_pb.publicKey = _key.publicKey
|
||||
if _message_pb.keyType not in CURVE_NAMES:
|
||||
raise Exception("Unknown curve")
|
||||
if get_key_type_from_dem(_message_pb.publicKey) != _message_pb.keyType:
|
||||
raise Exception("Curve mismatch")
|
||||
return cls._load(_key, _message_pb)
|
||||
|
||||
@classmethod
|
||||
def load_from_key_obj(cls, key, key_type):
|
||||
if key_type in ECDSA_CURVES:
|
||||
_key = _ECDSAKeyHelper(key)
|
||||
else:
|
||||
raise Exception("Unknown key type: %s" % str(type(key)))
|
||||
if key_type != _key.curve_name:
|
||||
raise Exception("Curve mismatch")
|
||||
msg = {
|
||||
"version": V_0_0_1,
|
||||
"keyType": key_type,
|
||||
"publicKey": _key.der,
|
||||
}
|
||||
return cls.load(msg)
|
||||
@@ -1,51 +0,0 @@
|
||||
from copy import deepcopy
|
||||
|
||||
from lbrynet.schema.proto import claim_pb2 as claim_pb
|
||||
from lbrynet.schema.schema import VERSION_MAP
|
||||
from lbrynet.schema.schema.signature import Signature
|
||||
from lbrynet.schema.schema.certificate import Certificate
|
||||
from lbrynet.schema.schema.schema import Schema
|
||||
from lbrynet.schema.schema.stream import Stream
|
||||
|
||||
|
||||
class Claim(Schema):
|
||||
CLAIM_TYPE_STREAM = 1
|
||||
CLAIM_TYPE_CERT = 2
|
||||
|
||||
@classmethod
|
||||
def load(cls, message):
|
||||
_claim = deepcopy(message)
|
||||
_message_pb = claim_pb.Claim()
|
||||
_message_pb.version = VERSION_MAP[_claim.pop("version")]
|
||||
|
||||
if "certificate" in _claim:
|
||||
_cert = _claim.pop("certificate")
|
||||
if isinstance(_cert, dict):
|
||||
cert = Certificate.load(_cert)
|
||||
else:
|
||||
cert = _cert
|
||||
claim_type = Claim.CLAIM_TYPE_CERT
|
||||
_message_pb.certificate.MergeFrom(cert)
|
||||
|
||||
elif "stream" in _claim:
|
||||
_stream = _claim.pop("stream")
|
||||
if isinstance(_stream, dict):
|
||||
stream = Stream.load(_stream)
|
||||
else:
|
||||
stream = _stream
|
||||
claim_type = Claim.CLAIM_TYPE_STREAM
|
||||
_message_pb.stream.MergeFrom(stream)
|
||||
else:
|
||||
raise AttributeError
|
||||
|
||||
_message_pb.claimType = claim_type
|
||||
|
||||
if "publisherSignature" in _claim:
|
||||
_publisherSignature = _claim.pop("publisherSignature")
|
||||
if isinstance(_publisherSignature, dict):
|
||||
publisherSignature = Signature.load(_publisherSignature)
|
||||
else:
|
||||
publisherSignature = _publisherSignature
|
||||
_message_pb.publisherSignature.MergeFrom(publisherSignature)
|
||||
|
||||
return cls._load(_claim, _message_pb)
|
||||
@@ -1,17 +0,0 @@
|
||||
from copy import deepcopy
|
||||
|
||||
from lbrynet.schema.schema.schema import Schema
|
||||
from lbrynet.schema.proto import fee_pb2 as fee_pb
|
||||
from lbrynet.schema.schema import VERSION_MAP, CURRENCY_MAP
|
||||
|
||||
|
||||
class Fee(Schema):
|
||||
@classmethod
|
||||
def load(cls, message):
|
||||
_fee = deepcopy(message)
|
||||
currency = CURRENCY_MAP[_fee.pop('currency')]
|
||||
_message_pb = fee_pb.Fee()
|
||||
_message_pb.version = VERSION_MAP[_fee.pop("version")]
|
||||
_message_pb.currency = currency
|
||||
_message_pb.address = _fee.pop('address')
|
||||
return cls._load(_fee, _message_pb)
|
||||
@@ -1,17 +0,0 @@
|
||||
from copy import deepcopy
|
||||
from lbrynet.schema.proto import metadata_pb2 as metadata_pb
|
||||
from lbrynet.schema.schema.fee import Fee
|
||||
from lbrynet.schema.schema.schema import Schema
|
||||
from lbrynet.schema.schema import VERSION_MAP
|
||||
|
||||
|
||||
class Metadata(Schema):
|
||||
@classmethod
|
||||
def load(cls, message):
|
||||
_metadata = deepcopy(message)
|
||||
_message_pb = metadata_pb.Metadata()
|
||||
_message_pb.version = VERSION_MAP[_metadata.pop("version")]
|
||||
if 'fee' in _metadata:
|
||||
fee_pb = Fee.load(_metadata.pop('fee'))
|
||||
_message_pb.fee.CopyFrom(fee_pb)
|
||||
return cls._load(_metadata, _message_pb)
|
||||
@@ -1,15 +0,0 @@
|
||||
import json
|
||||
import google.protobuf.json_format as json_pb # pylint: disable=no-name-in-module
|
||||
from google.protobuf.message import Message # pylint: disable=no-name-in-module,import-error
|
||||
|
||||
|
||||
class Schema(Message):
|
||||
@classmethod
|
||||
def load(cls, message):
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def _load(cls, data, message):
|
||||
if isinstance(data, dict):
|
||||
data = json.dumps(data)
|
||||
return json_pb.Parse(data, message)
|
||||
@@ -1,17 +0,0 @@
|
||||
from copy import deepcopy
|
||||
|
||||
from lbrynet.schema.proto import signature_pb2 as signature_pb
|
||||
from lbrynet.schema.schema import VERSION_MAP, ECDSA_CURVES
|
||||
from lbrynet.schema.schema.schema import Schema
|
||||
|
||||
|
||||
class Signature(Schema):
|
||||
@classmethod
|
||||
def load(cls, message):
|
||||
_signature = deepcopy(message)
|
||||
_message_pb = signature_pb.Signature()
|
||||
_message_pb.version = VERSION_MAP[_signature.pop("version")]
|
||||
_message_pb.signatureType = ECDSA_CURVES[_signature.pop("signatureType")]
|
||||
_message_pb.certificateId = _signature.pop("certificateId")
|
||||
_message_pb.signature = _signature.pop("signature")
|
||||
return cls._load(_signature, _message_pb)
|
||||
@@ -1,19 +0,0 @@
|
||||
from copy import deepcopy
|
||||
from lbrynet.schema.proto import source_pb2 as source_pb
|
||||
from lbrynet.schema.schema import SOURCE_TYPES, LBRY_SD_HASH_LENGTH, VERSION_MAP
|
||||
from lbrynet.schema.schema.schema import Schema
|
||||
from lbrynet.schema.error import InvalidSourceHashLength
|
||||
|
||||
|
||||
class Source(Schema):
|
||||
@classmethod
|
||||
def load(cls, message):
|
||||
_source = deepcopy(message)
|
||||
sd_hash = _source.pop('source')
|
||||
assert len(sd_hash) == LBRY_SD_HASH_LENGTH, InvalidSourceHashLength(len(sd_hash))
|
||||
_message_pb = source_pb.Source()
|
||||
_message_pb.version = VERSION_MAP[_source.pop("version")]
|
||||
_message_pb.sourceType = SOURCE_TYPES[_source.pop('sourceType')]
|
||||
_message_pb.source = sd_hash
|
||||
_message_pb.contentType = _source.pop('contentType')
|
||||
return cls._load(_source, _message_pb)
|
||||
@@ -1,20 +0,0 @@
|
||||
from copy import deepcopy
|
||||
|
||||
from lbrynet.schema.schema.source import Source
|
||||
from lbrynet.schema.proto import stream_pb2 as stream_pb
|
||||
from lbrynet.schema.schema import VERSION_MAP
|
||||
from lbrynet.schema.schema.metadata import Metadata
|
||||
from lbrynet.schema.schema.schema import Schema
|
||||
|
||||
|
||||
class Stream(Schema):
|
||||
@classmethod
|
||||
def load(cls, message):
|
||||
_claim = deepcopy(message)
|
||||
source = Source.load(_claim.pop('source'))
|
||||
metadata = Metadata.load(_claim.pop('metadata'))
|
||||
_message_pb = stream_pb.Stream()
|
||||
_message_pb.version = VERSION_MAP[_claim.pop("version")]
|
||||
_message_pb.source.CopyFrom(source)
|
||||
_message_pb.metadata.CopyFrom(metadata)
|
||||
return cls._load(_claim, _message_pb)
|
||||
@@ -1,44 +0,0 @@
|
||||
from collections import namedtuple
|
||||
|
||||
from typing import Union
|
||||
|
||||
LEGACY = namedtuple('Legacy', 'payload')
|
||||
NAMED_SECP256K1 = namedtuple('NamedSECP256k1', 'raw_signature certificate_id payload')
|
||||
FLAGS = {
|
||||
LEGACY: 0x80,
|
||||
NAMED_SECP256K1: 0x01
|
||||
}
|
||||
|
||||
class Signature:
|
||||
|
||||
def __init__(self, data: Union[LEGACY, NAMED_SECP256K1]):
|
||||
self.data = data
|
||||
|
||||
@property
|
||||
def payload(self):
|
||||
return self.data.payload
|
||||
|
||||
@property
|
||||
def certificate_id(self):
|
||||
if type(self.data) == NAMED_SECP256K1:
|
||||
return self.data.certificate_id
|
||||
|
||||
@property
|
||||
def raw_signature(self):
|
||||
if type(self.data) == NAMED_SECP256K1:
|
||||
return self.data.raw_signature
|
||||
|
||||
@classmethod
|
||||
def flagged_parse(cls, binary: bytes):
|
||||
flag = binary[0]
|
||||
if flag == FLAGS[NAMED_SECP256K1]:
|
||||
return cls(NAMED_SECP256K1(binary[1:65], binary[65:85], binary[85:]))
|
||||
else:
|
||||
return cls(LEGACY(binary))
|
||||
|
||||
@property
|
||||
def serialized(self):
|
||||
if isinstance(self.data, NAMED_SECP256K1):
|
||||
return (bytes([FLAGS[type(self.data)]]) + self.data.raw_signature + self.data.certificate_id + self.payload)
|
||||
elif isinstance(self.data, LEGACY):
|
||||
return self.payload
|
||||
@@ -1,121 +0,0 @@
|
||||
import ecdsa
|
||||
import hashlib
|
||||
import binascii
|
||||
from lbrynet.schema.address import decode_address
|
||||
from lbrynet.schema.encoding import decode_b64_fields
|
||||
from lbrynet.schema.schema.certificate import Certificate
|
||||
from lbrynet.schema.schema.claim import Claim
|
||||
from lbrynet.schema.signature import Signature, NAMED_SECP256K1
|
||||
from lbrynet.schema.validator import validate_claim_id
|
||||
from lbrynet.schema.schema import V_0_0_1, CLAIM_TYPE, CLAIM_TYPES, CERTIFICATE_TYPE, VERSION
|
||||
from lbrynet.schema.schema import NIST256p, NIST384p, SECP256k1, SHA256, SHA384
|
||||
|
||||
|
||||
class NIST_ECDSASigner(object):
|
||||
CURVE = None
|
||||
CURVE_NAME = None
|
||||
HASHFUNC = hashlib.sha256
|
||||
HASHFUNC_NAME = SHA256
|
||||
|
||||
def __init__(self, private_key):
|
||||
self._private_key = private_key
|
||||
|
||||
@property
|
||||
def private_key(self):
|
||||
return self._private_key
|
||||
|
||||
@property
|
||||
def public_key(self):
|
||||
return self.private_key.get_verifying_key()
|
||||
|
||||
@property
|
||||
def certificate(self):
|
||||
certificate_claim = {
|
||||
VERSION: V_0_0_1,
|
||||
CLAIM_TYPE: CERTIFICATE_TYPE,
|
||||
CLAIM_TYPES[CERTIFICATE_TYPE]: Certificate.load_from_key_obj(self.public_key,
|
||||
self.CURVE_NAME)
|
||||
}
|
||||
return Claim.load(certificate_claim)
|
||||
|
||||
@classmethod
|
||||
def load_pem(cls, pem_string):
|
||||
return cls(ecdsa.SigningKey.from_pem(pem_string, hashfunc=cls.HASHFUNC_NAME))
|
||||
|
||||
@classmethod
|
||||
def generate(cls):
|
||||
return cls(ecdsa.SigningKey.generate(curve=cls.CURVE, hashfunc=cls.HASHFUNC_NAME))
|
||||
|
||||
def sign(self, *fields):
|
||||
digest = self.HASHFUNC(bytearray(b''.join(fields))).digest()
|
||||
return self.private_key.sign_digest_deterministic(digest, hashfunc=self.HASHFUNC)
|
||||
|
||||
def sign_stream_claim(self, claim, claim_address, cert_claim_id, name, detached=False):
|
||||
validate_claim_id(cert_claim_id)
|
||||
raw_cert_id = binascii.unhexlify(cert_claim_id)
|
||||
decoded_addr = decode_address(claim_address)
|
||||
if detached:
|
||||
assert name, "Name is required for detached signatures"
|
||||
assert self.CURVE_NAME == SECP256k1, f"Only SECP256k1 is supported, not: {self.CURVE_NAME}"
|
||||
signature = self.sign(
|
||||
name.lower().encode(),
|
||||
decoded_addr,
|
||||
claim.serialized_no_signature,
|
||||
raw_cert_id,
|
||||
)
|
||||
else:
|
||||
signature = self.sign(decoded_addr, claim.serialized_no_signature, raw_cert_id)
|
||||
|
||||
if detached:
|
||||
return Claim.load(decode_b64_fields(claim.protobuf_dict)), Signature(NAMED_SECP256K1(
|
||||
signature,
|
||||
raw_cert_id,
|
||||
claim.serialized_no_signature
|
||||
))
|
||||
# -- Legacy signer (signature inside protobuf) --
|
||||
|
||||
if not isinstance(self.private_key, ecdsa.SigningKey):
|
||||
raise Exception("Not given a signing key")
|
||||
sig_dict = {
|
||||
"version": V_0_0_1,
|
||||
"signatureType": self.CURVE_NAME,
|
||||
"signature": signature,
|
||||
"certificateId": raw_cert_id
|
||||
}
|
||||
|
||||
msg = {
|
||||
"version": V_0_0_1,
|
||||
"stream": decode_b64_fields(claim.protobuf_dict)['stream'],
|
||||
"publisherSignature": sig_dict
|
||||
}
|
||||
|
||||
proto = Claim.load(msg)
|
||||
return proto, Signature.flagged_parse(proto.SerializeToString())
|
||||
|
||||
|
||||
class NIST256pSigner(NIST_ECDSASigner):
|
||||
CURVE = ecdsa.NIST256p
|
||||
CURVE_NAME = NIST256p
|
||||
|
||||
|
||||
class NIST384pSigner(NIST_ECDSASigner):
|
||||
CURVE = ecdsa.NIST384p
|
||||
CURVE_NAME = NIST384p
|
||||
HASHFUNC = hashlib.sha384
|
||||
HASHFUNC_NAME = SHA384
|
||||
|
||||
|
||||
class SECP256k1Signer(NIST_ECDSASigner):
|
||||
CURVE = ecdsa.SECP256k1
|
||||
CURVE_NAME = SECP256k1
|
||||
|
||||
|
||||
def get_signer(curve):
|
||||
if curve == NIST256p:
|
||||
return NIST256pSigner
|
||||
elif curve == NIST384p:
|
||||
return NIST384pSigner
|
||||
elif curve == SECP256k1:
|
||||
return SECP256k1Signer
|
||||
else:
|
||||
raise Exception("Unknown curve: %s" % str(curve))
|
||||
@@ -0,0 +1,6 @@
|
||||
from lbrynet.schema.base import Signable
|
||||
|
||||
|
||||
class Support(Signable):
|
||||
__slots__ = ()
|
||||
message_class = None # TODO: add support protobufs
|
||||
File renamed without changes.
File renamed without changes.
+18
-18
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/certificate.proto
|
||||
# source: certificate.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -16,16 +16,16 @@ _sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/certificate.proto',
|
||||
package='',
|
||||
name='certificate.proto',
|
||||
package='legacy_pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n&lbrynet/schema/proto/certificate.proto\"\x8e\x01\n\x0b\x43\x65rtificate\x12%\n\x07version\x18\x01 \x02(\x0e\x32\x14.Certificate.Version\x12\x19\n\x07keyType\x18\x02 \x02(\x0e\x32\x08.KeyType\x12\x11\n\tpublicKey\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01*Q\n\x07KeyType\x12\x1b\n\x17UNKNOWN_PUBLIC_KEY_TYPE\x10\x00\x12\x0c\n\x08NIST256p\x10\x01\x12\x0c\n\x08NIST384p\x10\x02\x12\r\n\tSECP256k1\x10\x03')
|
||||
serialized_pb=_b('\n\x11\x63\x65rtificate.proto\x12\tlegacy_pb\"\xa2\x01\n\x0b\x43\x65rtificate\x12/\n\x07version\x18\x01 \x02(\x0e\x32\x1e.legacy_pb.Certificate.Version\x12#\n\x07keyType\x18\x02 \x02(\x0e\x32\x12.legacy_pb.KeyType\x12\x11\n\tpublicKey\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01*Q\n\x07KeyType\x12\x1b\n\x17UNKNOWN_PUBLIC_KEY_TYPE\x10\x00\x12\x0c\n\x08NIST256p\x10\x01\x12\x0c\n\x08NIST384p\x10\x02\x12\r\n\tSECP256k1\x10\x03')
|
||||
)
|
||||
|
||||
_KEYTYPE = _descriptor.EnumDescriptor(
|
||||
name='KeyType',
|
||||
full_name='KeyType',
|
||||
full_name='legacy_pb.KeyType',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -48,8 +48,8 @@ _KEYTYPE = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=187,
|
||||
serialized_end=268,
|
||||
serialized_start=197,
|
||||
serialized_end=278,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_KEYTYPE)
|
||||
|
||||
@@ -62,7 +62,7 @@ SECP256k1 = 3
|
||||
|
||||
_CERTIFICATE_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Certificate.Version',
|
||||
full_name='legacy_pb.Certificate.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -77,35 +77,35 @@ _CERTIFICATE_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=143,
|
||||
serialized_end=185,
|
||||
serialized_start=153,
|
||||
serialized_end=195,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_CERTIFICATE_VERSION)
|
||||
|
||||
|
||||
_CERTIFICATE = _descriptor.Descriptor(
|
||||
name='Certificate',
|
||||
full_name='Certificate',
|
||||
full_name='legacy_pb.Certificate',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Certificate.version', index=0,
|
||||
name='version', full_name='legacy_pb.Certificate.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='keyType', full_name='Certificate.keyType', index=1,
|
||||
name='keyType', full_name='legacy_pb.Certificate.keyType', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='publicKey', full_name='Certificate.publicKey', index=2,
|
||||
name='publicKey', full_name='legacy_pb.Certificate.publicKey', index=2,
|
||||
number=4, type=12, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -124,8 +124,8 @@ _CERTIFICATE = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=43,
|
||||
serialized_end=185,
|
||||
serialized_start=33,
|
||||
serialized_end=195,
|
||||
)
|
||||
|
||||
_CERTIFICATE.fields_by_name['version'].enum_type = _CERTIFICATE_VERSION
|
||||
@@ -137,8 +137,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Certificate = _reflection.GeneratedProtocolMessageType('Certificate', (_message.Message,), dict(
|
||||
DESCRIPTOR = _CERTIFICATE,
|
||||
__module__ = 'lbrynet.schema.proto.certificate_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Certificate)
|
||||
__module__ = 'certificate_pb2'
|
||||
# @@protoc_insertion_point(class_scope:legacy_pb.Certificate)
|
||||
))
|
||||
_sym_db.RegisterMessage(Certificate)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/fee.proto
|
||||
# source: fee.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -15,18 +15,18 @@ _sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/fee.proto',
|
||||
package='',
|
||||
name='fee.proto',
|
||||
package='legacy_pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n\x1elbrynet/schema/proto/fee.proto\"\xcf\x01\n\x03\x46\x65\x65\x12\x1d\n\x07version\x18\x01 \x02(\x0e\x32\x0c.Fee.Version\x12\x1f\n\x08\x63urrency\x18\x02 \x02(\x0e\x32\r.Fee.Currency\x12\x0f\n\x07\x61\x64\x64ress\x18\x03 \x02(\x0c\x12\x0e\n\x06\x61mount\x18\x04 \x02(\x02\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\";\n\x08\x43urrency\x12\x14\n\x10UNKNOWN_CURRENCY\x10\x00\x12\x07\n\x03LBC\x10\x01\x12\x07\n\x03\x42TC\x10\x02\x12\x07\n\x03USD\x10\x03')
|
||||
serialized_pb=_b('\n\tfee.proto\x12\tlegacy_pb\"\xe3\x01\n\x03\x46\x65\x65\x12\'\n\x07version\x18\x01 \x02(\x0e\x32\x16.legacy_pb.Fee.Version\x12)\n\x08\x63urrency\x18\x02 \x02(\x0e\x32\x17.legacy_pb.Fee.Currency\x12\x0f\n\x07\x61\x64\x64ress\x18\x03 \x02(\x0c\x12\x0e\n\x06\x61mount\x18\x04 \x02(\x02\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\";\n\x08\x43urrency\x12\x14\n\x10UNKNOWN_CURRENCY\x10\x00\x12\x07\n\x03LBC\x10\x01\x12\x07\n\x03\x42TC\x10\x02\x12\x07\n\x03USD\x10\x03')
|
||||
)
|
||||
|
||||
|
||||
|
||||
_FEE_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Fee.Version',
|
||||
full_name='legacy_pb.Fee.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -41,14 +41,14 @@ _FEE_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=139,
|
||||
serialized_end=181,
|
||||
serialized_start=149,
|
||||
serialized_end=191,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_FEE_VERSION)
|
||||
|
||||
_FEE_CURRENCY = _descriptor.EnumDescriptor(
|
||||
name='Currency',
|
||||
full_name='Fee.Currency',
|
||||
full_name='legacy_pb.Fee.Currency',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -71,42 +71,42 @@ _FEE_CURRENCY = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=183,
|
||||
serialized_end=242,
|
||||
serialized_start=193,
|
||||
serialized_end=252,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_FEE_CURRENCY)
|
||||
|
||||
|
||||
_FEE = _descriptor.Descriptor(
|
||||
name='Fee',
|
||||
full_name='Fee',
|
||||
full_name='legacy_pb.Fee',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Fee.version', index=0,
|
||||
name='version', full_name='legacy_pb.Fee.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='currency', full_name='Fee.currency', index=1,
|
||||
name='currency', full_name='legacy_pb.Fee.currency', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='address', full_name='Fee.address', index=2,
|
||||
name='address', full_name='legacy_pb.Fee.address', index=2,
|
||||
number=3, type=12, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='amount', full_name='Fee.amount', index=3,
|
||||
name='amount', full_name='legacy_pb.Fee.amount', index=3,
|
||||
number=4, type=2, cpp_type=6, label=2,
|
||||
has_default_value=False, default_value=float(0),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -126,8 +126,8 @@ _FEE = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=35,
|
||||
serialized_end=242,
|
||||
serialized_start=25,
|
||||
serialized_end=252,
|
||||
)
|
||||
|
||||
_FEE.fields_by_name['version'].enum_type = _FEE_VERSION
|
||||
@@ -139,8 +139,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Fee = _reflection.GeneratedProtocolMessageType('Fee', (_message.Message,), dict(
|
||||
DESCRIPTOR = _FEE,
|
||||
__module__ = 'lbrynet.schema.proto.fee_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Fee)
|
||||
__module__ = 'fee_pb2'
|
||||
# @@protoc_insertion_point(class_scope:legacy_pb.Fee)
|
||||
))
|
||||
_sym_db.RegisterMessage(Fee)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/claim.proto
|
||||
# source: legacy_claim.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -12,25 +12,25 @@ from google.protobuf import symbol_database as _symbol_database
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
from lbrynet.schema.proto import stream_pb2 as lbrynet_dot_schema_dot_proto_dot_stream__pb2
|
||||
from lbrynet.schema.proto import certificate_pb2 as lbrynet_dot_schema_dot_proto_dot_certificate__pb2
|
||||
from lbrynet.schema.proto import signature_pb2 as lbrynet_dot_schema_dot_proto_dot_signature__pb2
|
||||
from . import stream_pb2 as stream__pb2
|
||||
from . import certificate_pb2 as certificate__pb2
|
||||
from . import signature_pb2 as signature__pb2
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/claim.proto',
|
||||
package='',
|
||||
name='legacy_claim.proto',
|
||||
package='legacy_pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n lbrynet/schema/proto/claim.proto\x1a!lbrynet/schema/proto/stream.proto\x1a&lbrynet/schema/proto/certificate.proto\x1a$lbrynet/schema/proto/signature.proto\"\xa7\x02\n\x05\x43laim\x12\x1f\n\x07version\x18\x01 \x02(\x0e\x32\x0e.Claim.Version\x12#\n\tclaimType\x18\x02 \x02(\x0e\x32\x10.Claim.ClaimType\x12\x17\n\x06stream\x18\x03 \x01(\x0b\x32\x07.Stream\x12!\n\x0b\x63\x65rtificate\x18\x04 \x01(\x0b\x32\x0c.Certificate\x12&\n\x12publisherSignature\x18\x05 \x01(\x0b\x32\n.Signature\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"H\n\tClaimType\x12\x16\n\x12UNKNOWN_CLAIM_TYPE\x10\x00\x12\x0e\n\nstreamType\x10\x01\x12\x13\n\x0f\x63\x65rtificateType\x10\x02')
|
||||
serialized_pb=_b('\n\x12legacy_claim.proto\x12\tlegacy_pb\x1a\x0cstream.proto\x1a\x11\x63\x65rtificate.proto\x1a\x0fsignature.proto\"\xd9\x02\n\x05\x43laim\x12)\n\x07version\x18\x01 \x02(\x0e\x32\x18.legacy_pb.Claim.Version\x12-\n\tclaimType\x18\x02 \x02(\x0e\x32\x1a.legacy_pb.Claim.ClaimType\x12!\n\x06stream\x18\x03 \x01(\x0b\x32\x11.legacy_pb.Stream\x12+\n\x0b\x63\x65rtificate\x18\x04 \x01(\x0b\x32\x16.legacy_pb.Certificate\x12\x30\n\x12publisherSignature\x18\x05 \x01(\x0b\x32\x14.legacy_pb.Signature\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"H\n\tClaimType\x12\x16\n\x12UNKNOWN_CLAIM_TYPE\x10\x00\x12\x0e\n\nstreamType\x10\x01\x12\x13\n\x0f\x63\x65rtificateType\x10\x02')
|
||||
,
|
||||
dependencies=[lbrynet_dot_schema_dot_proto_dot_stream__pb2.DESCRIPTOR,lbrynet_dot_schema_dot_proto_dot_certificate__pb2.DESCRIPTOR,lbrynet_dot_schema_dot_proto_dot_signature__pb2.DESCRIPTOR,])
|
||||
dependencies=[stream__pb2.DESCRIPTOR,certificate__pb2.DESCRIPTOR,signature__pb2.DESCRIPTOR,])
|
||||
|
||||
|
||||
|
||||
_CLAIM_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Claim.Version',
|
||||
full_name='legacy_pb.Claim.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -45,14 +45,14 @@ _CLAIM_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=329,
|
||||
serialized_end=371,
|
||||
serialized_start=313,
|
||||
serialized_end=355,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_CLAIM_VERSION)
|
||||
|
||||
_CLAIM_CLAIMTYPE = _descriptor.EnumDescriptor(
|
||||
name='ClaimType',
|
||||
full_name='Claim.ClaimType',
|
||||
full_name='legacy_pb.Claim.ClaimType',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -71,49 +71,49 @@ _CLAIM_CLAIMTYPE = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=373,
|
||||
serialized_end=445,
|
||||
serialized_start=357,
|
||||
serialized_end=429,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_CLAIM_CLAIMTYPE)
|
||||
|
||||
|
||||
_CLAIM = _descriptor.Descriptor(
|
||||
name='Claim',
|
||||
full_name='Claim',
|
||||
full_name='legacy_pb.Claim',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Claim.version', index=0,
|
||||
name='version', full_name='legacy_pb.Claim.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='claimType', full_name='Claim.claimType', index=1,
|
||||
name='claimType', full_name='legacy_pb.Claim.claimType', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='stream', full_name='Claim.stream', index=2,
|
||||
name='stream', full_name='legacy_pb.Claim.stream', index=2,
|
||||
number=3, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='certificate', full_name='Claim.certificate', index=3,
|
||||
name='certificate', full_name='legacy_pb.Claim.certificate', index=3,
|
||||
number=4, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='publisherSignature', full_name='Claim.publisherSignature', index=4,
|
||||
name='publisherSignature', full_name='legacy_pb.Claim.publisherSignature', index=4,
|
||||
number=5, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -133,15 +133,15 @@ _CLAIM = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=150,
|
||||
serialized_end=445,
|
||||
serialized_start=84,
|
||||
serialized_end=429,
|
||||
)
|
||||
|
||||
_CLAIM.fields_by_name['version'].enum_type = _CLAIM_VERSION
|
||||
_CLAIM.fields_by_name['claimType'].enum_type = _CLAIM_CLAIMTYPE
|
||||
_CLAIM.fields_by_name['stream'].message_type = lbrynet_dot_schema_dot_proto_dot_stream__pb2._STREAM
|
||||
_CLAIM.fields_by_name['certificate'].message_type = lbrynet_dot_schema_dot_proto_dot_certificate__pb2._CERTIFICATE
|
||||
_CLAIM.fields_by_name['publisherSignature'].message_type = lbrynet_dot_schema_dot_proto_dot_signature__pb2._SIGNATURE
|
||||
_CLAIM.fields_by_name['stream'].message_type = stream__pb2._STREAM
|
||||
_CLAIM.fields_by_name['certificate'].message_type = certificate__pb2._CERTIFICATE
|
||||
_CLAIM.fields_by_name['publisherSignature'].message_type = signature__pb2._SIGNATURE
|
||||
_CLAIM_VERSION.containing_type = _CLAIM
|
||||
_CLAIM_CLAIMTYPE.containing_type = _CLAIM
|
||||
DESCRIPTOR.message_types_by_name['Claim'] = _CLAIM
|
||||
@@ -149,8 +149,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Claim = _reflection.GeneratedProtocolMessageType('Claim', (_message.Message,), dict(
|
||||
DESCRIPTOR = _CLAIM,
|
||||
__module__ = 'lbrynet.schema.proto.claim_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Claim)
|
||||
__module__ = 'legacy_claim_pb2'
|
||||
# @@protoc_insertion_point(class_scope:legacy_pb.Claim)
|
||||
))
|
||||
_sym_db.RegisterMessage(Claim)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/metadata.proto
|
||||
# source: metadata.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -12,23 +12,23 @@ from google.protobuf import symbol_database as _symbol_database
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
from lbrynet.schema.proto import fee_pb2 as lbrynet_dot_schema_dot_proto_dot_fee__pb2
|
||||
from . import fee_pb2 as fee__pb2
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/metadata.proto',
|
||||
package='',
|
||||
name='metadata.proto',
|
||||
package='legacy_pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n#lbrynet/schema/proto/metadata.proto\x1a\x1elbrynet/schema/proto/fee.proto\"\xde\x0e\n\x08Metadata\x12\"\n\x07version\x18\x01 \x02(\x0e\x32\x11.Metadata.Version\x12$\n\x08language\x18\x02 \x02(\x0e\x32\x12.Metadata.Language\x12\r\n\x05title\x18\x03 \x02(\t\x12\x13\n\x0b\x64\x65scription\x18\x04 \x02(\t\x12\x0e\n\x06\x61uthor\x18\x05 \x02(\t\x12\x0f\n\x07license\x18\x06 \x02(\t\x12\x0c\n\x04nsfw\x18\x07 \x02(\x08\x12\x11\n\x03\x66\x65\x65\x18\x08 \x01(\x0b\x32\x04.Fee\x12\x11\n\tthumbnail\x18\t \x01(\t\x12\x0f\n\x07preview\x18\n \x01(\t\x12\x12\n\nlicenseUrl\x18\x0b \x01(\t\"N\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\x12\n\n\x06_0_0_2\x10\x02\x12\n\n\x06_0_0_3\x10\x03\x12\n\n\x06_0_1_0\x10\x04\"\x99\x0c\n\x08Language\x12\x14\n\x10UNKNOWN_LANGUAGE\x10\x00\x12\x06\n\x02\x65n\x10\x01\x12\x06\n\x02\x61\x61\x10\x02\x12\x06\n\x02\x61\x62\x10\x03\x12\x06\n\x02\x61\x65\x10\x04\x12\x06\n\x02\x61\x66\x10\x05\x12\x06\n\x02\x61k\x10\x06\x12\x06\n\x02\x61m\x10\x07\x12\x06\n\x02\x61n\x10\x08\x12\x06\n\x02\x61r\x10\t\x12\x06\n\x02\x61s\x10\n\x12\x06\n\x02\x61v\x10\x0b\x12\x06\n\x02\x61y\x10\x0c\x12\x06\n\x02\x61z\x10\r\x12\x06\n\x02\x62\x61\x10\x0e\x12\x06\n\x02\x62\x65\x10\x0f\x12\x06\n\x02\x62g\x10\x10\x12\x06\n\x02\x62h\x10\x11\x12\x06\n\x02\x62i\x10\x12\x12\x06\n\x02\x62m\x10\x13\x12\x06\n\x02\x62n\x10\x14\x12\x06\n\x02\x62o\x10\x15\x12\x06\n\x02\x62r\x10\x16\x12\x06\n\x02\x62s\x10\x17\x12\x06\n\x02\x63\x61\x10\x18\x12\x06\n\x02\x63\x65\x10\x19\x12\x06\n\x02\x63h\x10\x1a\x12\x06\n\x02\x63o\x10\x1b\x12\x06\n\x02\x63r\x10\x1c\x12\x06\n\x02\x63s\x10\x1d\x12\x06\n\x02\x63u\x10\x1e\x12\x06\n\x02\x63v\x10\x1f\x12\x06\n\x02\x63y\x10 \x12\x06\n\x02\x64\x61\x10!\x12\x06\n\x02\x64\x65\x10\"\x12\x06\n\x02\x64v\x10#\x12\x06\n\x02\x64z\x10$\x12\x06\n\x02\x65\x65\x10%\x12\x06\n\x02\x65l\x10&\x12\x06\n\x02\x65o\x10\'\x12\x06\n\x02\x65s\x10(\x12\x06\n\x02\x65t\x10)\x12\x06\n\x02\x65u\x10*\x12\x06\n\x02\x66\x61\x10+\x12\x06\n\x02\x66\x66\x10,\x12\x06\n\x02\x66i\x10-\x12\x06\n\x02\x66j\x10.\x12\x06\n\x02\x66o\x10/\x12\x06\n\x02\x66r\x10\x30\x12\x06\n\x02\x66y\x10\x31\x12\x06\n\x02ga\x10\x32\x12\x06\n\x02gd\x10\x33\x12\x06\n\x02gl\x10\x34\x12\x06\n\x02gn\x10\x35\x12\x06\n\x02gu\x10\x36\x12\x06\n\x02gv\x10\x37\x12\x06\n\x02ha\x10\x38\x12\x06\n\x02he\x10\x39\x12\x06\n\x02hi\x10:\x12\x06\n\x02ho\x10;\x12\x06\n\x02hr\x10<\x12\x06\n\x02ht\x10=\x12\x06\n\x02hu\x10>\x12\x06\n\x02hy\x10?\x12\x06\n\x02hz\x10@\x12\x06\n\x02ia\x10\x41\x12\x06\n\x02id\x10\x42\x12\x06\n\x02ie\x10\x43\x12\x06\n\x02ig\x10\x44\x12\x06\n\x02ii\x10\x45\x12\x06\n\x02ik\x10\x46\x12\x06\n\x02io\x10G\x12\x06\n\x02is\x10H\x12\x06\n\x02it\x10I\x12\x06\n\x02iu\x10J\x12\x06\n\x02ja\x10K\x12\x06\n\x02jv\x10L\x12\x06\n\x02ka\x10M\x12\x06\n\x02kg\x10N\x12\x06\n\x02ki\x10O\x12\x06\n\x02kj\x10P\x12\x06\n\x02kk\x10Q\x12\x06\n\x02kl\x10R\x12\x06\n\x02km\x10S\x12\x06\n\x02kn\x10T\x12\x06\n\x02ko\x10U\x12\x06\n\x02kr\x10V\x12\x06\n\x02ks\x10W\x12\x06\n\x02ku\x10X\x12\x06\n\x02kv\x10Y\x12\x06\n\x02kw\x10Z\x12\x06\n\x02ky\x10[\x12\x06\n\x02la\x10\\\x12\x06\n\x02lb\x10]\x12\x06\n\x02lg\x10^\x12\x06\n\x02li\x10_\x12\x06\n\x02ln\x10`\x12\x06\n\x02lo\x10\x61\x12\x06\n\x02lt\x10\x62\x12\x06\n\x02lu\x10\x63\x12\x06\n\x02lv\x10\x64\x12\x06\n\x02mg\x10\x65\x12\x06\n\x02mh\x10\x66\x12\x06\n\x02mi\x10g\x12\x06\n\x02mk\x10h\x12\x06\n\x02ml\x10i\x12\x06\n\x02mn\x10j\x12\x06\n\x02mr\x10k\x12\x06\n\x02ms\x10l\x12\x06\n\x02mt\x10m\x12\x06\n\x02my\x10n\x12\x06\n\x02na\x10o\x12\x06\n\x02nb\x10p\x12\x06\n\x02nd\x10q\x12\x06\n\x02ne\x10r\x12\x06\n\x02ng\x10s\x12\x06\n\x02nl\x10t\x12\x06\n\x02nn\x10u\x12\x06\n\x02no\x10v\x12\x06\n\x02nr\x10w\x12\x06\n\x02nv\x10x\x12\x06\n\x02ny\x10y\x12\x06\n\x02oc\x10z\x12\x06\n\x02oj\x10{\x12\x06\n\x02om\x10|\x12\x06\n\x02or\x10}\x12\x06\n\x02os\x10~\x12\x06\n\x02pa\x10\x7f\x12\x07\n\x02pi\x10\x80\x01\x12\x07\n\x02pl\x10\x81\x01\x12\x07\n\x02ps\x10\x82\x01\x12\x07\n\x02pt\x10\x83\x01\x12\x07\n\x02qu\x10\x84\x01\x12\x07\n\x02rm\x10\x85\x01\x12\x07\n\x02rn\x10\x86\x01\x12\x07\n\x02ro\x10\x87\x01\x12\x07\n\x02ru\x10\x88\x01\x12\x07\n\x02rw\x10\x89\x01\x12\x07\n\x02sa\x10\x8a\x01\x12\x07\n\x02sc\x10\x8b\x01\x12\x07\n\x02sd\x10\x8c\x01\x12\x07\n\x02se\x10\x8d\x01\x12\x07\n\x02sg\x10\x8e\x01\x12\x07\n\x02si\x10\x8f\x01\x12\x07\n\x02sk\x10\x90\x01\x12\x07\n\x02sl\x10\x91\x01\x12\x07\n\x02sm\x10\x92\x01\x12\x07\n\x02sn\x10\x93\x01\x12\x07\n\x02so\x10\x94\x01\x12\x07\n\x02sq\x10\x95\x01\x12\x07\n\x02sr\x10\x96\x01\x12\x07\n\x02ss\x10\x97\x01\x12\x07\n\x02st\x10\x98\x01\x12\x07\n\x02su\x10\x99\x01\x12\x07\n\x02sv\x10\x9a\x01\x12\x07\n\x02sw\x10\x9b\x01\x12\x07\n\x02ta\x10\x9c\x01\x12\x07\n\x02te\x10\x9d\x01\x12\x07\n\x02tg\x10\x9e\x01\x12\x07\n\x02th\x10\x9f\x01\x12\x07\n\x02ti\x10\xa0\x01\x12\x07\n\x02tk\x10\xa1\x01\x12\x07\n\x02tl\x10\xa2\x01\x12\x07\n\x02tn\x10\xa3\x01\x12\x07\n\x02to\x10\xa4\x01\x12\x07\n\x02tr\x10\xa5\x01\x12\x07\n\x02ts\x10\xa6\x01\x12\x07\n\x02tt\x10\xa7\x01\x12\x07\n\x02tw\x10\xa8\x01\x12\x07\n\x02ty\x1Line truncated
|
||||
serialized_pb=_b('\n\x0emetadata.proto\x12\tlegacy_pb\x1a\tfee.proto\"\xfc\x0e\n\x08Metadata\x12,\n\x07version\x18\x01 \x02(\x0e\x32\x1b.legacy_pb.Metadata.Version\x12.\n\x08language\x18\x02 \x02(\x0e\x32\x1c.legacy_pb.Metadata.Language\x12\r\n\x05title\x18\x03 \x02(\t\x12\x13\n\x0b\x64\x65scription\x18\x04 \x02(\t\x12\x0e\n\x06\x61uthor\x18\x05 \x02(\t\x12\x0f\n\x07license\x18\x06 \x02(\t\x12\x0c\n\x04nsfw\x18\x07 \x02(\x08\x12\x1b\n\x03\x66\x65\x65\x18\x08 \x01(\x0b\x32\x0e.legacy_pb.Fee\x12\x11\n\tthumbnail\x18\t \x01(\t\x12\x0f\n\x07preview\x18\n \x01(\t\x12\x12\n\nlicenseUrl\x18\x0b \x01(\t\"N\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\x12\n\n\x06_0_0_2\x10\x02\x12\n\n\x06_0_0_3\x10\x03\x12\n\n\x06_0_1_0\x10\x04\"\x99\x0c\n\x08Language\x12\x14\n\x10UNKNOWN_LANGUAGE\x10\x00\x12\x06\n\x02\x65n\x10\x01\x12\x06\n\x02\x61\x61\x10\x02\x12\x06\n\x02\x61\x62\x10\x03\x12\x06\n\x02\x61\x65\x10\x04\x12\x06\n\x02\x61\x66\x10\x05\x12\x06\n\x02\x61k\x10\x06\x12\x06\n\x02\x61m\x10\x07\x12\x06\n\x02\x61n\x10\x08\x12\x06\n\x02\x61r\x10\t\x12\x06\n\x02\x61s\x10\n\x12\x06\n\x02\x61v\x10\x0b\x12\x06\n\x02\x61y\x10\x0c\x12\x06\n\x02\x61z\x10\r\x12\x06\n\x02\x62\x61\x10\x0e\x12\x06\n\x02\x62\x65\x10\x0f\x12\x06\n\x02\x62g\x10\x10\x12\x06\n\x02\x62h\x10\x11\x12\x06\n\x02\x62i\x10\x12\x12\x06\n\x02\x62m\x10\x13\x12\x06\n\x02\x62n\x10\x14\x12\x06\n\x02\x62o\x10\x15\x12\x06\n\x02\x62r\x10\x16\x12\x06\n\x02\x62s\x10\x17\x12\x06\n\x02\x63\x61\x10\x18\x12\x06\n\x02\x63\x65\x10\x19\x12\x06\n\x02\x63h\x10\x1a\x12\x06\n\x02\x63o\x10\x1b\x12\x06\n\x02\x63r\x10\x1c\x12\x06\n\x02\x63s\x10\x1d\x12\x06\n\x02\x63u\x10\x1e\x12\x06\n\x02\x63v\x10\x1f\x12\x06\n\x02\x63y\x10 \x12\x06\n\x02\x64\x61\x10!\x12\x06\n\x02\x64\x65\x10\"\x12\x06\n\x02\x64v\x10#\x12\x06\n\x02\x64z\x10$\x12\x06\n\x02\x65\x65\x10%\x12\x06\n\x02\x65l\x10&\x12\x06\n\x02\x65o\x10\'\x12\x06\n\x02\x65s\x10(\x12\x06\n\x02\x65t\x10)\x12\x06\n\x02\x65u\x10*\x12\x06\n\x02\x66\x61\x10+\x12\x06\n\x02\x66\x66\x10,\x12\x06\n\x02\x66i\x10-\x12\x06\n\x02\x66j\x10.\x12\x06\n\x02\x66o\x10/\x12\x06\n\x02\x66r\x10\x30\x12\x06\n\x02\x66y\x10\x31\x12\x06\n\x02ga\x10\x32\x12\x06\n\x02gd\x10\x33\x12\x06\n\x02gl\x10\x34\x12\x06\n\x02gn\x10\x35\x12\x06\n\x02gu\x10\x36\x12\x06\n\x02gv\x10\x37\x12\x06\n\x02ha\x10\x38\x12\x06\n\x02he\x10\x39\x12\x06\n\x02hi\x10:\x12\x06\n\x02ho\x10;\x12\x06\n\x02hr\x10<\x12\x06\n\x02ht\x10=\x12\x06\n\x02hu\x10>\x12\x06\n\x02hy\x10?\x12\x06\n\x02hz\x10@\x12\x06\n\x02ia\x10\x41\x12\x06\n\x02id\x10\x42\x12\x06\n\x02ie\x10\x43\x12\x06\n\x02ig\x10\x44\x12\x06\n\x02ii\x10\x45\x12\x06\n\x02ik\x10\x46\x12\x06\n\x02io\x10G\x12\x06\n\x02is\x10H\x12\x06\n\x02it\x10I\x12\x06\n\x02iu\x10J\x12\x06\n\x02ja\x10K\x12\x06\n\x02jv\x10L\x12\x06\n\x02ka\x10M\x12\x06\n\x02kg\x10N\x12\x06\n\x02ki\x10O\x12\x06\n\x02kj\x10P\x12\x06\n\x02kk\x10Q\x12\x06\n\x02kl\x10R\x12\x06\n\x02km\x10S\x12\x06\n\x02kn\x10T\x12\x06\n\x02ko\x10U\x12\x06\n\x02kr\x10V\x12\x06\n\x02ks\x10W\x12\x06\n\x02ku\x10X\x12\x06\n\x02kv\x10Y\x12\x06\n\x02kw\x10Z\x12\x06\n\x02ky\x10[\x12\x06\n\x02la\x10\\\x12\x06\n\x02lb\x10]\x12\x06\n\x02lg\x10^\x12\x06\n\x02li\x10_\x12\x06\n\x02ln\x10`\x12\x06\n\x02lo\x10\x61\x12\x06\n\x02lt\x10\x62\x12\x06\n\x02lu\x10\x63\x12\x06\n\x02lv\x10\x64\x12\x06\n\x02mg\x10\x65\x12\x06\n\x02mh\x10\x66\x12\x06\n\x02mi\x10g\x12\x06\n\x02mk\x10h\x12\x06\n\x02ml\x10i\x12\x06\n\x02mn\x10j\x12\x06\n\x02mr\x10k\x12\x06\n\x02ms\x10l\x12\x06\n\x02mt\x10m\x12\x06\n\x02my\x10n\x12\x06\n\x02na\x10o\x12\x06\n\x02nb\x10p\x12\x06\n\x02nd\x10q\x12\x06\n\x02ne\x10r\x12\x06\n\x02ng\x10s\x12\x06\n\x02nl\x10t\x12\x06\n\x02nn\x10u\x12\x06\n\x02no\x10v\x12\x06\n\x02nr\x10w\x12\x06\n\x02nv\x10x\x12\x06\n\x02ny\x10y\x12\x06\n\x02oc\x10z\x12\x06\n\x02oj\x10{\x12\x06\n\x02om\x10|\x12\x06\n\x02or\x10}\x12\x06\n\x02os\x10~\x12\x06\n\x02pa\x10\x7f\x12\x07\n\x02pi\x10\x80\x01\x12\x07\n\x02pl\x10\x81\x01\x12\x07\n\x02ps\x10\x82\x01\x12\x07\n\x02pt\x10\x83\x01\x12\x07\n\x02qu\x10\x84\x01\x12\x07\n\x02rm\x10\x85\x01\x12\x07\n\x02rn\x10\x86\x01\x12\x07\n\x02ro\x10\x87\x01\x12\x07\n\x02ru\x10\x88\x01\x12\x07\n\x02rw\x10\x89\x01\x12\x07\n\x02sa\x10\x8a\x01\x12\x07\n\x02sc\x10\x8b\x01\x12\x07\n\x02sd\x10\x8c\x01\x12\x07\n\x02se\x10\x8d\x01\x12\x07\n\x02sg\x10\x8e\x01\x12\x07\n\x02si\x10\x8f\x01\x12\x07\n\x02sk\x10\x90\x01\x12\x07\n\x02sl\x10\x91\x01\x12\x07\n\x02sm\x10\x92\x01\x12\x07\n\x02sn\x10\x93\x01\x12\x07\n\x02so\x10\x94\x01\x12\x07\n\x02sq\x10\x95\x01\x12\x07\n\x02sr\x10\x96\x01\x12\x07\n\x02ss\x10\x97\x01\x12\x07\n\x02st\x10\x98\x01\x12\x07\n\x02su\x10\x99\x01\x12\x07\n\x02sv\x10\x9a\x01\x12\x07\n\x02sw\x10\x9b\x01\x12\x07\n\x02ta\x10\x9c\x01\x12\x07\n\x02te\x10\x9d\x01\x12\x07\n\x02tg\x10\x9e\x01\x12\x07\n\x02th\x10\x9f\x01\x12\x07\n\x02ti\x10\xa0\x01\x12\x07\n\x02tk\x10\xa1\x01\x12\x07\n\x02tl\x10\xa2\x01\x12\x07\n\x02tn\x10\xa3\x01\x12\x07\n\x02to\x10\xa4\x01\x12\x07\n\x02tr\x10\xa5\x01\x12\x07\n\x02ts\x10\xa6\x01\x12\x07\n\x02tt\x10\xa7\x01\x12\x07\n\x02tw\x10\xa8\x01\x12\x07\n\x02tyLine truncated
|
||||
,
|
||||
dependencies=[lbrynet_dot_schema_dot_proto_dot_fee__pb2.DESCRIPTOR,])
|
||||
dependencies=[fee__pb2.DESCRIPTOR,])
|
||||
|
||||
|
||||
|
||||
_METADATA_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Metadata.Version',
|
||||
full_name='legacy_pb.Metadata.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -55,14 +55,14 @@ _METADATA_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=316,
|
||||
serialized_end=394,
|
||||
serialized_start=315,
|
||||
serialized_end=393,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_METADATA_VERSION)
|
||||
|
||||
_METADATA_LANGUAGE = _descriptor.EnumDescriptor(
|
||||
name='Language',
|
||||
full_name='Metadata.Language',
|
||||
full_name='legacy_pb.Metadata.Language',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -809,91 +809,91 @@ _METADATA_LANGUAGE = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=397,
|
||||
serialized_end=1958,
|
||||
serialized_start=396,
|
||||
serialized_end=1957,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_METADATA_LANGUAGE)
|
||||
|
||||
|
||||
_METADATA = _descriptor.Descriptor(
|
||||
name='Metadata',
|
||||
full_name='Metadata',
|
||||
full_name='legacy_pb.Metadata',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Metadata.version', index=0,
|
||||
name='version', full_name='legacy_pb.Metadata.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='language', full_name='Metadata.language', index=1,
|
||||
name='language', full_name='legacy_pb.Metadata.language', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='title', full_name='Metadata.title', index=2,
|
||||
name='title', full_name='legacy_pb.Metadata.title', index=2,
|
||||
number=3, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='description', full_name='Metadata.description', index=3,
|
||||
name='description', full_name='legacy_pb.Metadata.description', index=3,
|
||||
number=4, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='author', full_name='Metadata.author', index=4,
|
||||
name='author', full_name='legacy_pb.Metadata.author', index=4,
|
||||
number=5, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='license', full_name='Metadata.license', index=5,
|
||||
name='license', full_name='legacy_pb.Metadata.license', index=5,
|
||||
number=6, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='nsfw', full_name='Metadata.nsfw', index=6,
|
||||
name='nsfw', full_name='legacy_pb.Metadata.nsfw', index=6,
|
||||
number=7, type=8, cpp_type=7, label=2,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='fee', full_name='Metadata.fee', index=7,
|
||||
name='fee', full_name='legacy_pb.Metadata.fee', index=7,
|
||||
number=8, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='thumbnail', full_name='Metadata.thumbnail', index=8,
|
||||
name='thumbnail', full_name='legacy_pb.Metadata.thumbnail', index=8,
|
||||
number=9, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='preview', full_name='Metadata.preview', index=9,
|
||||
name='preview', full_name='legacy_pb.Metadata.preview', index=9,
|
||||
number=10, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='licenseUrl', full_name='Metadata.licenseUrl', index=10,
|
||||
name='licenseUrl', full_name='legacy_pb.Metadata.licenseUrl', index=10,
|
||||
number=11, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -913,13 +913,13 @@ _METADATA = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=72,
|
||||
serialized_end=1958,
|
||||
serialized_start=41,
|
||||
serialized_end=1957,
|
||||
)
|
||||
|
||||
_METADATA.fields_by_name['version'].enum_type = _METADATA_VERSION
|
||||
_METADATA.fields_by_name['language'].enum_type = _METADATA_LANGUAGE
|
||||
_METADATA.fields_by_name['fee'].message_type = lbrynet_dot_schema_dot_proto_dot_fee__pb2._FEE
|
||||
_METADATA.fields_by_name['fee'].message_type = fee__pb2._FEE
|
||||
_METADATA_VERSION.containing_type = _METADATA
|
||||
_METADATA_LANGUAGE.containing_type = _METADATA
|
||||
DESCRIPTOR.message_types_by_name['Metadata'] = _METADATA
|
||||
@@ -927,8 +927,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Metadata = _reflection.GeneratedProtocolMessageType('Metadata', (_message.Message,), dict(
|
||||
DESCRIPTOR = _METADATA,
|
||||
__module__ = 'lbrynet.schema.proto.metadata_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Metadata)
|
||||
__module__ = 'metadata_pb2'
|
||||
# @@protoc_insertion_point(class_scope:legacy_pb.Metadata)
|
||||
))
|
||||
_sym_db.RegisterMessage(Metadata)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/signature.proto
|
||||
# source: signature.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -12,23 +12,23 @@ from google.protobuf import symbol_database as _symbol_database
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
from lbrynet.schema.proto import certificate_pb2 as lbrynet_dot_schema_dot_proto_dot_certificate__pb2
|
||||
from . import certificate_pb2 as certificate__pb2
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/signature.proto',
|
||||
package='',
|
||||
name='signature.proto',
|
||||
package='legacy_pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n$lbrynet/schema/proto/signature.proto\x1a&lbrynet/schema/proto/certificate.proto\"\xa7\x01\n\tSignature\x12#\n\x07version\x18\x01 \x02(\x0e\x32\x12.Signature.Version\x12\x1f\n\rsignatureType\x18\x02 \x02(\x0e\x32\x08.KeyType\x12\x11\n\tsignature\x18\x03 \x02(\x0c\x12\x15\n\rcertificateId\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
|
||||
serialized_pb=_b('\n\x0fsignature.proto\x12\tlegacy_pb\x1a\x11\x63\x65rtificate.proto\"\xbb\x01\n\tSignature\x12-\n\x07version\x18\x01 \x02(\x0e\x32\x1c.legacy_pb.Signature.Version\x12)\n\rsignatureType\x18\x02 \x02(\x0e\x32\x12.legacy_pb.KeyType\x12\x11\n\tsignature\x18\x03 \x02(\x0c\x12\x15\n\rcertificateId\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
|
||||
,
|
||||
dependencies=[lbrynet_dot_schema_dot_proto_dot_certificate__pb2.DESCRIPTOR,])
|
||||
dependencies=[certificate__pb2.DESCRIPTOR,])
|
||||
|
||||
|
||||
|
||||
_SIGNATURE_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Signature.Version',
|
||||
full_name='legacy_pb.Signature.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -43,42 +43,42 @@ _SIGNATURE_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=206,
|
||||
serialized_end=248,
|
||||
serialized_start=195,
|
||||
serialized_end=237,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_SIGNATURE_VERSION)
|
||||
|
||||
|
||||
_SIGNATURE = _descriptor.Descriptor(
|
||||
name='Signature',
|
||||
full_name='Signature',
|
||||
full_name='legacy_pb.Signature',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Signature.version', index=0,
|
||||
name='version', full_name='legacy_pb.Signature.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='signatureType', full_name='Signature.signatureType', index=1,
|
||||
name='signatureType', full_name='legacy_pb.Signature.signatureType', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='signature', full_name='Signature.signature', index=2,
|
||||
name='signature', full_name='legacy_pb.Signature.signature', index=2,
|
||||
number=3, type=12, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='certificateId', full_name='Signature.certificateId', index=3,
|
||||
name='certificateId', full_name='legacy_pb.Signature.certificateId', index=3,
|
||||
number=4, type=12, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -97,20 +97,20 @@ _SIGNATURE = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=81,
|
||||
serialized_end=248,
|
||||
serialized_start=50,
|
||||
serialized_end=237,
|
||||
)
|
||||
|
||||
_SIGNATURE.fields_by_name['version'].enum_type = _SIGNATURE_VERSION
|
||||
_SIGNATURE.fields_by_name['signatureType'].enum_type = lbrynet_dot_schema_dot_proto_dot_certificate__pb2._KEYTYPE
|
||||
_SIGNATURE.fields_by_name['signatureType'].enum_type = certificate__pb2._KEYTYPE
|
||||
_SIGNATURE_VERSION.containing_type = _SIGNATURE
|
||||
DESCRIPTOR.message_types_by_name['Signature'] = _SIGNATURE
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Signature = _reflection.GeneratedProtocolMessageType('Signature', (_message.Message,), dict(
|
||||
DESCRIPTOR = _SIGNATURE,
|
||||
__module__ = 'lbrynet.schema.proto.signature_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Signature)
|
||||
__module__ = 'signature_pb2'
|
||||
# @@protoc_insertion_point(class_scope:legacy_pb.Signature)
|
||||
))
|
||||
_sym_db.RegisterMessage(Signature)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/source.proto
|
||||
# source: source.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -15,18 +15,18 @@ _sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/source.proto',
|
||||
package='',
|
||||
name='source.proto',
|
||||
package='legacy_pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n!lbrynet/schema/proto/source.proto\"\xde\x01\n\x06Source\x12 \n\x07version\x18\x01 \x02(\x0e\x32\x0f.Source.Version\x12\'\n\nsourceType\x18\x02 \x02(\x0e\x32\x13.Source.SourceTypes\x12\x0e\n\x06source\x18\x03 \x02(\x0c\x12\x13\n\x0b\x63ontentType\x18\x04 \x02(\t\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"8\n\x0bSourceTypes\x12\x17\n\x13UNKNOWN_SOURCE_TYPE\x10\x00\x12\x10\n\x0clbry_sd_hash\x10\x01')
|
||||
serialized_pb=_b('\n\x0csource.proto\x12\tlegacy_pb\"\xf2\x01\n\x06Source\x12*\n\x07version\x18\x01 \x02(\x0e\x32\x19.legacy_pb.Source.Version\x12\x31\n\nsourceType\x18\x02 \x02(\x0e\x32\x1d.legacy_pb.Source.SourceTypes\x12\x0e\n\x06source\x18\x03 \x02(\x0c\x12\x13\n\x0b\x63ontentType\x18\x04 \x02(\t\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"8\n\x0bSourceTypes\x12\x17\n\x13UNKNOWN_SOURCE_TYPE\x10\x00\x12\x10\n\x0clbry_sd_hash\x10\x01')
|
||||
)
|
||||
|
||||
|
||||
|
||||
_SOURCE_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Source.Version',
|
||||
full_name='legacy_pb.Source.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -41,14 +41,14 @@ _SOURCE_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=160,
|
||||
serialized_end=202,
|
||||
serialized_start=170,
|
||||
serialized_end=212,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_SOURCE_VERSION)
|
||||
|
||||
_SOURCE_SOURCETYPES = _descriptor.EnumDescriptor(
|
||||
name='SourceTypes',
|
||||
full_name='Source.SourceTypes',
|
||||
full_name='legacy_pb.Source.SourceTypes',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -63,42 +63,42 @@ _SOURCE_SOURCETYPES = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=204,
|
||||
serialized_end=260,
|
||||
serialized_start=214,
|
||||
serialized_end=270,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_SOURCE_SOURCETYPES)
|
||||
|
||||
|
||||
_SOURCE = _descriptor.Descriptor(
|
||||
name='Source',
|
||||
full_name='Source',
|
||||
full_name='legacy_pb.Source',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Source.version', index=0,
|
||||
name='version', full_name='legacy_pb.Source.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='sourceType', full_name='Source.sourceType', index=1,
|
||||
name='sourceType', full_name='legacy_pb.Source.sourceType', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='source', full_name='Source.source', index=2,
|
||||
name='source', full_name='legacy_pb.Source.source', index=2,
|
||||
number=3, type=12, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='contentType', full_name='Source.contentType', index=3,
|
||||
name='contentType', full_name='legacy_pb.Source.contentType', index=3,
|
||||
number=4, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -118,8 +118,8 @@ _SOURCE = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=38,
|
||||
serialized_end=260,
|
||||
serialized_start=28,
|
||||
serialized_end=270,
|
||||
)
|
||||
|
||||
_SOURCE.fields_by_name['version'].enum_type = _SOURCE_VERSION
|
||||
@@ -131,8 +131,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Source = _reflection.GeneratedProtocolMessageType('Source', (_message.Message,), dict(
|
||||
DESCRIPTOR = _SOURCE,
|
||||
__module__ = 'lbrynet.schema.proto.source_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Source)
|
||||
__module__ = 'source_pb2'
|
||||
# @@protoc_insertion_point(class_scope:legacy_pb.Source)
|
||||
))
|
||||
_sym_db.RegisterMessage(Source)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/stream.proto
|
||||
# source: stream.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -12,24 +12,24 @@ from google.protobuf import symbol_database as _symbol_database
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
from lbrynet.schema.proto import metadata_pb2 as lbrynet_dot_schema_dot_proto_dot_metadata__pb2
|
||||
from lbrynet.schema.proto import source_pb2 as lbrynet_dot_schema_dot_proto_dot_source__pb2
|
||||
from . import metadata_pb2 as metadata__pb2
|
||||
from . import source_pb2 as source__pb2
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/stream.proto',
|
||||
package='',
|
||||
name='stream.proto',
|
||||
package='legacy_pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n!lbrynet/schema/proto/stream.proto\x1a#lbrynet/schema/proto/metadata.proto\x1a!lbrynet/schema/proto/source.proto\"\x8c\x01\n\x06Stream\x12 \n\x07version\x18\x01 \x02(\x0e\x32\x0f.Stream.Version\x12\x1b\n\x08metadata\x18\x02 \x02(\x0b\x32\t.Metadata\x12\x17\n\x06source\x18\x03 \x02(\x0b\x32\x07.Source\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
|
||||
serialized_pb=_b('\n\x0cstream.proto\x12\tlegacy_pb\x1a\x0emetadata.proto\x1a\x0csource.proto\"\xaa\x01\n\x06Stream\x12*\n\x07version\x18\x01 \x02(\x0e\x32\x19.legacy_pb.Stream.Version\x12%\n\x08metadata\x18\x02 \x02(\x0b\x32\x13.legacy_pb.Metadata\x12!\n\x06source\x18\x03 \x02(\x0b\x32\x11.legacy_pb.Source\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
|
||||
,
|
||||
dependencies=[lbrynet_dot_schema_dot_proto_dot_metadata__pb2.DESCRIPTOR,lbrynet_dot_schema_dot_proto_dot_source__pb2.DESCRIPTOR,])
|
||||
dependencies=[metadata__pb2.DESCRIPTOR,source__pb2.DESCRIPTOR,])
|
||||
|
||||
|
||||
|
||||
_STREAM_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Stream.Version',
|
||||
full_name='legacy_pb.Stream.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -44,35 +44,35 @@ _STREAM_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=208,
|
||||
serialized_end=250,
|
||||
serialized_start=186,
|
||||
serialized_end=228,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_STREAM_VERSION)
|
||||
|
||||
|
||||
_STREAM = _descriptor.Descriptor(
|
||||
name='Stream',
|
||||
full_name='Stream',
|
||||
full_name='legacy_pb.Stream',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Stream.version', index=0,
|
||||
name='version', full_name='legacy_pb.Stream.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='metadata', full_name='Stream.metadata', index=1,
|
||||
name='metadata', full_name='legacy_pb.Stream.metadata', index=1,
|
||||
number=2, type=11, cpp_type=10, label=2,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='source', full_name='Stream.source', index=2,
|
||||
name='source', full_name='legacy_pb.Stream.source', index=2,
|
||||
number=3, type=11, cpp_type=10, label=2,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -91,21 +91,21 @@ _STREAM = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=110,
|
||||
serialized_end=250,
|
||||
serialized_start=58,
|
||||
serialized_end=228,
|
||||
)
|
||||
|
||||
_STREAM.fields_by_name['version'].enum_type = _STREAM_VERSION
|
||||
_STREAM.fields_by_name['metadata'].message_type = lbrynet_dot_schema_dot_proto_dot_metadata__pb2._METADATA
|
||||
_STREAM.fields_by_name['source'].message_type = lbrynet_dot_schema_dot_proto_dot_source__pb2._SOURCE
|
||||
_STREAM.fields_by_name['metadata'].message_type = metadata__pb2._METADATA
|
||||
_STREAM.fields_by_name['source'].message_type = source__pb2._SOURCE
|
||||
_STREAM_VERSION.containing_type = _STREAM
|
||||
DESCRIPTOR.message_types_by_name['Stream'] = _STREAM
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Stream = _reflection.GeneratedProtocolMessageType('Stream', (_message.Message,), dict(
|
||||
DESCRIPTOR = _STREAM,
|
||||
__module__ = 'lbrynet.schema.proto.stream_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Stream)
|
||||
__module__ = 'stream_pb2'
|
||||
# @@protoc_insertion_point(class_scope:legacy_pb.Stream)
|
||||
))
|
||||
_sym_db.RegisterMessage(Stream)
|
||||
|
||||
File renamed without changes.
File diff suppressed because it is too large.
Load diff
@@ -1,5 +1,4 @@
|
||||
import re
|
||||
from lbrynet.schema.error import URIParseError
|
||||
|
||||
PROTOCOL = 'lbry://'
|
||||
CHANNEL_CHAR = '@'
|
||||
@@ -13,6 +12,10 @@ CLAIM_ID_MAX_LENGTH = 40
|
||||
CHANNEL_NAME_MIN_LENGTH = 1
|
||||
|
||||
|
||||
class URIParseError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class URI(object):
|
||||
__slots__ = ['name', 'claim_sequence', 'bid_position', 'claim_id', 'path']
|
||||
|
||||
@@ -139,7 +142,7 @@ def get_schema_regex():
|
||||
protocol = _named("protocol", re.escape(PROTOCOL))
|
||||
|
||||
# Define basic building blocks
|
||||
valid_name_char = "[a-zA-Z0-9\-]" # these characters are the only valid name characters
|
||||
valid_name_char = "[^=&#:$@%?/]" # from the grammar section of https://spec.lbry.io/
|
||||
name_content = valid_name_char + '+'
|
||||
name_min_channel_length = valid_name_char + '{' + str(CHANNEL_NAME_MIN_LENGTH) + ',}'
|
||||
|
||||
|
||||
@@ -1,142 +0,0 @@
|
||||
from string import hexdigits
|
||||
import ecdsa
|
||||
import hashlib
|
||||
import binascii
|
||||
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
from cryptography.hazmat.primitives.serialization import load_der_public_key
|
||||
from cryptography.hazmat.primitives import hashes
|
||||
from cryptography.hazmat.primitives.asymmetric import ec
|
||||
from cryptography.hazmat.primitives.asymmetric.utils import Prehashed
|
||||
from cryptography.exceptions import InvalidSignature
|
||||
from ecdsa.util import sigencode_der
|
||||
|
||||
from lbrynet.schema.address import decode_address
|
||||
from lbrynet.schema.schema import NIST256p, NIST384p, SECP256k1, ECDSA_CURVES, CURVE_NAMES
|
||||
|
||||
|
||||
def validate_claim_id(claim_id):
|
||||
if not len(claim_id) == 40:
|
||||
raise Exception("Incorrect claimid length: %i" % len(claim_id))
|
||||
if isinstance(claim_id, bytes):
|
||||
claim_id = claim_id.decode('utf-8')
|
||||
if set(claim_id).difference(hexdigits):
|
||||
raise Exception("Claim id is not hex encoded")
|
||||
|
||||
|
||||
class Validator:
|
||||
CURVE_NAME = None
|
||||
HASHFUNC = hashlib.sha256
|
||||
|
||||
def __init__(self, public_key, certificate_claim_id):
|
||||
validate_claim_id(certificate_claim_id)
|
||||
if CURVE_NAMES.get(get_key_type_from_dem(public_key)) != self.CURVE_NAME:
|
||||
raise Exception("Curve mismatch")
|
||||
self._public_key = public_key
|
||||
self._certificate_claim_id = certificate_claim_id
|
||||
|
||||
@property
|
||||
def public_key(self):
|
||||
return self._public_key
|
||||
|
||||
@property
|
||||
def certificate_claim_id(self):
|
||||
return self._certificate_claim_id
|
||||
|
||||
@classmethod
|
||||
def signing_key_from_pem(cls, pem):
|
||||
return ecdsa.SigningKey.from_pem(pem, hashfunc=cls.HASHFUNC)
|
||||
|
||||
@classmethod
|
||||
def signing_key_from_der(cls, der):
|
||||
return ecdsa.SigningKey.from_der(der, hashfunc=cls.HASHFUNC)
|
||||
|
||||
@classmethod
|
||||
def load_from_certificate(cls, certificate_claim, certificate_claim_id):
|
||||
certificate = certificate_claim.certificate
|
||||
return cls(certificate.publicKey, certificate_claim_id)
|
||||
|
||||
def validate_signature(self, digest, signature):
|
||||
public_key = load_der_public_key(self.public_key, default_backend())
|
||||
if len(signature) == 64:
|
||||
hash = hashes.SHA256()
|
||||
elif len(signature) == 96:
|
||||
hash = hashes.SHA384()
|
||||
signature = binascii.hexlify(signature)
|
||||
r = int(signature[:int(len(signature)/2)], 16)
|
||||
s = int(signature[int(len(signature)/2):], 16)
|
||||
encoded_sig = sigencode_der(r, s, len(signature)*4)
|
||||
try:
|
||||
public_key.verify(encoded_sig, digest, ec.ECDSA(Prehashed(hash)))
|
||||
return True
|
||||
except InvalidSignature:
|
||||
# TODO Fixme. This is what is expected today on the outer calls. This should be implementation independent
|
||||
# but requires changing everything calling that
|
||||
from ecdsa import BadSignatureError
|
||||
raise BadSignatureError
|
||||
|
||||
def validate_claim_signature(self, claim, claim_address, name):
|
||||
to_sign = bytearray()
|
||||
if claim.detached_signature and claim.detached_signature.raw_signature:
|
||||
assert name is not None, "Name is required for verifying detached signatures."
|
||||
to_sign.extend(name.lower().encode())
|
||||
signature = claim.detached_signature.raw_signature
|
||||
payload = claim.detached_signature.payload
|
||||
else:
|
||||
# extract and serialize the stream from the claim, then check the signature
|
||||
signature = binascii.unhexlify(claim.signature)
|
||||
payload = claim.serialized_no_signature
|
||||
decoded_address = decode_address(claim_address)
|
||||
|
||||
|
||||
if signature is None:
|
||||
raise Exception("No signature to validate")
|
||||
|
||||
to_sign.extend(decoded_address)
|
||||
to_sign.extend(payload)
|
||||
to_sign.extend(binascii.unhexlify(self.certificate_claim_id))
|
||||
|
||||
return self.validate_signature(self.HASHFUNC(to_sign).digest(), signature)
|
||||
|
||||
def validate_private_key(self, private_key):
|
||||
if not isinstance(private_key, ecdsa.SigningKey):
|
||||
raise TypeError("Not given a signing key, given a %s" % str(type(private_key)))
|
||||
return private_key.get_verifying_key().to_der() == self.public_key
|
||||
|
||||
|
||||
class NIST256pValidator(Validator):
|
||||
CURVE_NAME = NIST256p
|
||||
HASHFUNC = hashlib.sha256
|
||||
|
||||
|
||||
class NIST384pValidator(Validator):
|
||||
CURVE_NAME = NIST384p
|
||||
HASHFUNC = hashlib.sha384
|
||||
|
||||
|
||||
class SECP256k1Validator(Validator):
|
||||
CURVE_NAME = SECP256k1
|
||||
HASHFUNC = hashlib.sha256
|
||||
|
||||
|
||||
def get_validator(curve):
|
||||
if curve == NIST256p:
|
||||
return NIST256pValidator
|
||||
elif curve == NIST384p:
|
||||
return NIST384pValidator
|
||||
elif curve == SECP256k1:
|
||||
return SECP256k1Validator
|
||||
else:
|
||||
raise Exception("Unknown curve: %s" % str(curve))
|
||||
|
||||
|
||||
def get_key_type_from_dem(pubkey_dem):
|
||||
name = serialization.load_der_public_key(pubkey_dem, default_backend()).curve.name
|
||||
if name == 'secp256k1':
|
||||
return ECDSA_CURVES[SECP256k1]
|
||||
elif name == 'secp256r1':
|
||||
return ECDSA_CURVES[NIST256p]
|
||||
elif name == 'secp384r1':
|
||||
return ECDSA_CURVES[NIST384p]
|
||||
raise Exception("unexpected curve: %s" % name)
|
||||
@@ -1,136 +0,0 @@
|
||||
import os
|
||||
import binascii
|
||||
import logging
|
||||
import typing
|
||||
import asyncio
|
||||
from lbrynet.blob import MAX_BLOB_SIZE
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_info import BlobInfo
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_next_available_file_name(download_directory: str, file_name: str) -> str:
|
||||
base_name, ext = os.path.splitext(os.path.basename(file_name))
|
||||
i = 0
|
||||
while os.path.isfile(os.path.join(download_directory, file_name)):
|
||||
i += 1
|
||||
file_name = "%s_%i%s" % (base_name, i, ext)
|
||||
|
||||
return file_name
|
||||
|
||||
|
||||
async def get_next_available_file_name(loop: asyncio.BaseEventLoop, download_directory: str, file_name: str) -> str:
|
||||
return await loop.run_in_executor(None, _get_next_available_file_name, download_directory, file_name)
|
||||
|
||||
|
||||
class StreamAssembler:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', sd_hash: str,
|
||||
output_file_name: typing.Optional[str] = None):
|
||||
self.output_file_name = output_file_name
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.sd_hash = sd_hash
|
||||
self.sd_blob: 'BlobFile' = None
|
||||
self.descriptor: StreamDescriptor = None
|
||||
self.got_descriptor = asyncio.Event(loop=self.loop)
|
||||
self.wrote_bytes_event = asyncio.Event(loop=self.loop)
|
||||
self.stream_finished_event = asyncio.Event(loop=self.loop)
|
||||
self.output_path = ''
|
||||
self.stream_handle = None
|
||||
self.written_bytes: int = 0
|
||||
|
||||
async def _decrypt_blob(self, blob: 'BlobFile', blob_info: 'BlobInfo', key: str):
|
||||
if not blob or not self.stream_handle or self.stream_handle.closed:
|
||||
return False
|
||||
|
||||
def _decrypt_and_write():
|
||||
offset = blob_info.blob_num * (MAX_BLOB_SIZE - 1)
|
||||
self.stream_handle.seek(offset)
|
||||
_decrypted = blob.decrypt(
|
||||
binascii.unhexlify(key), binascii.unhexlify(blob_info.iv.encode())
|
||||
)
|
||||
self.stream_handle.write(_decrypted)
|
||||
self.stream_handle.flush()
|
||||
self.written_bytes += len(_decrypted)
|
||||
log.debug("decrypted %s", blob.blob_hash[:8])
|
||||
|
||||
await self.loop.run_in_executor(None, _decrypt_and_write)
|
||||
return True
|
||||
|
||||
async def setup(self):
|
||||
pass
|
||||
|
||||
async def after_got_descriptor(self):
|
||||
pass
|
||||
|
||||
async def after_finished(self):
|
||||
pass
|
||||
|
||||
async def assemble_decrypted_stream(self, output_dir: str, output_file_name: typing.Optional[str] = None):
|
||||
if not os.path.isdir(output_dir):
|
||||
raise OSError(f"output directory does not exist: '{output_dir}' '{output_file_name}'")
|
||||
await self.setup()
|
||||
self.sd_blob = await self.get_blob(self.sd_hash)
|
||||
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(self.loop, self.blob_manager.blob_dir,
|
||||
self.sd_blob)
|
||||
await self.after_got_descriptor()
|
||||
self.output_file_name = output_file_name or self.descriptor.suggested_file_name
|
||||
self.output_file_name = await get_next_available_file_name(self.loop, output_dir, self.output_file_name)
|
||||
self.output_path = os.path.join(output_dir, self.output_file_name)
|
||||
if not self.got_descriptor.is_set():
|
||||
self.got_descriptor.set()
|
||||
await self.blob_manager.storage.store_stream(
|
||||
self.sd_blob, self.descriptor
|
||||
)
|
||||
await self.blob_manager.blob_completed(self.sd_blob)
|
||||
written_blobs = None
|
||||
save_tasks = []
|
||||
try:
|
||||
with open(self.output_path, 'wb') as stream_handle:
|
||||
self.stream_handle = stream_handle
|
||||
for i, blob_info in enumerate(self.descriptor.blobs[:-1]):
|
||||
if blob_info.blob_num != i:
|
||||
log.error("sd blob %s is invalid, cannot assemble stream", self.descriptor.sd_hash)
|
||||
return
|
||||
while self.stream_handle and not self.stream_handle.closed:
|
||||
try:
|
||||
blob = await self.get_blob(blob_info.blob_hash, blob_info.length)
|
||||
if blob and blob.length != blob_info.length:
|
||||
log.warning("Found incomplete, deleting: %s", blob_info.blob_hash)
|
||||
await self.blob_manager.delete_blobs([blob_info.blob_hash])
|
||||
continue
|
||||
if await self._decrypt_blob(blob, blob_info, self.descriptor.key):
|
||||
save_tasks.append(asyncio.ensure_future(self.blob_manager.blob_completed(blob)))
|
||||
written_blobs = i
|
||||
if not self.wrote_bytes_event.is_set():
|
||||
self.wrote_bytes_event.set()
|
||||
log.debug("written %i/%i", written_blobs, len(self.descriptor.blobs) - 2)
|
||||
break
|
||||
except FileNotFoundError:
|
||||
log.debug("stream assembler stopped")
|
||||
return
|
||||
except (ValueError, IOError, OSError):
|
||||
log.warning("failed to decrypt blob %s for stream %s", blob_info.blob_hash,
|
||||
self.descriptor.sd_hash)
|
||||
continue
|
||||
finally:
|
||||
if written_blobs == len(self.descriptor.blobs) - 2:
|
||||
log.debug("finished decrypting and assembling stream")
|
||||
if save_tasks:
|
||||
await asyncio.wait(save_tasks)
|
||||
await self.after_finished()
|
||||
self.stream_finished_event.set()
|
||||
else:
|
||||
log.debug("stream decryption and assembly did not finish (%i/%i blobs are done)", written_blobs or 0,
|
||||
len(self.descriptor.blobs) - 2)
|
||||
if self.output_path and os.path.isfile(self.output_path):
|
||||
log.debug("erasing incomplete file assembly: %s", self.output_path)
|
||||
os.unlink(self.output_path)
|
||||
|
||||
async def get_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
|
||||
return self.blob_manager.get_blob(blob_hash, length)
|
||||
@@ -8,7 +8,7 @@ from collections import OrderedDict
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from lbrynet.blob import MAX_BLOB_SIZE
|
||||
from lbrynet.blob.blob_info import BlobInfo
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from lbrynet.blob.blob_file import AbstractBlob, BlobFile
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.error import InvalidStreamDescriptorError
|
||||
|
||||
@@ -47,6 +47,17 @@ def file_reader(file_path: str):
|
||||
|
||||
|
||||
class StreamDescriptor:
|
||||
__slots__ = [
|
||||
'loop',
|
||||
'blob_dir',
|
||||
'stream_name',
|
||||
'key',
|
||||
'suggested_file_name',
|
||||
'blobs',
|
||||
'stream_hash',
|
||||
'sd_hash'
|
||||
]
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, stream_name: str, key: str,
|
||||
suggested_file_name: str, blobs: typing.List[BlobInfo], stream_hash: typing.Optional[str] = None,
|
||||
sd_hash: typing.Optional[str] = None):
|
||||
@@ -108,30 +119,35 @@ class StreamDescriptor:
|
||||
h.update(self.old_sort_json())
|
||||
return h.hexdigest()
|
||||
|
||||
async def make_sd_blob(self, blob_file_obj: typing.Optional[BlobFile] = None,
|
||||
old_sort: typing.Optional[bool] = False):
|
||||
async def make_sd_blob(self, blob_file_obj: typing.Optional[AbstractBlob] = None,
|
||||
old_sort: typing.Optional[bool] = False,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None):
|
||||
sd_hash = self.calculate_sd_hash() if not old_sort else self.calculate_old_sort_sd_hash()
|
||||
if not old_sort:
|
||||
sd_data = self.as_json()
|
||||
else:
|
||||
sd_data = self.old_sort_json()
|
||||
sd_blob = blob_file_obj or BlobFile(self.loop, self.blob_dir, sd_hash, len(sd_data))
|
||||
sd_blob = blob_file_obj or BlobFile(self.loop, sd_hash, len(sd_data), blob_completed_callback, self.blob_dir)
|
||||
if blob_file_obj:
|
||||
blob_file_obj.set_length(len(sd_data))
|
||||
if not sd_blob.get_is_verified():
|
||||
writer = sd_blob.open_for_writing()
|
||||
writer = sd_blob.get_blob_writer()
|
||||
writer.write(sd_data)
|
||||
|
||||
await sd_blob.verified.wait()
|
||||
sd_blob.close()
|
||||
return sd_blob
|
||||
|
||||
@classmethod
|
||||
def _from_stream_descriptor_blob(cls, loop: asyncio.BaseEventLoop, blob_dir: str,
|
||||
blob: BlobFile) -> 'StreamDescriptor':
|
||||
assert os.path.isfile(blob.file_path)
|
||||
with open(blob.file_path, 'rb') as f:
|
||||
json_bytes = f.read()
|
||||
decoded = json.loads(json_bytes.decode())
|
||||
blob: AbstractBlob) -> 'StreamDescriptor':
|
||||
with blob.reader_context() as blob_reader:
|
||||
json_bytes = blob_reader.read()
|
||||
try:
|
||||
decoded = json.loads(json_bytes.decode())
|
||||
except json.JSONDecodeError:
|
||||
blob.delete()
|
||||
raise InvalidStreamDescriptorError("Does not decode as valid JSON")
|
||||
if decoded['blobs'][-1]['length'] != 0:
|
||||
raise InvalidStreamDescriptorError("Does not end with a zero-length blob.")
|
||||
if any([blob_info['length'] == 0 for blob_info in decoded['blobs'][:-1]]):
|
||||
@@ -156,8 +172,10 @@ class StreamDescriptor:
|
||||
|
||||
@classmethod
|
||||
async def from_stream_descriptor_blob(cls, loop: asyncio.BaseEventLoop, blob_dir: str,
|
||||
blob: BlobFile) -> 'StreamDescriptor':
|
||||
return await loop.run_in_executor(None, lambda: cls._from_stream_descriptor_blob(loop, blob_dir, blob))
|
||||
blob: AbstractBlob) -> 'StreamDescriptor':
|
||||
if not blob.is_readable():
|
||||
raise InvalidStreamDescriptorError(f"unreadable/missing blob: {blob.blob_hash}")
|
||||
return await loop.run_in_executor(None, cls._from_stream_descriptor_blob, loop, blob_dir, blob)
|
||||
|
||||
@staticmethod
|
||||
def get_blob_hashsum(b: typing.Dict):
|
||||
@@ -190,11 +208,12 @@ class StreamDescriptor:
|
||||
return h.hexdigest()
|
||||
|
||||
@classmethod
|
||||
async def create_stream(cls, loop: asyncio.BaseEventLoop, blob_dir: str,
|
||||
file_path: str, key: typing.Optional[bytes] = None,
|
||||
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None,
|
||||
old_sort: bool = False) -> 'StreamDescriptor':
|
||||
|
||||
async def create_stream(
|
||||
cls, loop: asyncio.BaseEventLoop, blob_dir: str, file_path: str, key: typing.Optional[bytes] = None,
|
||||
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None,
|
||||
old_sort: bool = False,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'],
|
||||
asyncio.Task]] = None) -> 'StreamDescriptor':
|
||||
blobs: typing.List[BlobInfo] = []
|
||||
|
||||
iv_generator = iv_generator or random_iv_generator()
|
||||
@@ -203,7 +222,7 @@ class StreamDescriptor:
|
||||
for blob_bytes in file_reader(file_path):
|
||||
blob_num += 1
|
||||
blob_info = await BlobFile.create_from_unencrypted(
|
||||
loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num
|
||||
loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num, blob_completed_callback
|
||||
)
|
||||
blobs.append(blob_info)
|
||||
blobs.append(
|
||||
@@ -212,7 +231,7 @@ class StreamDescriptor:
|
||||
loop, blob_dir, os.path.basename(file_path), binascii.hexlify(key).decode(), os.path.basename(file_path),
|
||||
blobs
|
||||
)
|
||||
sd_blob = await descriptor.make_sd_blob(old_sort=old_sort)
|
||||
sd_blob = await descriptor.make_sd_blob(old_sort=old_sort, blob_completed_callback=blob_completed_callback)
|
||||
descriptor.sd_hash = sd_blob.blob_hash
|
||||
return descriptor
|
||||
|
||||
@@ -224,7 +243,7 @@ class StreamDescriptor:
|
||||
return self.lower_bound_decrypted_length() + (AES.block_size // 8)
|
||||
|
||||
@classmethod
|
||||
async def recover(cls, blob_dir: str, sd_blob: 'BlobFile', stream_hash: str, stream_name: str,
|
||||
async def recover(cls, blob_dir: str, sd_blob: 'AbstractBlob', stream_hash: str, stream_name: str,
|
||||
suggested_file_name: str, key: str,
|
||||
blobs: typing.List['BlobInfo']) -> typing.Optional['StreamDescriptor']:
|
||||
descriptor = cls(asyncio.get_event_loop(), blob_dir, stream_name, key, suggested_file_name,
|
||||
|
||||
Loaded 100 of 185 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user