Compare commits
366
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
84908ec8ec | ||
|
|
dade49743b | ||
|
|
f29bf35c2a | ||
|
|
dfa6701c43 | ||
|
|
763ca69a73 | ||
|
|
6bf3b152bf | ||
|
|
aa19f85996 | ||
|
|
156d89567e | ||
|
|
ecc71baf61 | ||
|
|
90c743d963 | ||
|
|
b926293fa7 | ||
|
|
71a19191f8 | ||
|
|
38a0f20a33 | ||
|
|
c35192108c | ||
|
|
245b564f13 | ||
|
|
0d8d1ea4f3 | ||
|
|
27a427a363 | ||
|
|
2ff028a694 | ||
|
|
c211338218 | ||
|
|
8ac89af8bd | ||
|
|
bbbaf59591 | ||
|
|
169419896f | ||
|
|
0543dca502 | ||
|
|
cc6011d57a | ||
|
|
fc4407ef7e | ||
|
|
03735a125f | ||
|
|
5baeda9ff1 | ||
|
|
9b9794b5e0 | ||
|
|
0697d60a48 | ||
|
|
cfe6c82a31 | ||
|
|
3e30228d95 | ||
|
|
7264b53e5f | ||
|
|
60836d8523 | ||
|
|
ef89c2e47a | ||
|
|
2d9e3e1847 | ||
|
|
30136a9697 | ||
|
|
db7ccd66d3 | ||
|
|
cfe6483102 | ||
|
|
561566e723 | ||
|
|
c2dcc4c898 | ||
|
|
d09bfdc4ff | ||
|
|
358ef4536f | ||
|
|
5061a35e66 | ||
|
|
cd9a1e8c9e | ||
|
|
646902e75e | ||
|
|
40d26cb868 | ||
|
|
b64aa51c0c | ||
|
|
8206441834 | ||
|
|
d713783736 | ||
|
|
57dffaa2ce | ||
|
|
9e81dd2360 | ||
|
|
e2798969d7 | ||
|
|
1c31ec66f2 | ||
|
|
241f9fc7b0 | ||
|
|
270192486a | ||
|
|
a799503c97 | ||
|
|
9685928087 | ||
|
|
0e4b2fad99 | ||
|
|
3c4571a4e0 | ||
|
|
046147eb1d | ||
|
|
7834520e54 | ||
|
|
8e5b4d4b6f | ||
|
|
4544a074d9 | ||
|
|
9b78501392 | ||
|
|
f59ddcc88d | ||
|
|
a4955a2b79 | ||
|
|
92ae1a565b | ||
|
|
15a56ca25e | ||
|
|
9dcaa829ea | ||
|
|
9f65799a3d | ||
|
|
886587848b | ||
|
|
a97fc6dba8 | ||
|
|
c124e88d12 | ||
|
|
17f3870296 | ||
|
|
4626d42d08 | ||
|
|
e1e760055c | ||
|
|
45bf6c3bf3 | ||
|
|
fef0cc764d | ||
|
|
72049afcf6 | ||
|
|
d26c06dbf3 | ||
|
|
268decd655 | ||
|
|
7ae246c839 | ||
|
|
c7c454e4fb | ||
|
|
8e27297a81 | ||
|
|
2cdec72985 | ||
|
|
0085ac534d | ||
|
|
7828a79a96 | ||
|
|
5576c21e67 | ||
|
|
e49cfb1d2b | ||
|
|
1e541d0225 | ||
|
|
0974afd26d | ||
|
|
8d93594771 | ||
|
|
1136ac70e8 | ||
|
|
dc8d5a39ea | ||
|
|
8329e649b0 | ||
|
|
66da8b164f | ||
|
|
ea48577864 | ||
|
|
597146b136 | ||
|
|
30dd0c1e11 | ||
|
|
88772c4266 | ||
|
|
dc1d9e1c84 | ||
|
|
69ea65835d | ||
|
|
d5bae3a8c6 | ||
|
|
f14010bd5b | ||
|
|
87094fc83f | ||
|
|
7c179cfeab | ||
|
|
7582c221d1 | ||
|
|
c109895848 | ||
|
|
eccedada40 | ||
|
|
25d54accf8 | ||
|
|
d07685f0e9 | ||
|
|
2445c00c7e | ||
|
|
4c1d3ef514 | ||
|
|
4614c7d4c2 | ||
|
|
bbf1ef0dc3 | ||
|
|
3433c9e708 | ||
|
|
2cd5d75a2e | ||
|
|
2535b8adef | ||
|
|
4edab7bb7f | ||
|
|
fd8658e317 | ||
|
|
51d21d8c86 | ||
|
|
b4c3307cdf | ||
|
|
4e8d10cb44 | ||
|
|
e96875a425 | ||
|
|
5ab0035348 | ||
|
|
4ddff96b1e | ||
|
|
a08d84c1df | ||
|
|
21c71bfac1 | ||
|
|
6baaed3581 | ||
|
|
152dbfd5d1 | ||
|
|
a56d14086b | ||
|
|
aee87693f8 | ||
|
|
976b4affd9 | ||
|
|
e222b6ad9c | ||
|
|
19b17374e8 | ||
|
|
43989122bb | ||
|
|
72712d6047 | ||
|
|
0b52d2cc15 | ||
|
|
8304102136 | ||
|
|
3381aefcfa | ||
|
|
279a365cb1 | ||
|
|
2c9e00da56 | ||
|
|
f7cae69704 | ||
|
|
b7d58bcdbc | ||
|
|
13a856b843 | ||
|
|
8da38985c3 | ||
|
|
60cf6c6b97 | ||
|
|
35c2b34564 | ||
|
|
ef2e048efc | ||
|
|
6b3261aa33 | ||
|
|
1849c02cb6 | ||
|
|
1ec74a89e2 | ||
|
|
c591792de9 | ||
|
|
3108543ae5 | ||
|
|
1eb221c743 | ||
|
|
bebf6bc2e7 | ||
|
|
9e91cc2138 | ||
|
|
c5b939cfb7 | ||
|
|
5bd411ca27 | ||
|
|
a533cda6f0 | ||
|
|
fe4b07b8ae | ||
|
|
f9f2ccd904 | ||
|
|
d9e87d7c32 | ||
|
|
a0092c0770 | ||
|
|
3100131125 | ||
|
|
988880cf83 | ||
|
|
c3fb9672c4 | ||
|
|
0a2d94e425 | ||
|
|
8d9073cd31 | ||
|
|
d075961ffa | ||
|
|
7a72409b61 | ||
|
|
34fc530fba | ||
|
|
f257ff2f97 | ||
|
|
7ad5822c5b | ||
|
|
9a8f9f0a94 | ||
|
|
6421cecafb | ||
|
|
be544d6d89 | ||
|
|
3c89ecafdd | ||
|
|
35ec4eec52 | ||
|
|
e47f737a2f | ||
|
|
ac671a065b | ||
|
|
74116cc550 | ||
|
|
406070a5c3 | ||
|
|
0ccafd5b53 | ||
|
|
940f517aa3 | ||
|
|
216e5f65ad | ||
|
|
a74685d66d | ||
|
|
b7791d2845 | ||
|
|
d151a82d78 | ||
|
|
8ce61fbd52 | ||
|
|
90c24aade3 | ||
|
|
6b3f787fee | ||
|
|
4ebe4ce1b7 | ||
|
|
8c79740ee8 | ||
|
|
59d027ca02 | ||
|
|
37a7345a90 | ||
|
|
c519d4651b | ||
|
|
9b3b609e40 | ||
|
|
6254f53716 | ||
|
|
f05dc46432 | ||
|
|
3de0982a4a | ||
|
|
c2184fb3bf | ||
|
|
919c09fcb0 | ||
|
|
1d9dbd40ec | ||
|
|
0cd953a6f3 | ||
|
|
4db2b72351 | ||
|
|
dd54fcbdbd | ||
|
|
3123cf7ac6 | ||
|
|
6b579dd4ce | ||
|
|
16dfaa3e27 | ||
|
|
d7842b9f84 | ||
|
|
115034fccb | ||
|
|
309e957a85 | ||
|
|
d7007e402e | ||
|
|
91323a21cf | ||
|
|
fea893d76c | ||
|
|
761bc6ba4c | ||
|
|
75172feb4e | ||
|
|
3285fb1608 | ||
|
|
03a4c6910d | ||
|
|
485b958599 | ||
|
|
da47ba2f67 | ||
|
|
c39195488a | ||
|
|
227fb0ae9b | ||
|
|
b12ff5b503 | ||
|
|
0946c72b88 | ||
|
|
7d49b046d4 | ||
|
|
5f0426c840 | ||
|
|
73e239cc5f | ||
|
|
ad670f721a | ||
|
|
028a4a70cf | ||
|
|
77d7960347 | ||
|
|
39821146bd | ||
|
|
7d505a41ac | ||
|
|
e457b2f0d6 | ||
|
|
c9cf7fd4d4 | ||
|
|
b0371dd33d | ||
|
|
25e16c3565 | ||
|
|
7b39527863 | ||
|
|
d861b08866 | ||
|
|
fb438dc108 | ||
|
|
4e6b4f179b | ||
|
|
00d038c8f3 | ||
|
|
a9f6a68952 | ||
|
|
b9142bbc5a | ||
|
|
6c812f663e | ||
|
|
a93ec9783a | ||
|
|
2d184d77b6 | ||
|
|
bce299ccc7 | ||
|
|
235cebd14a | ||
|
|
a638aa9d53 | ||
|
|
67cce0ef7e | ||
|
|
82f4267bf6 | ||
|
|
45a9ca29c4 | ||
|
|
7f4e813277 | ||
|
|
3805ff4a0c | ||
|
|
464cfd475e | ||
|
|
fe469ae57f | ||
|
|
550ef9a1c4 | ||
|
|
935adfb51a | ||
|
|
3974df4a62 | ||
|
|
4870974161 | ||
|
|
8c4b0037f5 | ||
|
|
2c6f763ef2 | ||
|
|
ca28de02d8 | ||
|
|
bfc15ea029 | ||
|
|
6e8b8a5920 | ||
|
|
099f3b6a62 | ||
|
|
142d182bc1 | ||
|
|
1437871d88 | ||
|
|
352bf69409 | ||
|
|
9bdf3d23e1 | ||
|
|
be8ecfa707 | ||
|
|
51da0d0259 | ||
|
|
f55b78a994 | ||
|
|
e1a44c93f8 | ||
|
|
07e7087a09 | ||
|
|
2c79c7e2f6 | ||
|
|
09f6637fe0 | ||
|
|
3784db3308 | ||
|
|
2b950ff5dd | ||
|
|
09339c9cfb | ||
|
|
ccadd88af5 | ||
|
|
cc02a0efc2 | ||
|
|
43a1385b79 | ||
|
|
5101464e3b | ||
|
|
3d71478d38 | ||
|
|
4989ed445e | ||
|
|
d9413039ec | ||
|
|
eba0c9be34 | ||
|
|
48c9e9f3cc | ||
|
|
81ebde88db | ||
|
|
79ced9d0f8 | ||
|
|
a4058b84ce | ||
|
|
7bf211a52b | ||
|
|
d5f722792f | ||
|
|
0f02906c9b | ||
|
|
9582e228b1 | ||
|
|
45f20431f9 | ||
|
|
7554e6d7f9 | ||
|
|
cb8f26f177 | ||
|
|
b5dfce7861 | ||
|
|
2ca5a65544 | ||
|
|
17deb136db | ||
|
|
8c9710c76c | ||
|
|
32f7ecb261 | ||
|
|
fb77fde710 | ||
|
|
3c67bb90d7 | ||
|
|
dabb168853 | ||
|
|
45e5b3b219 | ||
|
|
a6b7469923 | ||
|
|
cb5dab3033 | ||
|
|
21d0038ff2 | ||
|
|
c094d8f2e8 | ||
|
|
c465d6a6c2 | ||
|
|
73d35bc985 | ||
|
|
2a8ccb065b | ||
|
|
8f04a50ce1 | ||
|
|
888aa5586b | ||
|
|
99f56f5d22 | ||
|
|
ad6281090d | ||
|
|
f0d334d3e2 | ||
|
|
5f829b048f | ||
|
|
1a961e66ff | ||
|
|
fdb0e22656 | ||
|
|
132ee1915f | ||
|
|
44bf4f3c8f | ||
|
|
6237767d5a | ||
|
|
dec9d96417 | ||
|
|
b167c87267 | ||
|
|
2280fe8e8e | ||
|
|
575d6dcd2d | ||
|
|
f729490c6b | ||
|
|
b32124cdd6 | ||
|
|
3d4321ee38 | ||
|
|
85034b382e | ||
|
|
77a51d1ad4 | ||
|
|
33e0cdc2d7 | ||
|
|
6519faa2fe | ||
|
|
5e3a234cbe | ||
|
|
e54c31d2d5 | ||
|
|
66c0537251 | ||
|
|
ac58516593 | ||
|
|
c3da6322b5 | ||
|
|
3d241500cf | ||
|
|
ded8224f66 | ||
|
|
f8814881a1 | ||
|
|
cc2852cd48 | ||
|
|
467637a9eb | ||
|
|
3cfc292d84 | ||
|
|
6acf94a810 | ||
|
|
31367fb4c4 | ||
|
|
12d6074e3b | ||
|
|
ff30386051 | ||
|
|
601f99ac16 | ||
|
|
87fe5c6101 | ||
|
|
68399ca31c | ||
|
|
2a6d7fd80f | ||
|
|
4725f510d8 | ||
|
|
be0ba22222 | ||
|
|
c8781392be | ||
|
|
b97164fcfb | ||
|
|
0dfb92281b | ||
|
|
4fe80c40da | ||
|
|
f0fac5115a | ||
|
|
46dd389d0d |
No files matched your search
+166
-59
@@ -1,5 +1,6 @@
|
||||
name: ci
|
||||
on: pull_request
|
||||
on: ["push", "pull_request"]
|
||||
|
||||
jobs:
|
||||
|
||||
lint:
|
||||
@@ -10,58 +11,18 @@ jobs:
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- run: make install tools
|
||||
- name: extract pip cache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
|
||||
restore-keys: ${{ runner.os }}-pip-
|
||||
- run: pip install --user --upgrade pip wheel
|
||||
- run: pip install -e .[torrent,lint]
|
||||
- run: make lint
|
||||
|
||||
tests-unit:
|
||||
name: "tests / unit"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- run: make install tools
|
||||
- working-directory: lbry
|
||||
env:
|
||||
HOME: /tmp
|
||||
run: coverage run -p --source=lbry -m unittest discover -vv tests.unit
|
||||
|
||||
tests-integration:
|
||||
name: "tests / integration"
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
test:
|
||||
- datanetwork
|
||||
- blockchain
|
||||
- other
|
||||
steps:
|
||||
- name: Configure sysctl limits
|
||||
run: |
|
||||
sudo swapoff -a
|
||||
sudo sysctl -w vm.swappiness=1
|
||||
sudo sysctl -w fs.file-max=262144
|
||||
sudo sysctl -w vm.max_map_count=262144
|
||||
|
||||
- name: Runs Elasticsearch
|
||||
uses: elastic/elastic-github-actions/elasticsearch@master
|
||||
with:
|
||||
stack-version: 7.6.0
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- if: matrix.test == 'other'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends ffmpeg
|
||||
- run: pip install tox-travis
|
||||
- run: tox -e ${{ matrix.test }}
|
||||
|
||||
build:
|
||||
needs: ["lint", "tests-unit", "tests-integration"]
|
||||
name: "build"
|
||||
strategy:
|
||||
matrix:
|
||||
os:
|
||||
@@ -74,22 +35,168 @@ jobs:
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- name: Setup
|
||||
- name: set pip cache dir
|
||||
id: pip-cache
|
||||
run: echo "::set-output name=dir::$(pip cache dir)"
|
||||
- name: extract pip cache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ${{ steps.pip-cache.outputs.dir }}
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
|
||||
restore-keys: ${{ runner.os }}-pip-
|
||||
- id: os-name
|
||||
uses: ASzc/change-string-case-action@v1
|
||||
with:
|
||||
string: ${{ runner.os }}
|
||||
- run: pip install --user --upgrade pip wheel
|
||||
- if: startsWith(runner.os, 'linux')
|
||||
run: pip install -e .[torrent,test]
|
||||
- if: startsWith(runner.os, 'linux')
|
||||
env:
|
||||
HOME: /tmp
|
||||
run: make test-unit-coverage
|
||||
- if: startsWith(runner.os, 'linux') != true
|
||||
run: pip install -e .[test]
|
||||
- if: startsWith(runner.os, 'linux') != true
|
||||
env:
|
||||
HOME: /tmp
|
||||
run: coverage run --source=lbry -m unittest tests/unit/test_conf.py
|
||||
- name: submit coverage report
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
COVERALLS_FLAG_NAME: tests-unit-${{ steps.os-name.outputs.lowercase }}
|
||||
COVERALLS_PARALLEL: true
|
||||
run: |
|
||||
pip install pyinstaller
|
||||
pip install -e .
|
||||
# https://stackoverflow.com/a/61693590
|
||||
# https://github.com/pypa/setuptools/issues/1963
|
||||
pip install --upgrade 'setuptools<45.0.0'
|
||||
- if: startsWith(matrix.os, 'windows') == false
|
||||
pip install coveralls
|
||||
coveralls --service=github
|
||||
|
||||
tests-integration:
|
||||
name: "tests / integration"
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
test:
|
||||
- datanetwork
|
||||
- blockchain
|
||||
- blockchain_legacy_search
|
||||
- other
|
||||
steps:
|
||||
- name: Configure sysctl limits
|
||||
run: |
|
||||
sudo swapoff -a
|
||||
sudo sysctl -w vm.swappiness=1
|
||||
sudo sysctl -w fs.file-max=262144
|
||||
sudo sysctl -w vm.max_map_count=262144
|
||||
- name: Runs Elasticsearch
|
||||
uses: elastic/elastic-github-actions/elasticsearch@master
|
||||
with:
|
||||
stack-version: 7.6.0
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- if: matrix.test == 'other'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends ffmpeg
|
||||
- name: extract pip cache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ./.tox
|
||||
key: tox-integration-${{ matrix.test }}-${{ hashFiles('setup.py') }}
|
||||
restore-keys: txo-integration-${{ matrix.test }}-
|
||||
- run: pip install tox coverage coveralls
|
||||
- run: tox -e ${{ matrix.test }}
|
||||
- name: submit coverage report
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
COVERALLS_FLAG_NAME: tests-integration-${{ matrix.test }}
|
||||
COVERALLS_PARALLEL: true
|
||||
run: |
|
||||
coverage combine tests
|
||||
coveralls --service=github
|
||||
|
||||
|
||||
coverage:
|
||||
needs: ["tests-unit", "tests-integration"]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: finalize coverage report submission
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
pip install coveralls
|
||||
coveralls --service=github --finish
|
||||
|
||||
build:
|
||||
needs: ["lint", "tests-unit", "tests-integration"]
|
||||
name: "build / binary"
|
||||
strategy:
|
||||
matrix:
|
||||
os:
|
||||
- ubuntu-18.04
|
||||
- macos-latest
|
||||
- windows-latest
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- id: os-name
|
||||
uses: ASzc/change-string-case-action@v1
|
||||
with:
|
||||
string: ${{ runner.os }}
|
||||
- name: set pip cache dir
|
||||
id: pip-cache
|
||||
run: echo "::set-output name=dir::$(pip cache dir)"
|
||||
- name: extract pip cache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ${{ steps.pip-cache.outputs.dir }}
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
|
||||
restore-keys: ${{ runner.os }}-pip-
|
||||
- run: pip install pyinstaller==4.4
|
||||
- run: pip install -e .
|
||||
- if: startsWith(github.ref, 'refs/tags/v')
|
||||
run: python docker/set_build.py
|
||||
- if: startsWith(runner.os, 'linux') || startsWith(runner.os, 'mac')
|
||||
name: Build & Run (Unix)
|
||||
run: |
|
||||
pyinstaller --onefile --name lbrynet lbry/extras/cli.py
|
||||
chmod +x dist/lbrynet
|
||||
dist/lbrynet --version
|
||||
- if: startsWith(matrix.os, 'windows')
|
||||
- if: startsWith(runner.os, 'windows')
|
||||
name: Build & Run (Windows)
|
||||
run: |
|
||||
pip install pywin32
|
||||
pip install pywin32==301
|
||||
pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico --onefile --name lbrynet lbry/extras/cli.py
|
||||
dist/lbrynet.exe --version
|
||||
- uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: lbrynet-${{ steps.os-name.outputs.lowercase }}
|
||||
path: dist/
|
||||
|
||||
release:
|
||||
name: "release"
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
needs: ["build"]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/download-artifact@v2
|
||||
- name: upload binaries
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.RELEASE_API_TOKEN }}
|
||||
run: |
|
||||
pip install githubrelease
|
||||
chmod +x lbrynet-macos/lbrynet
|
||||
chmod +x lbrynet-linux/lbrynet
|
||||
zip --junk-paths lbrynet-mac.zip lbrynet-macos/lbrynet
|
||||
zip --junk-paths lbrynet-linux.zip lbrynet-linux/lbrynet
|
||||
zip --junk-paths lbrynet-windows.zip lbrynet-windows/lbrynet.exe
|
||||
ls -lh
|
||||
githubrelease release lbryio/lbry-sdk info ${GITHUB_REF#refs/tags/}
|
||||
githubrelease asset lbryio/lbry-sdk upload ${GITHUB_REF#refs/tags/} \
|
||||
lbrynet-mac.zip lbrynet-linux.zip lbrynet-windows.zip
|
||||
githubrelease release lbryio/lbry-sdk publish ${GITHUB_REF#refs/tags/}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
name: slack
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
|
||||
jobs:
|
||||
release:
|
||||
name: "slack notification"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: LoveToKnow/slackify-markdown-action@v1.0.0
|
||||
id: markdown
|
||||
with:
|
||||
text: "There is a new SDK release: ${{github.event.release.html_url}}\n${{ github.event.release.body }}"
|
||||
- uses: slackapi/slack-github-action@v1.14.0
|
||||
env:
|
||||
CHANGELOG: '<!channel> ${{ steps.markdown.outputs.text }}'
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_RELEASE_BOT_WEBHOOK }}
|
||||
with:
|
||||
payload: '{"type": "mrkdwn", "text": ${{ toJSON(env.CHANGELOG) }} }'
|
||||
|
||||
-218
@@ -1,218 +0,0 @@
|
||||
default:
|
||||
image: python:3.7
|
||||
services:
|
||||
- name: docker.elastic.co/elasticsearch/elasticsearch:7.11.2
|
||||
alias: elasticsearch
|
||||
variables:
|
||||
ELASTIC_HOST: elasticsearch
|
||||
|
||||
|
||||
#cache:
|
||||
# directories:
|
||||
# - $HOME/venv
|
||||
# - $HOME/.cache/pip
|
||||
# - $HOME/Library/Caches/pip
|
||||
# - $HOME/Library/Caches/Homebrew
|
||||
# - $TRAVIS_BUILD_DIR/.tox
|
||||
|
||||
|
||||
stages:
|
||||
- test
|
||||
- build
|
||||
- assets
|
||||
- release
|
||||
|
||||
|
||||
.tagged:
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG =~ /^v[0-9\.]+$/'
|
||||
when: on_success
|
||||
|
||||
|
||||
|
||||
test:lint:
|
||||
stage: test
|
||||
script:
|
||||
- make install tools
|
||||
- make lint
|
||||
|
||||
test:unit:
|
||||
stage: test
|
||||
script:
|
||||
- make install tools
|
||||
- HOME=/tmp coverage run -p --source=lbry -m unittest discover -vv tests.unit
|
||||
|
||||
test:datanetwork-integration:
|
||||
stage: test
|
||||
script:
|
||||
- pip install tox-travis
|
||||
- tox -e datanetwork --recreate
|
||||
|
||||
test:blockchain-integration:
|
||||
stage: test
|
||||
script:
|
||||
- pip install tox-travis
|
||||
- tox -e blockchain
|
||||
|
||||
test:other-integration:
|
||||
stage: test
|
||||
script:
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends ffmpeg
|
||||
- pip install tox-travis
|
||||
- tox -e other
|
||||
|
||||
test:json-api:
|
||||
stage: test
|
||||
script:
|
||||
- make install tools
|
||||
- HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
|
||||
|
||||
|
||||
|
||||
.build:
|
||||
stage: build
|
||||
artifacts:
|
||||
expire_in: 1 day
|
||||
paths:
|
||||
- lbrynet-${OS}.zip
|
||||
script:
|
||||
- pip install --upgrade 'setuptools<45.0.0'
|
||||
- pip install pyinstaller
|
||||
- pip install -e .
|
||||
- python3.7 docker/set_build.py # must come after lbry is installed because it imports lbry
|
||||
- pyinstaller --onefile --name lbrynet lbry/extras/cli.py
|
||||
- chmod +x dist/lbrynet
|
||||
- zip --junk-paths ${CI_PROJECT_DIR}/lbrynet-${OS}.zip dist/lbrynet # gitlab expects artifacts to be in $CI_PROJECT_DIR
|
||||
- openssl dgst -sha256 ${CI_PROJECT_DIR}/lbrynet-${OS}.zip | egrep -o [0-9a-f]+$ # get sha256 of asset. works on mac and ubuntu
|
||||
- dist/lbrynet --version
|
||||
|
||||
build:linux:
|
||||
extends: .build
|
||||
image: ubuntu:16.04
|
||||
variables:
|
||||
OS: linux
|
||||
before_script:
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends software-properties-common zip curl build-essential
|
||||
- add-apt-repository -y ppa:deadsnakes/ppa
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends python3.7-dev
|
||||
- python3.7 <(curl -q https://bootstrap.pypa.io/get-pip.py) # make sure we get pip with python3.7
|
||||
- pip install lbry-libtorrent
|
||||
|
||||
build:mac:
|
||||
extends: .build
|
||||
tags: [macos] # makes gitlab use the mac runner
|
||||
variables:
|
||||
OS: mac
|
||||
GIT_DEPTH: 5
|
||||
VENV: /tmp/gitlab-lbry-sdk-venv
|
||||
before_script:
|
||||
# - brew upgrade python || true
|
||||
- python3 --version | grep -q '^Python 3\.7\.' # dont upgrade python on every run. just make sure we're on the right Python
|
||||
# - pip3 install --user --upgrade pip virtualenv
|
||||
- pip3 --version | grep -q '\(python 3\.7\)'
|
||||
- virtualenv --python=python3.7 "${VENV}"
|
||||
- source "${VENV}/bin/activate"
|
||||
after_script:
|
||||
- rm -rf "${VENV}"
|
||||
|
||||
build:windows:
|
||||
extends: .build
|
||||
tags: [windows] # makes gitlab use the windows runner
|
||||
variables:
|
||||
OS: windows
|
||||
GIT_DEPTH: 5
|
||||
before_script:
|
||||
- ./docker/install_choco.ps1
|
||||
- choco install -y --x64 python --version=3.7.9
|
||||
- choco install -y 7zip checksum
|
||||
- python --version # | findstr /B "Python 3\.7\." # dont upgrade python on every run. just make sure we're on the right Python
|
||||
- pip --version # | findstr /E '\(python 3\.7\)'
|
||||
- python -c "import sys;print(f'{str(64 if sys.maxsize > 2**32 else 32)} bit python\n{sys.platform}')"
|
||||
- pip install virtualenv pywin32
|
||||
- virtualenv venv
|
||||
- venv/Scripts/activate.ps1
|
||||
- pip install pip==19.3.1; $true # $true ignores errors. need this to get the correct coincurve wheel. see commit notes for details.
|
||||
after_script:
|
||||
- rmdir -Recurse venv
|
||||
script:
|
||||
- pip install --upgrade 'setuptools<45.0.0'
|
||||
- pip install pyinstaller==3.5
|
||||
- pip install -e .
|
||||
- python docker/set_build.py # must come after lbry is installed because it imports lbry
|
||||
- pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico -F -n lbrynet lbry/extras/cli.py
|
||||
- 7z a -tzip $env:CI_PROJECT_DIR/lbrynet-${OS}.zip ./dist/lbrynet.exe
|
||||
- checksum --type=sha256 --file=$env:CI_PROJECT_DIR/lbrynet-${OS}.zip
|
||||
- dist/lbrynet.exe --version
|
||||
|
||||
|
||||
|
||||
# s3 = upload asset to s3 (build.lbry.io)
|
||||
.s3:
|
||||
stage: assets
|
||||
variables:
|
||||
GIT_STRATEGY: none
|
||||
script:
|
||||
- "[ -f lbrynet-${OS}.zip ]" # check that asset exists before trying to upload
|
||||
- pip install awscli
|
||||
- S3_PATH="daemon/gitlab-build-${CI_PIPELINE_ID}_commit-${CI_COMMIT_SHA:0:7}$( if [ ! -z ${CI_COMMIT_TAG} ]; then echo _tag-${CI_COMMIT_TAG}; else echo _branch-${CI_COMMIT_REF_NAME}; fi )"
|
||||
- AWS_ACCESS_KEY_ID=${ARTIFACTS_KEY} AWS_SECRET_ACCESS_KEY=${ARTIFACTS_SECRET} AWS_REGION=${ARTIFACTS_REGION}
|
||||
aws s3 cp lbrynet-${OS}.zip s3://${ARTIFACTS_BUCKET}/${S3_PATH}/lbrynet-${OS}.zip
|
||||
|
||||
s3:linux:
|
||||
extends: .s3
|
||||
variables: {OS: linux}
|
||||
needs: ["build:linux"]
|
||||
|
||||
s3:mac:
|
||||
extends: .s3
|
||||
variables: {OS: mac}
|
||||
needs: ["build:mac"]
|
||||
|
||||
s3:windows:
|
||||
extends: .s3
|
||||
variables: {OS: windows}
|
||||
needs: ["build:windows"]
|
||||
|
||||
|
||||
|
||||
# github = upload assets to github when there's a tagged release
|
||||
.github:
|
||||
extends: .tagged
|
||||
stage: assets
|
||||
variables:
|
||||
GIT_STRATEGY: none
|
||||
script:
|
||||
- "[ -f lbrynet-${OS}.zip ]" # check that asset exists before trying to upload. githubrelease won't error if its missing
|
||||
- pip install githubrelease
|
||||
- githubrelease --no-progress --github-token ${GITHUB_CI_USER_ACCESS_TOKEN} asset lbryio/lbry-sdk upload ${CI_COMMIT_TAG} lbrynet-${OS}.zip
|
||||
|
||||
github:linux:
|
||||
extends: .github
|
||||
variables: {OS: linux}
|
||||
needs: ["build:linux"]
|
||||
|
||||
github:mac:
|
||||
extends: .github
|
||||
variables: {OS: mac}
|
||||
needs: ["build:mac"]
|
||||
|
||||
github:windows:
|
||||
extends: .github
|
||||
variables: {OS: windows}
|
||||
needs: ["build:windows"]
|
||||
|
||||
|
||||
|
||||
publish:
|
||||
extends: .tagged
|
||||
stage: release
|
||||
variables:
|
||||
GIT_STRATEGY: none
|
||||
script:
|
||||
- pip install githubrelease
|
||||
- githubrelease --no-progress --github-token ${GITHUB_CI_USER_ACCESS_TOKEN} release lbryio/lbry-sdk publish ${CI_COMMIT_TAG}
|
||||
- >
|
||||
curl -X POST -H 'Content-type: application/json' --data '{"text":"<!channel> There is a new SDK release: https://github.com/lbryio/lbry-sdk/releases/tag/'"${CI_COMMIT_TAG}"'\n'"$(curl -s "https://api.github.com/repos/lbryio/lbry-sdk/releases/tags/${CI_COMMIT_TAG}" | egrep '\w*\"body\":' | cut -d':' -f 2- | tail -c +3 | head -c -2)"'", "channel":"tech"}' "$(echo ${SLACK_WEBHOOK_URL_BASE64} | base64 -d)"
|
||||
+113
-44
@@ -9,20 +9,29 @@ Here's a video walkthrough of this setup, which is itself hosted by the LBRY net
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Running `lbrynet` from source requires Python 3.7 or higher. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/).
|
||||
Running `lbrynet` from source requires Python 3.7. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/).
|
||||
|
||||
After installing python 3, you'll need to install some additional libraries depending on your operating system.
|
||||
After installing Python 3.7, you'll need to install some additional libraries depending on your operating system.
|
||||
|
||||
Because of [issue #2769](https://github.com/lbryio/lbry-sdk/issues/2769)
|
||||
at the moment the `lbrynet` daemon will only work correctly with Python 3.7.
|
||||
If Python 3.8+ is used, the daemon will start but the RPC server
|
||||
may not accept messages, returning the following:
|
||||
```
|
||||
Could not connect to daemon. Are you sure it's running?
|
||||
```
|
||||
|
||||
### macOS
|
||||
|
||||
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
|
||||
|
||||
These environment variables also need to be set:
|
||||
1. PYTHONUNBUFFERED=1
|
||||
2. EVENT_NOKQUEUE=1
|
||||
```
|
||||
PYTHONUNBUFFERED=1
|
||||
EVENT_NOKQUEUE=1
|
||||
```
|
||||
|
||||
Remaining dependencies can then be installed by running:
|
||||
|
||||
```
|
||||
brew install python protobuf
|
||||
```
|
||||
@@ -31,14 +40,17 @@ Assistance installing Python3: https://docs.python-guide.org/starting/install3/o
|
||||
|
||||
### Linux
|
||||
|
||||
On Ubuntu (16.04 minimum, we recommend 18.04), install the following:
|
||||
|
||||
On Ubuntu (we recommend 18.04 or 20.04), install the following:
|
||||
```
|
||||
sudo add-apt-repository ppa:deadsnakes/ppa
|
||||
sudo apt-get update
|
||||
sudo apt-get install build-essential python3.7 python3.7-dev git python3.7-venv libssl-dev python-protobuf
|
||||
```
|
||||
|
||||
The [deadsnakes PPA](https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa) provides Python 3.7
|
||||
for those Ubuntu distributions that no longer have it in their
|
||||
official repositories.
|
||||
|
||||
On Raspbian, you will also need to install `python-pyparsing`.
|
||||
|
||||
If you're running another Linux distro, install the equivalent of the above packages for your system.
|
||||
@@ -47,62 +59,119 @@ If you're running another Linux distro, install the equivalent of the above pack
|
||||
|
||||
### Linux/Mac
|
||||
|
||||
To install on Linux/Mac:
|
||||
Clone the repository:
|
||||
```bash
|
||||
git clone https://github.com/lbryio/lbry-sdk.git
|
||||
cd lbry-sdk
|
||||
```
|
||||
|
||||
```
|
||||
Clone the repository:
|
||||
$ git clone https://github.com/lbryio/lbry-sdk.git
|
||||
$ cd lbry-sdk
|
||||
Create a Python virtual environment for lbry-sdk:
|
||||
```bash
|
||||
python3.7 -m venv lbry-venv
|
||||
```
|
||||
|
||||
Create a Python virtual environment for lbry-sdk:
|
||||
$ python3.7 -m venv lbry-venv
|
||||
|
||||
Activating lbry-sdk virtual environment:
|
||||
$ source lbry-venv/bin/activate
|
||||
|
||||
Make sure you're on Python 3.7+ (as the default Python in virtual environment):
|
||||
$ python --version
|
||||
Activate virtual environment:
|
||||
```bash
|
||||
source lbry-venv/bin/activate
|
||||
```
|
||||
|
||||
Install packages:
|
||||
$ make install
|
||||
Make sure you're on Python 3.7+ as default in the virtual environment:
|
||||
```bash
|
||||
python --version
|
||||
```
|
||||
|
||||
If you are on Linux and using PyCharm, generates initial configs:
|
||||
$ make idea
|
||||
```
|
||||
Install packages:
|
||||
```bash
|
||||
make install
|
||||
```
|
||||
|
||||
To verify your installation, `which lbrynet` should return a path inside of the `lbry-venv` folder created by the `python3.7 -m venv lbry-venv` command.
|
||||
If you are on Linux and using PyCharm, generates initial configs:
|
||||
```bash
|
||||
make idea
|
||||
```
|
||||
|
||||
To verify your installation, `which lbrynet` should return a path inside
|
||||
of the `lbry-venv` folder.
|
||||
```bash
|
||||
(lbry-venv) $ which lbrynet
|
||||
/opt/lbry-sdk/lbry-venv/bin/lbrynet
|
||||
```
|
||||
|
||||
To exit the virtual environment simply use the command `deactivate`.
|
||||
|
||||
### Windows
|
||||
|
||||
To install on Windows:
|
||||
Clone the repository:
|
||||
```bash
|
||||
git clone https://github.com/lbryio/lbry-sdk.git
|
||||
cd lbry-sdk
|
||||
```
|
||||
|
||||
```
|
||||
Clone the repository:
|
||||
> git clone https://github.com/lbryio/lbry-sdk.git
|
||||
> cd lbry-sdk
|
||||
Create a Python virtual environment for lbry-sdk:
|
||||
```bash
|
||||
python -m venv lbry-venv
|
||||
```
|
||||
|
||||
Create a Python virtual environment for lbry-sdk:
|
||||
> python -m venv lbry-venv
|
||||
Activate virtual environment:
|
||||
```bash
|
||||
lbry-venv\Scripts\activate
|
||||
```
|
||||
|
||||
Activating lbry-sdk virtual environment:
|
||||
> lbry-venv\Scripts\activate
|
||||
|
||||
Install packages:
|
||||
> pip install -e .
|
||||
```
|
||||
Install packages:
|
||||
```bash
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
## Run the tests
|
||||
### Elasticsearch
|
||||
|
||||
To run the unit tests from the repo directory:
|
||||
For running integration tests, Elasticsearch is required to be available at localhost:9200/
|
||||
|
||||
```
|
||||
python -m unittest discover tests.unit
|
||||
```
|
||||
The easiest way to start it is using docker with:
|
||||
```bash
|
||||
make elastic-docker
|
||||
```
|
||||
|
||||
Alternative installation methods are available [at Elasticsearch website](https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html).
|
||||
|
||||
To run the unit and integration tests from the repo directory:
|
||||
```
|
||||
python -m unittest discover tests.unit
|
||||
python -m unittest discover tests.integration
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
To start the API server:
|
||||
`lbrynet start`
|
||||
```
|
||||
lbrynet start
|
||||
```
|
||||
|
||||
Whenever the code inside [lbry-sdk/lbry](./lbry)
|
||||
is modified we should run `make install` to recompile the `lbrynet`
|
||||
executable with the newest code.
|
||||
|
||||
## Development
|
||||
|
||||
When developing, remember to enter the environment,
|
||||
and if you wish start the server interactively.
|
||||
```bash
|
||||
$ source lbry-venv/bin/activate
|
||||
|
||||
(lbry-venv) $ python lbry/extras/cli.py start
|
||||
```
|
||||
|
||||
Parameters can be passed in the same way.
|
||||
```bash
|
||||
(lbry-venv) $ python lbry/extras/cli.py wallet balance
|
||||
```
|
||||
|
||||
If a Python debugger (`pdb` or `ipdb`) is installed we can also start it
|
||||
in this way, set up break points, and step through the code.
|
||||
```bash
|
||||
(lbry-venv) $ pip install ipdb
|
||||
|
||||
(lbry-venv) $ ipdb lbry/extras/cli.py
|
||||
```
|
||||
|
||||
Happy hacking!
|
||||
@@ -1,24 +1,26 @@
|
||||
.PHONY: install tools lint test idea
|
||||
.PHONY: install tools lint test test-unit test-unit-coverage test-integration idea
|
||||
|
||||
install:
|
||||
pip install https://s3.amazonaws.com/files.lbry.io/python_libtorrent-1.2.4-py3-none-any.whl
|
||||
CFLAGS="-DSQLITE_MAX_VARIABLE_NUMBER=2500000" pip install -U https://github.com/rogerbinns/apsw/releases/download/3.30.1-r1/apsw-3.30.1-r1.zip \
|
||||
--global-option=fetch \
|
||||
--global-option=--version --global-option=3.30.1 --global-option=--all \
|
||||
--global-option=build --global-option=--enable --global-option=fts5
|
||||
pip install -e .
|
||||
|
||||
tools:
|
||||
pip install mypy==0.701 pylint==2.4.4
|
||||
pip install coverage astroid pylint
|
||||
|
||||
lint:
|
||||
pylint --rcfile=setup.cfg lbry
|
||||
#mypy --ignore-missing-imports lbry
|
||||
|
||||
test:
|
||||
test: test-unit test-integration
|
||||
|
||||
test-unit:
|
||||
python -m unittest discover tests.unit
|
||||
|
||||
test-unit-coverage:
|
||||
coverage run --source=lbry -m unittest discover -vv tests.unit
|
||||
|
||||
test-integration:
|
||||
tox
|
||||
|
||||
idea:
|
||||
mkdir -p .idea
|
||||
cp -r scripts/idea/* .idea
|
||||
|
||||
elastic-docker:
|
||||
docker run -d -v lbryhub:/usr/share/elasticsearch/data -p 9200:9200 -p 9300:9300 -e"ES_JAVA_OPTS=-Xms512m -Xmx512m" -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.12.1
|
||||
@@ -1,10 +1,10 @@
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry-sdk/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://ci.lbry.tech/lbry/lbry-sdk)
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry-sdk/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://github.com/lbryio/lbry-sdk/actions/workflows/main.yml) [](https://coveralls.io/github/lbryio/lbry-sdk)
|
||||
|
||||
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
|
||||
LBRY SDK for Python is currently the most fully featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components include:
|
||||
|
||||
* Built on Python 3.7+ and `asyncio`.
|
||||
* Built on Python 3.7 and `asyncio`.
|
||||
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbry.dht](https://github.com/lbryio/lbry-sdk/tree/master/lbry/dht)).
|
||||
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbry.blob_exchange](https://github.com/lbryio/lbry-sdk/tree/master/lbry/blob_exchange)).
|
||||
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbry.schema](https://github.com/lbryio/lbry-sdk/tree/master/lbry/schema)).
|
||||
@@ -41,7 +41,7 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
|
||||
|
||||
## Security
|
||||
|
||||
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.com/faq/gpg-key) if you need it.
|
||||
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our PGP key is here](https://lbry.com/faq/pgp-key) if you need it.
|
||||
|
||||
## Contact
|
||||
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
# Security Policy
|
||||
|
||||
## Supported Versions
|
||||
|
||||
While we are not at v1.0 yet, only the latest release will be supported.
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
See https://lbry.com/faq/security
|
||||
@@ -0,0 +1,38 @@
|
||||
FROM debian:10-slim
|
||||
|
||||
ARG user=lbry
|
||||
ARG projects_dir=/home/$user
|
||||
|
||||
ARG DOCKER_TAG
|
||||
ARG DOCKER_COMMIT=docker
|
||||
ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get -y --no-install-recommends install \
|
||||
wget \
|
||||
automake libtool \
|
||||
tar unzip \
|
||||
build-essential \
|
||||
pkg-config \
|
||||
libleveldb-dev \
|
||||
python3.7 \
|
||||
python3-dev \
|
||||
python3-pip \
|
||||
python3-wheel \
|
||||
python3-setuptools && \
|
||||
update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN groupadd -g 999 $user && useradd -m -u 999 -g $user $user
|
||||
|
||||
COPY . $projects_dir
|
||||
RUN chown -R $user:$user $projects_dir
|
||||
|
||||
USER $user
|
||||
WORKDIR $projects_dir
|
||||
|
||||
RUN make install
|
||||
RUN python3 docker/set_build.py
|
||||
RUN rm ~/.cache -rf
|
||||
ENTRYPOINT ["python3", "scripts/dht_node.py"]
|
||||
|
||||
@@ -13,6 +13,7 @@ RUN apt-get update && \
|
||||
wget \
|
||||
tar unzip \
|
||||
build-essential \
|
||||
automake libtool \
|
||||
pkg-config \
|
||||
libleveldb-dev \
|
||||
python3.7 \
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
FROM debian:10-slim
|
||||
|
||||
ARG user=lbry
|
||||
ARG downloads_dir=/database
|
||||
ARG projects_dir=/home/$user
|
||||
|
||||
ARG DOCKER_TAG
|
||||
ARG DOCKER_COMMIT=docker
|
||||
ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get -y --no-install-recommends install \
|
||||
wget \
|
||||
automake libtool \
|
||||
tar unzip \
|
||||
build-essential \
|
||||
pkg-config \
|
||||
libleveldb-dev \
|
||||
python3.7 \
|
||||
python3-dev \
|
||||
python3-pip \
|
||||
python3-wheel \
|
||||
python3-setuptools && \
|
||||
update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN groupadd -g 999 $user && useradd -m -u 999 -g $user $user
|
||||
RUN mkdir -p $downloads_dir
|
||||
RUN chown -R $user:$user $downloads_dir
|
||||
|
||||
COPY . $projects_dir
|
||||
RUN chown -R $user:$user $projects_dir
|
||||
|
||||
USER $user
|
||||
WORKDIR $projects_dir
|
||||
|
||||
RUN pip install uvloop
|
||||
RUN make install
|
||||
RUN python3 docker/set_build.py
|
||||
RUN rm ~/.cache -rf
|
||||
|
||||
# entry point
|
||||
VOLUME $downloads_dir
|
||||
COPY ./docker/webconf.yaml /webconf.yaml
|
||||
ENTRYPOINT ["/home/lbry/.local/bin/lbrynet", "start", "--config=/webconf.yaml"]
|
||||
@@ -0,0 +1,9 @@
|
||||
### How to run with docker-compose
|
||||
1. Edit config file and after that fix permissions with
|
||||
```
|
||||
sudo chown -R 999:999 webconf.yaml
|
||||
```
|
||||
2. Start SDK with
|
||||
```
|
||||
docker-compose up -d
|
||||
```
|
||||
@@ -16,11 +16,16 @@ services:
|
||||
- "2112:2112" # uncomment to enable prometheus
|
||||
volumes:
|
||||
- "wallet_server:/database"
|
||||
env_file: [/home/lbry/wallet-server-env]
|
||||
environment:
|
||||
- DAEMON_URL=http://lbry:lbry@127.0.0.1:9245
|
||||
- TCP_PORT=50001
|
||||
- PROMETHEUS_PORT=2112
|
||||
- QUERY_TIMEOUT_MS=3000 # how long search queries allowed to run before cancelling, in milliseconds
|
||||
- TRENDING_ALGORITHMS=variable_decay
|
||||
- MAX_SEND=10000000000000 # deprecated. leave it high until its removed
|
||||
- MAX_SUBS=1000000000000 # deprecated. leave it high until its removed
|
||||
- FILTERING_CHANNEL_IDS=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8
|
||||
- BLOCKING_CHANNEL_IDS=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6 e4e230b131082f6b10c8f7994bbb83f29e8e6fb9
|
||||
es01:
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0
|
||||
container_name: es01
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
version: '3'
|
||||
services:
|
||||
websdk:
|
||||
image: vshyba/websdk
|
||||
ports:
|
||||
- '5279:5279'
|
||||
- '5280:5280'
|
||||
volumes:
|
||||
- ./webconf.yaml:/webconf.yaml
|
||||
+5
-3
@@ -20,7 +20,7 @@ def _check_and_set(d: dict, key: str, value: str):
|
||||
def main():
|
||||
build_info = {item: build_info_mod.__dict__[item] for item in dir(build_info_mod) if not item.startswith("__")}
|
||||
|
||||
commit_hash = os.getenv('DOCKER_COMMIT', os.getenv('CI_COMMIT_SHA', os.getenv('TRAVIS_COMMIT')))
|
||||
commit_hash = os.getenv('DOCKER_COMMIT', os.getenv('GITHUB_SHA'))
|
||||
if commit_hash is None:
|
||||
raise ValueError("Commit hash not found in env vars")
|
||||
_check_and_set(build_info, "COMMIT_HASH", commit_hash[:6])
|
||||
@@ -30,8 +30,10 @@ def main():
|
||||
_check_and_set(build_info, "DOCKER_TAG", docker_tag)
|
||||
_check_and_set(build_info, "BUILD", "docker")
|
||||
else:
|
||||
ci_tag = os.getenv('CI_COMMIT_TAG', os.getenv('TRAVIS_TAG'))
|
||||
_check_and_set(build_info, "BUILD", "release" if re.match(r'v\d+\.\d+\.\d+$', str(ci_tag)) else "qa")
|
||||
if re.match(r'refs/tags/v\d+\.\d+\.\d+$', str(os.getenv('GITHUB_REF'))):
|
||||
_check_and_set(build_info, "BUILD", "release")
|
||||
else:
|
||||
_check_and_set(build_info, "BUILD", "qa")
|
||||
|
||||
log.debug("build info: %s", ", ".join([f"{k}={v}" for k, v in build_info.items()]))
|
||||
with open(build_info_mod.__file__, 'w') as f:
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
allowed_origin: "*"
|
||||
max_key_fee: "0.0 USD"
|
||||
save_files: false
|
||||
save_blobs: false
|
||||
streaming_server: "0.0.0.0:5280"
|
||||
api: "0.0.0.0:5279"
|
||||
data_dir: /tmp
|
||||
download_dir: /tmp
|
||||
wallet_dir: /tmp
|
||||
+194
-488
File diff suppressed because it is too large.
Load diff
+1
-1
@@ -1,2 +1,2 @@
|
||||
__version__ = "0.92.0"
|
||||
__version__ = "0.105.0"
|
||||
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
|
||||
+34
-22
@@ -1,5 +1,6 @@
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import asyncio
|
||||
import binascii
|
||||
import logging
|
||||
@@ -70,12 +71,16 @@ class AbstractBlob:
|
||||
'writers',
|
||||
'verified',
|
||||
'writing',
|
||||
'readers'
|
||||
'readers',
|
||||
'added_on',
|
||||
'is_mine',
|
||||
]
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None):
|
||||
def __init__(
|
||||
self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None, added_on: typing.Optional[int] = None, is_mine: bool = False,
|
||||
):
|
||||
self.loop = loop
|
||||
self.blob_hash = blob_hash
|
||||
self.length = length
|
||||
@@ -85,6 +90,8 @@ class AbstractBlob:
|
||||
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||
self.writing: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||
self.readers: typing.List[typing.BinaryIO] = []
|
||||
self.added_on = added_on or time.time()
|
||||
self.is_mine = is_mine
|
||||
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise InvalidBlobHashError(blob_hash)
|
||||
@@ -180,20 +187,21 @@ class AbstractBlob:
|
||||
|
||||
@classmethod
|
||||
async def create_from_unencrypted(
|
||||
cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
|
||||
unencrypted: bytes, blob_num: int,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None) -> BlobInfo:
|
||||
cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
|
||||
unencrypted: bytes, blob_num: int, added_on: int, is_mine: bool,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None,
|
||||
) -> BlobInfo:
|
||||
"""
|
||||
Create an encrypted BlobFile from plaintext bytes
|
||||
"""
|
||||
|
||||
blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted)
|
||||
length = len(blob_bytes)
|
||||
blob = cls(loop, blob_hash, length, blob_completed_callback, blob_dir)
|
||||
blob = cls(loop, blob_hash, length, blob_completed_callback, blob_dir, added_on, is_mine)
|
||||
writer = blob.get_blob_writer()
|
||||
writer.write(blob_bytes)
|
||||
await blob.verified.wait()
|
||||
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash)
|
||||
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash, added_on, is_mine)
|
||||
|
||||
def save_verified_blob(self, verified_bytes: bytes):
|
||||
if self.verified.is_set():
|
||||
@@ -248,11 +256,13 @@ class BlobBuffer(AbstractBlob):
|
||||
"""
|
||||
An in-memory only blob
|
||||
"""
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None):
|
||||
def __init__(
|
||||
self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None, added_on: typing.Optional[int] = None, is_mine: bool = False
|
||||
):
|
||||
self._verified_bytes: typing.Optional[BytesIO] = None
|
||||
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
|
||||
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory, added_on, is_mine)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
@@ -289,10 +299,12 @@ class BlobFile(AbstractBlob):
|
||||
"""
|
||||
A blob existing on the local file system
|
||||
"""
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None):
|
||||
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
|
||||
def __init__(
|
||||
self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None, added_on: typing.Optional[int] = None, is_mine: bool = False
|
||||
):
|
||||
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory, added_on, is_mine)
|
||||
if not blob_directory or not os.path.isdir(blob_directory):
|
||||
raise OSError(f"invalid blob directory '{blob_directory}'")
|
||||
self.file_path = os.path.join(self.blob_directory, self.blob_hash)
|
||||
@@ -343,12 +355,12 @@ class BlobFile(AbstractBlob):
|
||||
|
||||
@classmethod
|
||||
async def create_from_unencrypted(
|
||||
cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
|
||||
unencrypted: bytes, blob_num: int,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'],
|
||||
asyncio.Task]] = None) -> BlobInfo:
|
||||
cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
|
||||
unencrypted: bytes, blob_num: int, added_on: float, is_mine: bool,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None
|
||||
) -> BlobInfo:
|
||||
if not blob_dir or not os.path.isdir(blob_dir):
|
||||
raise OSError(f"cannot create blob in directory: '{blob_dir}'")
|
||||
return await super().create_from_unencrypted(
|
||||
loop, blob_dir, key, iv, unencrypted, blob_num, blob_completed_callback
|
||||
loop, blob_dir, key, iv, unencrypted, blob_num, added_on, is_mine, blob_completed_callback
|
||||
)
|
||||
@@ -7,13 +7,19 @@ class BlobInfo:
|
||||
'blob_num',
|
||||
'length',
|
||||
'iv',
|
||||
'added_on',
|
||||
'is_mine'
|
||||
]
|
||||
|
||||
def __init__(self, blob_num: int, length: int, iv: str, blob_hash: typing.Optional[str] = None):
|
||||
def __init__(
|
||||
self, blob_num: int, length: int, iv: str,
|
||||
blob_hash: typing.Optional[str] = None, added_on=0, is_mine=False):
|
||||
self.blob_hash = blob_hash
|
||||
self.blob_num = blob_num
|
||||
self.length = length
|
||||
self.iv = iv
|
||||
self.added_on = added_on
|
||||
self.is_mine = is_mine
|
||||
|
||||
def as_dict(self) -> typing.Dict:
|
||||
d = {
|
||||
|
||||
@@ -36,30 +36,30 @@ class BlobManager:
|
||||
self.config.blob_lru_cache_size)
|
||||
self.connection_manager = ConnectionManager(loop)
|
||||
|
||||
def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None):
|
||||
def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None, is_mine: bool = False):
|
||||
if self.config.save_blobs or (
|
||||
is_valid_blobhash(blob_hash) and os.path.isfile(os.path.join(self.blob_dir, blob_hash))):
|
||||
return BlobFile(
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir, is_mine=is_mine
|
||||
)
|
||||
return BlobBuffer(
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir, is_mine=is_mine
|
||||
)
|
||||
|
||||
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
|
||||
def get_blob(self, blob_hash, length: typing.Optional[int] = None, is_mine: bool = False):
|
||||
if blob_hash in self.blobs:
|
||||
if self.config.save_blobs and isinstance(self.blobs[blob_hash], BlobBuffer):
|
||||
buffer = self.blobs.pop(blob_hash)
|
||||
if blob_hash in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.remove(blob_hash)
|
||||
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
|
||||
self.blobs[blob_hash] = self._get_blob(blob_hash, length, is_mine)
|
||||
if buffer.is_readable():
|
||||
with buffer.reader_context() as reader:
|
||||
self.blobs[blob_hash].write_blob(reader.read())
|
||||
if length and self.blobs[blob_hash].length is None:
|
||||
self.blobs[blob_hash].set_length(length)
|
||||
else:
|
||||
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
|
||||
self.blobs[blob_hash] = self._get_blob(blob_hash, length, is_mine)
|
||||
return self.blobs[blob_hash]
|
||||
|
||||
def is_blob_verified(self, blob_hash: str, length: typing.Optional[int] = None) -> bool:
|
||||
@@ -105,9 +105,13 @@ class BlobManager:
|
||||
if isinstance(blob, BlobFile):
|
||||
if blob.blob_hash not in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.add(blob.blob_hash)
|
||||
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=True))
|
||||
return self.loop.create_task(self.storage.add_blobs(
|
||||
(blob.blob_hash, blob.length, blob.added_on, blob.is_mine), finished=True)
|
||||
)
|
||||
else:
|
||||
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=False))
|
||||
return self.loop.create_task(self.storage.add_blobs(
|
||||
(blob.blob_hash, blob.length, blob.added_on, blob.is_mine), finished=False)
|
||||
)
|
||||
|
||||
def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]:
|
||||
"""Returns of the blobhashes_to_check, which are valid"""
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DiskSpaceManager:
|
||||
|
||||
def __init__(self, config, db, blob_manager, cleaning_interval=30 * 60, analytics=None):
|
||||
self.config = config
|
||||
self.db = db
|
||||
self.blob_manager = blob_manager
|
||||
self.cleaning_interval = cleaning_interval
|
||||
self.running = False
|
||||
self.task = None
|
||||
self.analytics = analytics
|
||||
|
||||
async def get_space_used_bytes(self):
|
||||
return await self.db.get_stored_blob_disk_usage()
|
||||
|
||||
async def get_space_used_mb(self):
|
||||
return int(await self.get_space_used_bytes()/1024.0/1024.0)
|
||||
|
||||
async def clean(self):
|
||||
space_used_bytes = await self.get_space_used_bytes()
|
||||
storage_limit = self.config.blob_storage_limit*1024*1024 if self.config.blob_storage_limit else None
|
||||
if self.analytics:
|
||||
asyncio.create_task(self.analytics.send_disk_space_used(space_used_bytes, storage_limit))
|
||||
if not storage_limit:
|
||||
return 0
|
||||
delete = []
|
||||
available = storage_limit - space_used_bytes
|
||||
if available > 0:
|
||||
return 0
|
||||
for blob_hash, file_size, _ in await self.db.get_stored_blobs(is_mine=False):
|
||||
delete.append(blob_hash)
|
||||
available += file_size
|
||||
if available > 0:
|
||||
break
|
||||
if delete:
|
||||
await self.db.stop_all_files()
|
||||
await self.blob_manager.delete_blobs(delete, delete_from_db=True)
|
||||
return len(delete)
|
||||
|
||||
async def cleaning_loop(self):
|
||||
while self.running:
|
||||
await asyncio.sleep(self.cleaning_interval)
|
||||
await self.clean()
|
||||
|
||||
async def start(self):
|
||||
self.running = True
|
||||
self.task = asyncio.create_task(self.cleaning_loop())
|
||||
self.task.add_done_callback(lambda _: log.info("Stopping blob cleanup service."))
|
||||
|
||||
async def stop(self):
|
||||
if self.running:
|
||||
self.running = False
|
||||
self.task.cancel()
|
||||
+106
-18
@@ -1,8 +1,8 @@
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import typing
|
||||
import logging
|
||||
from typing import List, Dict, Tuple, Union, TypeVar, Generic, Optional
|
||||
from argparse import ArgumentParser
|
||||
from contextlib import contextmanager
|
||||
from appdirs import user_data_dir, user_config_dir
|
||||
@@ -15,7 +15,7 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
NOT_SET = type('NOT_SET', (object,), {}) # pylint: disable=invalid-name
|
||||
T = typing.TypeVar('T')
|
||||
T = TypeVar('T')
|
||||
|
||||
CURRENCIES = {
|
||||
'BTC': {'type': 'crypto'},
|
||||
@@ -24,11 +24,11 @@ CURRENCIES = {
|
||||
}
|
||||
|
||||
|
||||
class Setting(typing.Generic[T]):
|
||||
class Setting(Generic[T]):
|
||||
|
||||
def __init__(self, doc: str, default: typing.Optional[T] = None,
|
||||
previous_names: typing.Optional[typing.List[str]] = None,
|
||||
metavar: typing.Optional[str] = None):
|
||||
def __init__(self, doc: str, default: Optional[T] = None,
|
||||
previous_names: Optional[List[str]] = None,
|
||||
metavar: Optional[str] = None):
|
||||
self.doc = doc
|
||||
self.default = default
|
||||
self.previous_names = previous_names or []
|
||||
@@ -45,7 +45,7 @@ class Setting(typing.Generic[T]):
|
||||
def no_cli_name(self):
|
||||
return f"--no-{self.name.replace('_', '-')}"
|
||||
|
||||
def __get__(self, obj: typing.Optional['BaseConfig'], owner) -> T:
|
||||
def __get__(self, obj: Optional['BaseConfig'], owner) -> T:
|
||||
if obj is None:
|
||||
return self
|
||||
for location in obj.search_order:
|
||||
@@ -53,7 +53,7 @@ class Setting(typing.Generic[T]):
|
||||
return location[self.name]
|
||||
return self.default
|
||||
|
||||
def __set__(self, obj: 'BaseConfig', val: typing.Union[T, NOT_SET]):
|
||||
def __set__(self, obj: 'BaseConfig', val: Union[T, NOT_SET]):
|
||||
if val == NOT_SET:
|
||||
for location in obj.modify_order:
|
||||
if self.name in location:
|
||||
@@ -63,6 +63,18 @@ class Setting(typing.Generic[T]):
|
||||
for location in obj.modify_order:
|
||||
location[self.name] = val
|
||||
|
||||
def is_set(self, obj: 'BaseConfig') -> bool:
|
||||
for location in obj.search_order:
|
||||
if self.name in location:
|
||||
return True
|
||||
return False
|
||||
|
||||
def is_set_to_default(self, obj: 'BaseConfig') -> bool:
|
||||
for location in obj.search_order:
|
||||
if self.name in location:
|
||||
return location[self.name] == self.default
|
||||
return False
|
||||
|
||||
def validate(self, value):
|
||||
raise NotImplementedError()
|
||||
|
||||
@@ -87,7 +99,7 @@ class String(Setting[str]):
|
||||
f"Setting '{self.name}' must be a string."
|
||||
|
||||
# TODO: removes this after pylint starts to understand generics
|
||||
def __get__(self, obj: typing.Optional['BaseConfig'], owner) -> str: # pylint: disable=useless-super-delegation
|
||||
def __get__(self, obj: Optional['BaseConfig'], owner) -> str: # pylint: disable=useless-super-delegation
|
||||
return super().__get__(obj, owner)
|
||||
|
||||
|
||||
@@ -191,7 +203,7 @@ class MaxKeyFee(Setting[dict]):
|
||||
)
|
||||
parser.add_argument(
|
||||
self.no_cli_name,
|
||||
help=f"Disable maximum key fee check.",
|
||||
help="Disable maximum key fee check.",
|
||||
dest=self.name,
|
||||
const=None,
|
||||
action="store_const",
|
||||
@@ -200,7 +212,7 @@ class MaxKeyFee(Setting[dict]):
|
||||
|
||||
|
||||
class StringChoice(String):
|
||||
def __init__(self, doc: str, valid_values: typing.List[str], default: str, *args, **kwargs):
|
||||
def __init__(self, doc: str, valid_values: List[str], default: str, *args, **kwargs):
|
||||
super().__init__(doc, default, *args, **kwargs)
|
||||
if not valid_values:
|
||||
raise ValueError("No valid values provided")
|
||||
@@ -273,6 +285,75 @@ class Strings(ListSetting):
|
||||
f"'{self.name}' must be a string."
|
||||
|
||||
|
||||
class KnownHubsList:
|
||||
|
||||
def __init__(self, config: 'Config' = None, file_name: str = 'known_hubs.yml'):
|
||||
self.file_name = file_name
|
||||
self.path = os.path.join(config.wallet_dir, self.file_name) if config else None
|
||||
self.hubs: Dict[Tuple[str, int], Dict] = {}
|
||||
if self.exists:
|
||||
self.load()
|
||||
|
||||
@property
|
||||
def exists(self):
|
||||
return self.path and os.path.exists(self.path)
|
||||
|
||||
@property
|
||||
def serialized(self) -> Dict[str, Dict]:
|
||||
return {f"{host}:{port}": details for (host, port), details in self.hubs.items()}
|
||||
|
||||
def filter(self, match_none=False, **kwargs):
|
||||
if not kwargs:
|
||||
return self.hubs
|
||||
result = {}
|
||||
for hub, details in self.hubs.items():
|
||||
for key, constraint in kwargs.items():
|
||||
value = details.get(key)
|
||||
if value == constraint or (match_none and value is None):
|
||||
result[hub] = details
|
||||
break
|
||||
return result
|
||||
|
||||
def load(self):
|
||||
if self.path:
|
||||
with open(self.path, 'r') as known_hubs_file:
|
||||
raw = known_hubs_file.read()
|
||||
for hub, details in yaml.safe_load(raw).items():
|
||||
self.set(hub, details)
|
||||
|
||||
def save(self):
|
||||
if self.path:
|
||||
with open(self.path, 'w') as known_hubs_file:
|
||||
known_hubs_file.write(yaml.safe_dump(self.serialized, default_flow_style=False))
|
||||
|
||||
def set(self, hub: str, details: Dict):
|
||||
if hub and hub.count(':') == 1:
|
||||
host, port = hub.split(':')
|
||||
hub_parts = (host, int(port))
|
||||
if hub_parts not in self.hubs:
|
||||
self.hubs[hub_parts] = details
|
||||
return hub
|
||||
|
||||
def add_hubs(self, hubs: List[str]):
|
||||
added = False
|
||||
for hub in hubs:
|
||||
if self.set(hub, {}) is not None:
|
||||
added = True
|
||||
return added
|
||||
|
||||
def items(self):
|
||||
return self.hubs.items()
|
||||
|
||||
def __bool__(self):
|
||||
return len(self) > 0
|
||||
|
||||
def __len__(self):
|
||||
return self.hubs.__len__()
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self.hubs)
|
||||
|
||||
|
||||
class EnvironmentAccess:
|
||||
PREFIX = 'LBRY_'
|
||||
|
||||
@@ -377,7 +458,7 @@ class ConfigFileAccess:
|
||||
del self.data[key]
|
||||
|
||||
|
||||
TBC = typing.TypeVar('TBC', bound='BaseConfig')
|
||||
TBC = TypeVar('TBC', bound='BaseConfig')
|
||||
|
||||
|
||||
class BaseConfig:
|
||||
@@ -508,6 +589,9 @@ class CLIConfig(TranscodeConfig):
|
||||
|
||||
|
||||
class Config(CLIConfig):
|
||||
|
||||
jurisdiction = String("Limit interactions to wallet server in this jurisdiction.")
|
||||
|
||||
# directories
|
||||
data_dir = Path("Directory path to store blobs.", metavar='DIR')
|
||||
download_dir = Path(
|
||||
@@ -544,11 +628,13 @@ class Config(CLIConfig):
|
||||
# protocol timeouts
|
||||
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
|
||||
blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
|
||||
hub_timeout = Float("Timeout when making a hub request", 30.0)
|
||||
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0)
|
||||
node_rpc_timeout = Float("Timeout when making a DHT request", constants.RPC_TIMEOUT)
|
||||
|
||||
# blob announcement and download
|
||||
save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True)
|
||||
blob_storage_limit = Integer("Disk space in MB to be allocated for blob storage. 0 = no limit", 0)
|
||||
blob_lru_cache_size = Integer(
|
||||
"LRU cache size for decrypted downloaded blobs used to minimize re-downloading the same blobs when "
|
||||
"replying to a range request. Set to 0 to disable.", 32
|
||||
@@ -565,6 +651,7 @@ class Config(CLIConfig):
|
||||
"Maximum number of peers to connect to while downloading a blob", 4,
|
||||
previous_names=['max_connections_per_stream']
|
||||
)
|
||||
concurrent_hub_requests = Integer("Maximum number of concurrent hub requests", 32)
|
||||
fixed_peer_delay = Float(
|
||||
"Amount of seconds before adding the reflector servers as potential peers to download from in case dht"
|
||||
"peers are not found or are slow", 2.0
|
||||
@@ -605,19 +692,19 @@ class Config(CLIConfig):
|
||||
('spv19.lbry.com', 50001),
|
||||
])
|
||||
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
|
||||
('dht.lbry.grin.io', 4444), # Grin
|
||||
('dht.lbry.madiator.com', 4444), # Madiator
|
||||
('lbrynet1.lbry.com', 4444), # US EAST
|
||||
('lbrynet2.lbry.com', 4444), # US WEST
|
||||
('lbrynet3.lbry.com', 4444), # EU
|
||||
('lbrynet4.lbry.com', 4444) # ASIA
|
||||
])
|
||||
|
||||
comment_server = String("Comment server API URL", "https://comments.lbry.com/api/v2")
|
||||
|
||||
# blockchain
|
||||
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
|
||||
|
||||
# daemon
|
||||
save_files = Toggle("Save downloaded files when calling `get` by default", True)
|
||||
save_files = Toggle("Save downloaded files when calling `get` by default", False)
|
||||
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
|
||||
share_usage_data = Toggle(
|
||||
"Whether to share usage stats and diagnostic info with LBRY.", False,
|
||||
@@ -655,6 +742,7 @@ class Config(CLIConfig):
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.set_default_paths()
|
||||
self.known_hubs = KnownHubsList(self)
|
||||
|
||||
def set_default_paths(self):
|
||||
if 'darwin' in sys.platform.lower():
|
||||
@@ -676,7 +764,7 @@ class Config(CLIConfig):
|
||||
return os.path.join(self.data_dir, 'lbrynet.log')
|
||||
|
||||
|
||||
def get_windows_directories() -> typing.Tuple[str, str, str]:
|
||||
def get_windows_directories() -> Tuple[str, str, str]:
|
||||
from lbry.winpaths import get_path, FOLDERID, UserHandle, \
|
||||
PathNotFoundException # pylint: disable=import-outside-toplevel
|
||||
|
||||
@@ -698,14 +786,14 @@ def get_windows_directories() -> typing.Tuple[str, str, str]:
|
||||
return data_dir, lbryum_dir, download_dir
|
||||
|
||||
|
||||
def get_darwin_directories() -> typing.Tuple[str, str, str]:
|
||||
def get_darwin_directories() -> Tuple[str, str, str]:
|
||||
data_dir = user_data_dir('LBRY')
|
||||
lbryum_dir = os.path.expanduser('~/.lbryum')
|
||||
download_dir = os.path.expanduser('~/Downloads')
|
||||
return data_dir, lbryum_dir, download_dir
|
||||
|
||||
|
||||
def get_linux_directories() -> typing.Tuple[str, str, str]:
|
||||
def get_linux_directories() -> Tuple[str, str, str]:
|
||||
try:
|
||||
with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg:
|
||||
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read())
|
||||
|
||||
+3
-4
@@ -1,7 +1,6 @@
|
||||
import logging
|
||||
import asyncio
|
||||
import typing
|
||||
import binascii
|
||||
import socket
|
||||
from lbry.utils import resolve_host
|
||||
from lbry.dht import constants
|
||||
@@ -80,7 +79,7 @@ class Node:
|
||||
await fut
|
||||
|
||||
async def announce_blob(self, blob_hash: str) -> typing.List[bytes]:
|
||||
hash_value = binascii.unhexlify(blob_hash.encode())
|
||||
hash_value = bytes.fromhex(blob_hash)
|
||||
assert len(hash_value) == constants.HASH_LENGTH
|
||||
peers = await self.peer_search(hash_value)
|
||||
|
||||
@@ -95,7 +94,7 @@ class Node:
|
||||
stored_to = [node_id for node_id, contacted in stored_to_tup if contacted]
|
||||
if stored_to:
|
||||
log.debug(
|
||||
"Stored %s to %i of %i attempted peers", binascii.hexlify(hash_value).decode()[:8],
|
||||
"Stored %s to %i of %i attempted peers", hash_value.hex()[:8],
|
||||
len(stored_to), len(peers)
|
||||
)
|
||||
else:
|
||||
@@ -223,7 +222,7 @@ class Node:
|
||||
# prioritize peers who reply to a dht ping first
|
||||
# this minimizes attempting to make tcp connections that won't work later to dead or unreachable peers
|
||||
|
||||
async for results in self.get_iterative_value_finder(binascii.unhexlify(blob_hash.encode())):
|
||||
async for results in self.get_iterative_value_finder(bytes.fromhex(blob_hash)):
|
||||
to_put = []
|
||||
for peer in results:
|
||||
if peer.address == self.protocol.external_ip and self.protocol.peer_port == peer.tcp_port:
|
||||
|
||||
+12
-13
@@ -1,10 +1,9 @@
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
from binascii import hexlify
|
||||
from dataclasses import dataclass, field
|
||||
from functools import lru_cache
|
||||
from lbry.utils import is_valid_public_ipv4 as _is_valid_public_ipv4
|
||||
from lbry.utils import is_valid_public_ipv4 as _is_valid_public_ipv4, LRUCache
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
|
||||
|
||||
@@ -31,12 +30,12 @@ class PeerManager:
|
||||
self._rpc_failures: typing.Dict[
|
||||
typing.Tuple[str, int], typing.Tuple[typing.Optional[float], typing.Optional[float]]
|
||||
] = {}
|
||||
self._last_replied: typing.Dict[typing.Tuple[str, int], float] = {}
|
||||
self._last_sent: typing.Dict[typing.Tuple[str, int], float] = {}
|
||||
self._last_requested: typing.Dict[typing.Tuple[str, int], float] = {}
|
||||
self._node_id_mapping: typing.Dict[typing.Tuple[str, int], bytes] = {}
|
||||
self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = {}
|
||||
self._node_tokens: typing.Dict[bytes, (float, bytes)] = {}
|
||||
self._last_replied: typing.Dict[typing.Tuple[str, int], float] = LRUCache(2048)
|
||||
self._last_sent: typing.Dict[typing.Tuple[str, int], float] = LRUCache(2048)
|
||||
self._last_requested: typing.Dict[typing.Tuple[str, int], float] = LRUCache(2048)
|
||||
self._node_id_mapping: typing.Dict[typing.Tuple[str, int], bytes] = LRUCache(2048)
|
||||
self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = LRUCache(2048)
|
||||
self._node_tokens: typing.Dict[bytes, (float, bytes)] = LRUCache(2048)
|
||||
|
||||
def reset(self):
|
||||
for statistic in (self._rpc_failures, self._last_replied, self._last_sent, self._last_requested):
|
||||
@@ -154,11 +153,11 @@ class KademliaPeer:
|
||||
def __post_init__(self):
|
||||
if self._node_id is not None:
|
||||
if not len(self._node_id) == constants.HASH_LENGTH:
|
||||
raise ValueError("invalid node_id: {}".format(hexlify(self._node_id).decode()))
|
||||
if self.udp_port is not None and not 1 <= self.udp_port <= 65535:
|
||||
raise ValueError("invalid udp port")
|
||||
if self.tcp_port is not None and not 1 <= self.tcp_port <= 65535:
|
||||
raise ValueError("invalid tcp port")
|
||||
raise ValueError("invalid node_id: {}".format(self._node_id.hex()))
|
||||
if self.udp_port is not None and not 1024 <= self.udp_port <= 65535:
|
||||
raise ValueError(f"invalid udp port: {self.address}:{self.udp_port}")
|
||||
if self.tcp_port is not None and not 1024 <= self.tcp_port <= 65535:
|
||||
raise ValueError(f"invalid tcp port: {self.address}:{self.tcp_port}")
|
||||
if not is_valid_public_ipv4(self.address, self.allow_localhost):
|
||||
raise ValueError(f"invalid ip address: '{self.address}'")
|
||||
|
||||
|
||||
@@ -16,6 +16,9 @@ class DictDataStore:
|
||||
self._peer_manager = peer_manager
|
||||
self.completed_blobs: typing.Set[str] = set()
|
||||
|
||||
def __len__(self):
|
||||
return self._data_store.__len__()
|
||||
|
||||
def removed_expired_peers(self):
|
||||
now = self.loop.time()
|
||||
keys = list(self._data_store.keys())
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import asyncio
|
||||
from binascii import hexlify
|
||||
from itertools import chain
|
||||
from collections import defaultdict
|
||||
import typing
|
||||
@@ -198,7 +197,7 @@ class IterativeFinder:
|
||||
added += 1
|
||||
log.debug("running %d probes", len(self.running_probes))
|
||||
if not added and not self.running_probes:
|
||||
log.debug("search for %s exhausted", hexlify(self.key)[:8])
|
||||
log.debug("search for %s exhausted", self.key.hex()[:8])
|
||||
self.search_exhausted()
|
||||
|
||||
def _schedule_probe(self, peer: 'KademliaPeer'):
|
||||
@@ -271,7 +270,7 @@ class IterativeNodeFinder(IterativeFinder):
|
||||
self.yielded_peers: typing.Set['KademliaPeer'] = set()
|
||||
|
||||
async def send_probe(self, peer: 'KademliaPeer') -> FindNodeResponse:
|
||||
log.debug("probing %s:%d %s", peer.address, peer.udp_port, hexlify(peer.node_id)[:8] if peer.node_id else '')
|
||||
log.debug("probing %s:%d %s", peer.address, peer.udp_port, peer.node_id.hex()[:8] if peer.node_id else '')
|
||||
response = await self.protocol.get_rpc_peer(peer).find_node(self.key)
|
||||
return FindNodeResponse(self.key, response)
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@ import functools
|
||||
import hashlib
|
||||
import asyncio
|
||||
import typing
|
||||
import binascii
|
||||
import random
|
||||
from asyncio.protocols import DatagramProtocol
|
||||
from asyncio.transports import DatagramTransport
|
||||
@@ -97,7 +96,7 @@ class KademliaRPC:
|
||||
if not rpc_contact.tcp_port or peer.compact_address_tcp() != rpc_contact.compact_address_tcp()
|
||||
]
|
||||
# if we don't have k storing peers to return and we have this hash locally, include our contact information
|
||||
if len(peers) < constants.K and binascii.hexlify(key).decode() in self.protocol.data_store.completed_blobs:
|
||||
if len(peers) < constants.K and key.hex() in self.protocol.data_store.completed_blobs:
|
||||
peers.append(self.compact_address())
|
||||
if not peers:
|
||||
response[PAGE_KEY] = 0
|
||||
@@ -415,8 +414,8 @@ class KademliaProtocol(DatagramProtocol):
|
||||
self._wakeup_routing_task.clear()
|
||||
|
||||
def _handle_rpc(self, sender_contact: 'KademliaPeer', message: RequestDatagram):
|
||||
assert sender_contact.node_id != self.node_id, (binascii.hexlify(sender_contact.node_id)[:8].decode(),
|
||||
binascii.hexlify(self.node_id)[:8].decode())
|
||||
assert sender_contact.node_id != self.node_id, (sender_contact.node_id.hex()[:8],
|
||||
self.node_id.hex()[:8])
|
||||
method = message.method
|
||||
if method not in [b'ping', b'store', b'findNode', b'findValue']:
|
||||
raise AttributeError('Invalid method: %s' % message.method.decode())
|
||||
@@ -452,7 +451,11 @@ class KademliaProtocol(DatagramProtocol):
|
||||
try:
|
||||
peer = self.routing_table.get_peer(request_datagram.node_id)
|
||||
except IndexError:
|
||||
peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
try:
|
||||
peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
except ValueError as err:
|
||||
log.warning("error replying to %s: %s", address[0], str(err))
|
||||
return
|
||||
try:
|
||||
self._handle_rpc(peer, request_datagram)
|
||||
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
|
||||
@@ -552,12 +555,12 @@ class KademliaProtocol(DatagramProtocol):
|
||||
address[0], address[1], OLD_PROTOCOL_ERRORS[error_datagram.response]
|
||||
)
|
||||
|
||||
def datagram_received(self, datagram: bytes, address: typing.Tuple[str, int]) -> None: # pylint: disable=arguments-differ
|
||||
def datagram_received(self, datagram: bytes, address: typing.Tuple[str, int]) -> None: # pylint: disable=arguments-renamed
|
||||
try:
|
||||
message = decode_datagram(datagram)
|
||||
except (ValueError, TypeError, DecodeError):
|
||||
self.peer_manager.report_failure(address[0], address[1])
|
||||
log.warning("Couldn't decode dht datagram from %s: %s", address, binascii.hexlify(datagram).decode())
|
||||
log.warning("Couldn't decode dht datagram from %s: %s", address, datagram.hex())
|
||||
return
|
||||
|
||||
if isinstance(message, RequestDatagram):
|
||||
@@ -599,7 +602,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
if len(data) > constants.MSG_SIZE_LIMIT:
|
||||
log.warning("cannot send datagram larger than %i bytes (packet is %i bytes)",
|
||||
constants.MSG_SIZE_LIMIT, len(data))
|
||||
log.debug("Packet is too large to send: %s", binascii.hexlify(data[:3500]).decode())
|
||||
log.debug("Packet is too large to send: %s", data[:3500].hex())
|
||||
raise ValueError(
|
||||
f"cannot send datagram larger than {constants.MSG_SIZE_LIMIT} bytes (packet is {len(data)} bytes)"
|
||||
)
|
||||
@@ -659,13 +662,13 @@ class KademliaProtocol(DatagramProtocol):
|
||||
res = await self.get_rpc_peer(peer).store(hash_value)
|
||||
if res != b"OK":
|
||||
raise ValueError(res)
|
||||
log.debug("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
log.debug("Stored %s to %s", hash_value.hex()[:8], peer)
|
||||
return peer.node_id, True
|
||||
|
||||
try:
|
||||
return await __store()
|
||||
except asyncio.TimeoutError:
|
||||
log.debug("Timeout while storing blob_hash %s at %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
log.debug("Timeout while storing blob_hash %s at %s", hash_value.hex()[:8], peer)
|
||||
return peer.node_id, False
|
||||
except ValueError as err:
|
||||
log.error("Unexpected response: %s", err)
|
||||
|
||||
@@ -50,7 +50,7 @@ class KBucket:
|
||||
self.peers.append(peer)
|
||||
return True
|
||||
else:
|
||||
for i in range(len(self.peers)):
|
||||
for i, _ in enumerate(self.peers):
|
||||
local_peer = self.peers[i]
|
||||
if local_peer.node_id == peer.node_id:
|
||||
self.peers.remove(local_peer)
|
||||
|
||||
@@ -181,7 +181,7 @@ def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDa
|
||||
def make_compact_ip(address: str) -> bytearray:
|
||||
compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), address.split('.'), bytearray())
|
||||
if len(compact_ip) != 4:
|
||||
raise ValueError(f"invalid IPv4 length")
|
||||
raise ValueError("invalid IPv4 length")
|
||||
return compact_ip
|
||||
|
||||
|
||||
@@ -190,7 +190,7 @@ def make_compact_address(node_id: bytes, address: str, port: int) -> bytearray:
|
||||
if not 0 < port < 65536:
|
||||
raise ValueError(f'Invalid port: {port}')
|
||||
if len(node_id) != constants.HASH_BITS // 8:
|
||||
raise ValueError(f"invalid node node_id length")
|
||||
raise ValueError("invalid node node_id length")
|
||||
return compact_ip + port.to_bytes(2, 'big') + node_id
|
||||
|
||||
|
||||
@@ -201,5 +201,5 @@ def decode_compact_address(compact_address: bytes) -> typing.Tuple[bytes, str, i
|
||||
if not 0 < port < 65536:
|
||||
raise ValueError(f'Invalid port: {port}')
|
||||
if len(node_id) != constants.HASH_BITS // 8:
|
||||
raise ValueError(f"invalid node node_id length")
|
||||
raise ValueError("invalid node node_id length")
|
||||
return node_id, address, port
|
||||
+12
-1
@@ -34,6 +34,10 @@ Code | Name | Message
|
||||
**11x** | InputValue(ValueError) | Invalid argument value provided to command.
|
||||
111 | GenericInputValue | The value '{value}' for argument '{argument}' is not valid.
|
||||
112 | InputValueIsNone | None or null is not valid value for argument '{argument}'.
|
||||
113 | ConflictingInputValue | Only '{first_argument}' or '{second_argument}' is allowed, not both.
|
||||
114 | InputStringIsBlank | {argument} cannot be blank.
|
||||
115 | EmptyPublishedFile | Cannot publish empty file: {file_path}
|
||||
116 | MissingPublishedFile | File does not exist: {file_path}
|
||||
**2xx** | Configuration | Configuration errors.
|
||||
201 | ConfigWrite | Cannot write configuration file '{path}'. -- When writing the default config fails on startup, such as due to permission issues.
|
||||
202 | ConfigRead | Cannot find provided configuration file '{path}'. -- Can't open the config file user provided via command line args.
|
||||
@@ -51,15 +55,22 @@ Code | Name | Message
|
||||
405 | ChannelKeyNotFound | Channel signing key not found.
|
||||
406 | ChannelKeyInvalid | Channel signing key is out of date. -- For example, channel was updated but you don't have the updated key.
|
||||
407 | DataDownload | Failed to download blob. *generic*
|
||||
408 | PrivateKeyNotFound | Couldn't find private key for {key} '{value}'.
|
||||
410 | Resolve | Failed to resolve '{url}'.
|
||||
411 | ResolveTimeout | Failed to resolve '{url}' within the timeout.
|
||||
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{claim_id(censor_hash)}'.
|
||||
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{censor_id}'.
|
||||
420 | KeyFeeAboveMaxAllowed | {message}
|
||||
421 | InvalidPassword | Password is invalid.
|
||||
422 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version.
|
||||
423 | TooManyClaimSearchParameters | {key} cant have more than {limit} items.
|
||||
424 | AlreadyPurchased | You already have a purchase for claim_id '{claim_id_hex}'. Use --allow-duplicate-purchase flag to override.
|
||||
431 | ServerPaymentInvalidAddress | Invalid address from wallet server: '{address}' - skipping payment round.
|
||||
432 | ServerPaymentWalletLocked | Cannot spend funds with locked wallet, skipping payment round.
|
||||
433 | ServerPaymentFeeAboveMaxAllowed | Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.
|
||||
434 | WalletNotLoaded | Wallet {wallet_id} is not loaded.
|
||||
435 | WalletAlreadyLoaded | Wallet {wallet_path} is already loaded.
|
||||
436 | WalletNotFound | Wallet not found at {wallet_path}.
|
||||
437 | WalletAlreadyExists | Wallet {wallet_path} already exists, use `wallet_add` to load it.
|
||||
**5xx** | Blob | **Blobs**
|
||||
500 | BlobNotFound | Blob not found.
|
||||
501 | BlobPermissionDenied | Permission denied to read blob.
|
||||
|
||||
+86
-3
@@ -76,6 +76,35 @@ class InputValueIsNoneError(InputValueError):
|
||||
super().__init__(f"None or null is not valid value for argument '{argument}'.")
|
||||
|
||||
|
||||
class ConflictingInputValueError(InputValueError):
|
||||
|
||||
def __init__(self, first_argument, second_argument):
|
||||
self.first_argument = first_argument
|
||||
self.second_argument = second_argument
|
||||
super().__init__(f"Only '{first_argument}' or '{second_argument}' is allowed, not both.")
|
||||
|
||||
|
||||
class InputStringIsBlankError(InputValueError):
|
||||
|
||||
def __init__(self, argument):
|
||||
self.argument = argument
|
||||
super().__init__(f"{argument} cannot be blank.")
|
||||
|
||||
|
||||
class EmptyPublishedFileError(InputValueError):
|
||||
|
||||
def __init__(self, file_path):
|
||||
self.file_path = file_path
|
||||
super().__init__(f"Cannot publish empty file: {file_path}")
|
||||
|
||||
|
||||
class MissingPublishedFileError(InputValueError):
|
||||
|
||||
def __init__(self, file_path):
|
||||
self.file_path = file_path
|
||||
super().__init__(f"File does not exist: {file_path}")
|
||||
|
||||
|
||||
class ConfigurationError(BaseError):
|
||||
"""
|
||||
Configuration errors.
|
||||
@@ -199,6 +228,14 @@ class DataDownloadError(WalletError):
|
||||
super().__init__("Failed to download blob. *generic*")
|
||||
|
||||
|
||||
class PrivateKeyNotFoundError(WalletError):
|
||||
|
||||
def __init__(self, key, value):
|
||||
self.key = key
|
||||
self.value = value
|
||||
super().__init__(f"Couldn't find private key for {key} '{value}'.")
|
||||
|
||||
|
||||
class ResolveError(WalletError):
|
||||
|
||||
def __init__(self, url):
|
||||
@@ -215,10 +252,10 @@ class ResolveTimeoutError(WalletError):
|
||||
|
||||
class ResolveCensoredError(WalletError):
|
||||
|
||||
def __init__(self, url, censor_hash):
|
||||
def __init__(self, url, censor_id):
|
||||
self.url = url
|
||||
self.censor_hash = censor_hash
|
||||
super().__init__(f"Resolve of '{url}' was censored by channel with claim id '{claim_id(censor_hash)}'.")
|
||||
self.censor_id = censor_id
|
||||
super().__init__(f"Resolve of '{url}' was censored by channel with claim id '{censor_id}'.")
|
||||
|
||||
|
||||
class KeyFeeAboveMaxAllowedError(WalletError):
|
||||
@@ -242,6 +279,24 @@ class IncompatibleWalletServerError(WalletError):
|
||||
super().__init__(f"'{server}:{port}' has an incompatibly old version.")
|
||||
|
||||
|
||||
class TooManyClaimSearchParametersError(WalletError):
|
||||
|
||||
def __init__(self, key, limit):
|
||||
self.key = key
|
||||
self.limit = limit
|
||||
super().__init__(f"{key} cant have more than {limit} items.")
|
||||
|
||||
|
||||
class AlreadyPurchasedError(WalletError):
|
||||
"""
|
||||
allow-duplicate-purchase flag to override.
|
||||
"""
|
||||
|
||||
def __init__(self, claim_id_hex):
|
||||
self.claim_id_hex = claim_id_hex
|
||||
super().__init__(f"You already have a purchase for claim_id '{claim_id_hex}'. Use")
|
||||
|
||||
|
||||
class ServerPaymentInvalidAddressError(WalletError):
|
||||
|
||||
def __init__(self, address):
|
||||
@@ -263,6 +318,34 @@ class ServerPaymentFeeAboveMaxAllowedError(WalletError):
|
||||
super().__init__(f"Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.")
|
||||
|
||||
|
||||
class WalletNotLoadedError(WalletError):
|
||||
|
||||
def __init__(self, wallet_id):
|
||||
self.wallet_id = wallet_id
|
||||
super().__init__(f"Wallet {wallet_id} is not loaded.")
|
||||
|
||||
|
||||
class WalletAlreadyLoadedError(WalletError):
|
||||
|
||||
def __init__(self, wallet_path):
|
||||
self.wallet_path = wallet_path
|
||||
super().__init__(f"Wallet {wallet_path} is already loaded.")
|
||||
|
||||
|
||||
class WalletNotFoundError(WalletError):
|
||||
|
||||
def __init__(self, wallet_path):
|
||||
self.wallet_path = wallet_path
|
||||
super().__init__(f"Wallet not found at {wallet_path}.")
|
||||
|
||||
|
||||
class WalletAlreadyExistsError(WalletError):
|
||||
|
||||
def __init__(self, wallet_path):
|
||||
self.wallet_path = wallet_path
|
||||
super().__init__(f"Wallet {wallet_path} already exists, use `wallet_add` to load it.")
|
||||
|
||||
|
||||
class BlobError(BaseError):
|
||||
"""
|
||||
**Blobs**
|
||||
|
||||
@@ -63,7 +63,7 @@ class ErrorClass:
|
||||
@staticmethod
|
||||
def get_fields(args):
|
||||
if len(args) > 1:
|
||||
return f''.join(f'\n{INDENT*2}self.{field} = {field}' for field in args[1:])
|
||||
return ''.join(f'\n{INDENT*2}self.{field} = {field}' for field in args[1:])
|
||||
return ''
|
||||
|
||||
@staticmethod
|
||||
|
||||
+1
-1
@@ -101,7 +101,7 @@ class ArgumentParser(argparse.ArgumentParser):
|
||||
self._optionals.title = 'Options'
|
||||
if group_name is None:
|
||||
self.epilog = (
|
||||
f"Run 'lbrynet COMMAND --help' for more information on a command or group."
|
||||
"Run 'lbrynet COMMAND --help' for more information on a command or group."
|
||||
)
|
||||
else:
|
||||
self.epilog = (
|
||||
|
||||
@@ -18,6 +18,7 @@ DOWNLOAD_STARTED = 'Download Started'
|
||||
DOWNLOAD_ERRORED = 'Download Errored'
|
||||
DOWNLOAD_FINISHED = 'Download Finished'
|
||||
HEARTBEAT = 'Heartbeat'
|
||||
DISK_SPACE = 'Disk Space'
|
||||
CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon
|
||||
NEW_CHANNEL = 'New Channel'
|
||||
CREDITS_SENT = 'Credits Sent'
|
||||
@@ -169,6 +170,14 @@ class AnalyticsManager:
|
||||
})
|
||||
)
|
||||
|
||||
async def send_disk_space_used(self, storage_used, storage_limit):
|
||||
await self.track(
|
||||
self._event(DISK_SPACE, {
|
||||
'used': storage_used,
|
||||
'limit': storage_limit,
|
||||
})
|
||||
)
|
||||
|
||||
async def send_server_startup(self):
|
||||
await self.track(self._event(SERVER_STARTUP))
|
||||
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
import logging
|
||||
import time
|
||||
import hashlib
|
||||
import binascii
|
||||
|
||||
import ecdsa
|
||||
from lbry import utils
|
||||
from lbry.crypto.hash import sha256
|
||||
from lbry.wallet.transaction import Output
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_encoded_signature(signature):
|
||||
signature = signature.encode() if isinstance(signature, str) else signature
|
||||
r = int(signature[:int(len(signature) / 2)], 16)
|
||||
s = int(signature[int(len(signature) / 2):], 16)
|
||||
return ecdsa.util.sigencode_der(r, s, len(signature) * 4)
|
||||
|
||||
|
||||
def cid2hash(claim_id: str) -> bytes:
|
||||
return binascii.unhexlify(claim_id.encode())[::-1]
|
||||
|
||||
|
||||
def is_comment_signed_by_channel(comment: dict, channel: Output, sign_comment_id=False):
|
||||
if isinstance(channel, Output):
|
||||
try:
|
||||
signing_field = comment['comment_id'] if sign_comment_id else comment['comment']
|
||||
return verify(channel, signing_field.encode(), comment, cid2hash(comment['channel_id']))
|
||||
except KeyError:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def verify(channel, data, signature, channel_hash=None):
|
||||
pieces = [
|
||||
signature['signing_ts'].encode(),
|
||||
channel_hash or channel.claim_hash,
|
||||
data
|
||||
]
|
||||
return Output.is_signature_valid(
|
||||
get_encoded_signature(signature['signature']),
|
||||
sha256(b''.join(pieces)),
|
||||
channel.claim.channel.public_key_bytes
|
||||
)
|
||||
|
||||
|
||||
def sign_comment(comment: dict, channel: Output, sign_comment_id=False):
|
||||
signing_field = comment['comment_id'] if sign_comment_id else comment['comment']
|
||||
comment.update(sign(channel, signing_field.encode()))
|
||||
|
||||
|
||||
def sign(channel, data):
|
||||
timestamp = str(int(time.time()))
|
||||
pieces = [timestamp.encode(), channel.claim_hash, data]
|
||||
digest = sha256(b''.join(pieces))
|
||||
signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
|
||||
return {
|
||||
'signature': binascii.hexlify(signature).decode(),
|
||||
'signing_ts': timestamp
|
||||
}
|
||||
|
||||
|
||||
def sign_reaction(reaction: dict, channel: Output):
|
||||
signing_field = reaction['channel_name']
|
||||
reaction.update(sign(channel, signing_field.encode()))
|
||||
|
||||
|
||||
async def jsonrpc_post(url: str, method: str, params: dict = None, **kwargs) -> any:
|
||||
params = params or {}
|
||||
params.update(kwargs)
|
||||
json_body = {'jsonrpc': '2.0', 'id': 1, 'method': method, 'params': params}
|
||||
async with utils.aiohttp_request('POST', url, json=json_body) as response:
|
||||
try:
|
||||
result = await response.json()
|
||||
return result['result'] if 'result' in result else result
|
||||
except Exception as cte:
|
||||
log.exception('Unable to decode response from server: %s', cte)
|
||||
return await response.text()
|
||||
@@ -15,6 +15,7 @@ from lbry.dht.node import Node
|
||||
from lbry.dht.peer import is_valid_public_ipv4
|
||||
from lbry.dht.blob_announcer import BlobAnnouncer
|
||||
from lbry.blob.blob_manager import BlobManager
|
||||
from lbry.blob.disk_space_manager import DiskSpaceManager
|
||||
from lbry.blob_exchange.server import BlobServer
|
||||
from lbry.stream.stream_manager import StreamManager
|
||||
from lbry.file.file_manager import FileManager
|
||||
@@ -40,6 +41,7 @@ WALLET_SERVER_PAYMENTS_COMPONENT = "wallet_server_payments"
|
||||
DHT_COMPONENT = "dht"
|
||||
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
|
||||
FILE_MANAGER_COMPONENT = "file_manager"
|
||||
DISK_SPACE_COMPONENT = "disk_space"
|
||||
PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
|
||||
UPNP_COMPONENT = "upnp"
|
||||
EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager"
|
||||
@@ -59,7 +61,7 @@ class DatabaseComponent(Component):
|
||||
|
||||
@staticmethod
|
||||
def get_current_db_revision():
|
||||
return 14
|
||||
return 15
|
||||
|
||||
@property
|
||||
def revision_filename(self):
|
||||
@@ -138,7 +140,7 @@ class WalletComponent(Component):
|
||||
'availability': session.available,
|
||||
} for session in sessions
|
||||
],
|
||||
'known_servers': len(self.wallet_manager.ledger.network.config['default_servers']),
|
||||
'known_servers': len(self.wallet_manager.ledger.network.known_hubs),
|
||||
'available_servers': 1 if is_connected else 0
|
||||
}
|
||||
|
||||
@@ -375,6 +377,39 @@ class FileManagerComponent(Component):
|
||||
self.file_manager.stop()
|
||||
|
||||
|
||||
class DiskSpaceComponent(Component):
|
||||
component_name = DISK_SPACE_COMPONENT
|
||||
depends_on = [DATABASE_COMPONENT, BLOB_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.disk_space_manager: typing.Optional[DiskSpaceManager] = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[DiskSpaceManager]:
|
||||
return self.disk_space_manager
|
||||
|
||||
async def get_status(self):
|
||||
if self.disk_space_manager:
|
||||
return {
|
||||
'space_used': str(await self.disk_space_manager.get_space_used_mb()),
|
||||
'running': self.disk_space_manager.running,
|
||||
}
|
||||
return {'space_used': '0', 'running': False}
|
||||
|
||||
async def start(self):
|
||||
db = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
self.disk_space_manager = DiskSpaceManager(
|
||||
self.conf, db, blob_manager,
|
||||
analytics=self.component_manager.analytics_manager
|
||||
)
|
||||
await self.disk_space_manager.start()
|
||||
|
||||
async def stop(self):
|
||||
await self.disk_space_manager.stop()
|
||||
|
||||
|
||||
class TorrentComponent(Component):
|
||||
component_name = LIBTORRENT_COMPONENT
|
||||
|
||||
@@ -481,6 +516,10 @@ class UPnPComponent(Component):
|
||||
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
|
||||
if external_ip:
|
||||
self.external_ip = external_ip
|
||||
dht_component = self.component_manager.get_component(DHT_COMPONENT)
|
||||
if dht_component:
|
||||
dht_node = dht_component.component
|
||||
dht_node.protocol.external_ip = external_ip
|
||||
# assert self.external_ip is not None # TODO: handle going/starting offline
|
||||
|
||||
if not self.upnp_redirects and self.upnp: # setup missing redirects
|
||||
@@ -539,8 +578,10 @@ class UPnPComponent(Component):
|
||||
success = False
|
||||
await self._maintain_redirects()
|
||||
if self.upnp:
|
||||
if not self.upnp_redirects and not all([x in self.component_manager.skip_components for x in
|
||||
(DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT)]):
|
||||
if not self.upnp_redirects and not all(
|
||||
x in self.component_manager.skip_components
|
||||
for x in (DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT)
|
||||
):
|
||||
log.error("failed to setup upnp")
|
||||
else:
|
||||
success = True
|
||||
|
||||
+253
-497
File diff suppressed because it is too large.
Load diff
@@ -5,7 +5,7 @@ import logging
|
||||
from statistics import median
|
||||
from decimal import Decimal
|
||||
from typing import Optional, Iterable, Type
|
||||
from aiohttp.client_exceptions import ContentTypeError
|
||||
from aiohttp.client_exceptions import ContentTypeError, ClientConnectionError
|
||||
from lbry.error import InvalidExchangeRateResponseError, CurrencyConversionError
|
||||
from lbry.utils import aiohttp_request
|
||||
from lbry.wallet.dewies import lbc_to_dewies
|
||||
@@ -79,18 +79,23 @@ class MarketFeed:
|
||||
log.debug("Saving rate update %f for %s from %s", rate, self.market, self.name)
|
||||
self.rate = ExchangeRate(self.market, rate, int(time.time()))
|
||||
self.last_check = time.time()
|
||||
self.event.set()
|
||||
return self.rate
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("Timed out fetching exchange rate from %s.", self.name)
|
||||
except json.JSONDecodeError as e:
|
||||
log.warning("Could not parse exchange rate response from %s: %s", self.name, e.doc)
|
||||
msg = e.doc if '<html>' not in e.doc else 'unexpected content type.'
|
||||
log.warning("Could not parse exchange rate response from %s: %s", self.name, msg)
|
||||
log.debug(e.doc)
|
||||
except InvalidExchangeRateResponseError as e:
|
||||
log.warning(str(e))
|
||||
except ClientConnectionError as e:
|
||||
log.warning("Error trying to connect to exchange rate %s: %s", self.name, str(e))
|
||||
except Exception as e:
|
||||
log.exception("Exchange rate error (%s from %s):", self.market, self.name)
|
||||
finally:
|
||||
self.event.set()
|
||||
|
||||
async def keep_updated(self):
|
||||
while True:
|
||||
@@ -130,27 +135,6 @@ class BittrexUSDFeed(BaseBittrexFeed):
|
||||
url = "https://api.bittrex.com/v3/markets/LBC-USD/ticker"
|
||||
|
||||
|
||||
class BaseCryptonatorFeed(MarketFeed):
|
||||
name = "Cryptonator"
|
||||
market = None
|
||||
url = None
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'ticker' not in json_response or 'price' not in json_response['ticker']:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return float(json_response['ticker']['price'])
|
||||
|
||||
|
||||
class CryptonatorBTCFeed(BaseCryptonatorFeed):
|
||||
market = "BTCLBC"
|
||||
url = "https://api.cryptonator.com/api/ticker/btc-lbc"
|
||||
|
||||
|
||||
class CryptonatorUSDFeed(BaseCryptonatorFeed):
|
||||
market = "USDLBC"
|
||||
url = "https://api.cryptonator.com/api/ticker/usd-lbc"
|
||||
|
||||
|
||||
class BaseCoinExFeed(MarketFeed):
|
||||
name = "CoinEx"
|
||||
market = None
|
||||
@@ -202,7 +186,7 @@ class UPbitBTCFeed(MarketFeed):
|
||||
params = {"markets": "BTC-LBC"}
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if len(json_response) != 1 or 'trade_price' not in json_response[0]:
|
||||
if "error" in json_response or len(json_response) != 1 or 'trade_price' not in json_response[0]:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return 1.0 / float(json_response[0]['trade_price'])
|
||||
|
||||
@@ -210,8 +194,6 @@ class UPbitBTCFeed(MarketFeed):
|
||||
FEEDS: Iterable[Type[MarketFeed]] = (
|
||||
BittrexBTCFeed,
|
||||
BittrexUSDFeed,
|
||||
CryptonatorBTCFeed,
|
||||
CryptonatorUSDFeed,
|
||||
CoinExBTCFeed,
|
||||
CoinExUSDFeed,
|
||||
HotbitBTCFeed,
|
||||
|
||||
@@ -123,7 +123,7 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
self.ledger = ledger
|
||||
self.include_protobuf = include_protobuf
|
||||
|
||||
def default(self, obj): # pylint: disable=method-hidden,arguments-differ,too-many-return-statements
|
||||
def default(self, obj): # pylint: disable=method-hidden,arguments-renamed,too-many-return-statements
|
||||
if isinstance(obj, Account):
|
||||
return self.encode_account(obj)
|
||||
if isinstance(obj, Wallet):
|
||||
@@ -234,8 +234,6 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
output['value_type'] = txo.claim.claim_type
|
||||
if txo.claim.is_channel:
|
||||
output['has_signing_key'] = txo.has_private_key
|
||||
elif txo.script.is_support_claim_data:
|
||||
output['value_type'] = 'emoji'
|
||||
if check_signature and txo.signable.is_signed:
|
||||
if txo.channel is not None:
|
||||
output['signing_channel'] = self.encode_output(txo.channel)
|
||||
|
||||
@@ -35,6 +35,8 @@ def migrate_db(conf, start, end):
|
||||
from .migrate12to13 import do_migration
|
||||
elif current == 13:
|
||||
from .migrate13to14 import do_migration
|
||||
elif current == 14:
|
||||
from .migrate14to15 import do_migration
|
||||
else:
|
||||
raise Exception(f"DB migration of version {current} to {current+1} is not available")
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
import os
|
||||
import sqlite3
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
cursor.executescript("""
|
||||
alter table blob add column added_on integer not null default 0;
|
||||
alter table blob add column is_mine integer not null default 1;
|
||||
""")
|
||||
|
||||
connection.commit()
|
||||
connection.close()
|
||||
@@ -170,8 +170,8 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
|
||||
def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'):
|
||||
# add all blobs, except the last one, which is empty
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
((blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
((blob.blob_hash, blob.length, 0, 0, "pending", 0, 0, blob.added_on, blob.is_mine)
|
||||
for blob in (descriptor.blobs[:-1] if len(descriptor.blobs) > 1 else descriptor.blobs) + [sd_blob])
|
||||
).fetchall()
|
||||
# associate the blobs to the stream
|
||||
@@ -242,7 +242,9 @@ class SQLiteStorage(SQLiteMixin):
|
||||
should_announce integer not null default 0,
|
||||
status text not null,
|
||||
last_announced_time integer,
|
||||
single_announce integer
|
||||
single_announce integer,
|
||||
added_on integer not null,
|
||||
is_mine integer not null default 0
|
||||
);
|
||||
|
||||
create table if not exists stream (
|
||||
@@ -356,19 +358,19 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
# # # # # # # # # blob functions # # # # # # # # #
|
||||
|
||||
async def add_blobs(self, *blob_hashes_and_lengths: typing.Tuple[str, int], finished=False):
|
||||
async def add_blobs(self, *blob_hashes_and_lengths: typing.Tuple[str, int, int, int], finished=False):
|
||||
def _add_blobs(transaction: sqlite3.Connection):
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
(blob_hash, length, 0, 0, "pending" if not finished else "finished", 0, 0)
|
||||
for blob_hash, length in blob_hashes_and_lengths
|
||||
(blob_hash, length, 0, 0, "pending" if not finished else "finished", 0, 0, added_on, is_mine)
|
||||
for blob_hash, length, added_on, is_mine in blob_hashes_and_lengths
|
||||
)
|
||||
).fetchall()
|
||||
if finished:
|
||||
transaction.executemany(
|
||||
"update blob set status='finished' where blob.blob_hash=?", (
|
||||
(blob_hash, ) for blob_hash, _ in blob_hashes_and_lengths
|
||||
(blob_hash, ) for blob_hash, _, _, _ in blob_hashes_and_lengths
|
||||
)
|
||||
).fetchall()
|
||||
return await self.db.run(_add_blobs)
|
||||
@@ -435,6 +437,29 @@ class SQLiteStorage(SQLiteMixin):
|
||||
def get_all_blob_hashes(self):
|
||||
return self.run_and_return_list("select blob_hash from blob")
|
||||
|
||||
async def get_stored_blobs(self, is_mine: bool):
|
||||
is_mine = 1 if is_mine else 0
|
||||
return await self.db.execute_fetchall(
|
||||
"select blob_hash, blob_length, added_on from blob where is_mine=? order by added_on asc",
|
||||
(is_mine,)
|
||||
)
|
||||
|
||||
async def get_stored_blob_disk_usage(self, is_mine: Optional[bool] = None):
|
||||
if is_mine is None:
|
||||
sql, args = "select coalesce(sum(blob_length), 0) from blob", ()
|
||||
else:
|
||||
is_mine = 1 if is_mine else 0
|
||||
sql, args = "select coalesce(sum(blob_length), 0) from blob where is_mine=?", (is_mine,)
|
||||
return (await self.db.execute_fetchone(sql, args))[0]
|
||||
|
||||
async def update_blob_ownership(self, sd_hash, is_mine: bool):
|
||||
is_mine = 1 if is_mine else 0
|
||||
await self.db.execute_fetchall(
|
||||
"update blob set is_mine = ? where blob_hash in ("
|
||||
" select blob_hash from blob natural join stream_blob natural join stream where sd_hash = ?"
|
||||
") OR blob_hash = ?", (is_mine, sd_hash, sd_hash)
|
||||
)
|
||||
|
||||
def sync_missing_blobs(self, blob_files: typing.Set[str]) -> typing.Awaitable[typing.Set[str]]:
|
||||
def _sync_blobs(transaction: sqlite3.Connection) -> typing.Set[str]:
|
||||
finished_blob_hashes = tuple(
|
||||
@@ -570,6 +595,10 @@ class SQLiteStorage(SQLiteMixin):
|
||||
log.debug("update file status %s -> %s", stream_hash, new_status)
|
||||
return self.db.execute_fetchall("update file set status=? where stream_hash=?", (new_status, stream_hash))
|
||||
|
||||
def stop_all_files(self):
|
||||
log.debug("stopping all files")
|
||||
return self.db.execute_fetchall("update file set status=?", ("stopped",))
|
||||
|
||||
async def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: typing.Optional[str],
|
||||
file_name: typing.Optional[str]):
|
||||
if not file_name or not download_dir:
|
||||
@@ -617,7 +646,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
).fetchall()
|
||||
download_dir = binascii.hexlify(self.conf.download_dir.encode()).decode()
|
||||
transaction.executemany(
|
||||
f"update file set download_directory=? where stream_hash=?",
|
||||
"update file set download_directory=? where stream_hash=?",
|
||||
((download_dir, stream_hash) for stream_hash in stream_hashes)
|
||||
).fetchall()
|
||||
await self.db.run_with_foreign_keys_disabled(_recover)
|
||||
@@ -861,6 +890,6 @@ class SQLiteStorage(SQLiteMixin):
|
||||
transaction.execute('delete from peer').fetchall()
|
||||
transaction.executemany(
|
||||
'insert into peer(node_id, address, udp_port, tcp_port) values (?, ?, ?, ?)',
|
||||
tuple([(binascii.hexlify(p.node_id), p.address, p.udp_port, p.tcp_port) for p in peers])
|
||||
((binascii.hexlify(p.node_id), p.address, p.udp_port, p.tcp_port) for p in peers)
|
||||
).fetchall()
|
||||
return await self.db.run(_save_kademlia_peers)
|
||||
@@ -132,7 +132,7 @@ class SourceManager:
|
||||
else:
|
||||
streams = list(self._sources.values())
|
||||
if sort_by:
|
||||
streams.sort(key=lambda s: getattr(s, sort_by))
|
||||
streams.sort(key=lambda s: getattr(s, sort_by) or "")
|
||||
if reverse:
|
||||
streams.reverse()
|
||||
return streams
|
||||
@@ -69,8 +69,8 @@ class VideoFileAnalyzer:
|
||||
version = str(e)
|
||||
if code != 0 or not version.startswith("ffmpeg"):
|
||||
log.warning("Unable to run ffmpeg, but it was requested. Code: %d; Message: %s", code, version)
|
||||
raise FileNotFoundError(f"Unable to locate or run ffmpeg or ffprobe. Please install FFmpeg "
|
||||
f"and ensure that it is callable via PATH or conf.ffmpeg_path")
|
||||
raise FileNotFoundError("Unable to locate or run ffmpeg or ffprobe. Please install FFmpeg "
|
||||
"and ensure that it is callable via PATH or conf.ffmpeg_path")
|
||||
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which_ffmpeg)
|
||||
return version
|
||||
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
Schema
|
||||
=====
|
||||
|
||||
Those files are generated from the [types repo](https://github.com/lbryio/types). If you are modifying/adding a new type, make sure it is cloned in the same root folder as the SDK repo, like:
|
||||
|
||||
```
|
||||
repos/
|
||||
- lbry-sdk/
|
||||
- types/
|
||||
```
|
||||
|
||||
Then, [download protoc 3.2.0](https://github.com/protocolbuffers/protobuf/releases/tag/v3.2.0), add it to your PATH. On linux it is:
|
||||
|
||||
```bash
|
||||
cd ~/.local/bin
|
||||
wget https://github.com/protocolbuffers/protobuf/releases/download/v3.2.0/protoc-3.2.0-linux-x86_64.zip
|
||||
unzip protoc-3.2.0-linux-x86_64.zip bin/protoc -d..
|
||||
```
|
||||
|
||||
Finally, `make` should update everything in place.
|
||||
|
||||
|
||||
### Why protoc 3.2.0?
|
||||
Different/newer versions will generate larger diffs and we need to make sure they are good. In theory, we can just update to latest and it will all work, but it is a good practice to check blockchain data and retro compatibility before bumping versions (if you do, please update this section!).
|
||||
+16
-7
@@ -10,6 +10,7 @@ from google.protobuf.json_format import MessageToDict
|
||||
|
||||
from lbry.crypto.base58 import Base58
|
||||
from lbry.constants import COIN
|
||||
from lbry.error import MissingPublishedFileError, EmptyPublishedFileError
|
||||
|
||||
from lbry.schema.mime_types import guess_media_type
|
||||
from lbry.schema.base import Metadata, BaseMessageList
|
||||
@@ -32,6 +33,17 @@ def calculate_sha384_file_hash(file_path):
|
||||
return sha384.digest()
|
||||
|
||||
|
||||
def country_int_to_str(country: int) -> str:
|
||||
r = LocationMessage.Country.Name(country)
|
||||
return r[1:] if r.startswith('R') else r
|
||||
|
||||
|
||||
def country_str_to_int(country: str) -> int:
|
||||
if len(country) == 3:
|
||||
country = 'R' + country
|
||||
return LocationMessage.Country.Value(country)
|
||||
|
||||
|
||||
class Dimmensional(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
@@ -128,10 +140,10 @@ class Source(Metadata):
|
||||
self.name = os.path.basename(file_path)
|
||||
self.media_type, stream_type = guess_media_type(file_path)
|
||||
if not os.path.isfile(file_path):
|
||||
raise Exception(f"File does not exist: {file_path}")
|
||||
raise MissingPublishedFileError(file_path)
|
||||
self.size = os.path.getsize(file_path)
|
||||
if self.size == 0:
|
||||
raise Exception(f"Cannot publish empty file: {file_path}")
|
||||
raise EmptyPublishedFileError(file_path)
|
||||
self.file_hash_bytes = calculate_sha384_file_hash(file_path)
|
||||
return stream_type
|
||||
|
||||
@@ -423,14 +435,11 @@ class Language(Metadata):
|
||||
@property
|
||||
def region(self) -> str:
|
||||
if self.message.region:
|
||||
r = LocationMessage.Country.Name(self.message.region)
|
||||
return r[1:] if r.startswith('R') else r
|
||||
return country_int_to_str(self.message.region)
|
||||
|
||||
@region.setter
|
||||
def region(self, region: str):
|
||||
if len(region) == 3:
|
||||
region = 'R'+region
|
||||
self.message.region = LocationMessage.Country.Value(region)
|
||||
self.message.region = country_str_to_int(region)
|
||||
|
||||
|
||||
class LanguageList(BaseMessageList[Language]):
|
||||
|
||||
+15
-2
@@ -40,7 +40,7 @@ class Censor:
|
||||
|
||||
def censor(self, row) -> bool:
|
||||
if self.is_censored(row):
|
||||
censoring_channel_hash = row['censoring_channel_hash']
|
||||
censoring_channel_hash = bytes.fromhex(row['censoring_channel_id'])[::-1]
|
||||
self.censored.setdefault(censoring_channel_hash, set())
|
||||
self.censored[censoring_channel_hash].add(row['tx_hash'])
|
||||
return True
|
||||
@@ -150,6 +150,19 @@ class Outputs:
|
||||
outputs.blocked, outputs.blocked_total
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_grpc(cls, outputs: OutputsMessage) -> 'Outputs':
|
||||
txs = set()
|
||||
for txo_message in chain(outputs.txos, outputs.extra_txos):
|
||||
if txo_message.WhichOneof('meta') == 'error':
|
||||
continue
|
||||
txs.add((hexlify(txo_message.tx_hash[::-1]).decode(), txo_message.height))
|
||||
return cls(
|
||||
outputs.txos, outputs.extra_txos, txs,
|
||||
outputs.offset, outputs.total,
|
||||
outputs.blocked, outputs.blocked_total
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def to_base64(cls, txo_rows, extra_txo_rows, offset=0, total=None, blocked=None) -> str:
|
||||
return base64.b64encode(cls.to_bytes(txo_rows, extra_txo_rows, offset, total, blocked)).decode()
|
||||
@@ -179,7 +192,7 @@ class Outputs:
|
||||
txo_message.error.code = ErrorMessage.NOT_FOUND
|
||||
elif isinstance(txo, ResolveCensoredError):
|
||||
txo_message.error.code = ErrorMessage.BLOCKED
|
||||
set_reference(txo_message.error.blocked.channel, extra_row_dict.get(txo.censor_hash))
|
||||
set_reference(txo_message.error.blocked.channel, extra_row_dict.get(bytes.fromhex(txo.censor_id)[::-1]))
|
||||
return
|
||||
txo_message.tx_hash = txo['txo_hash'][:32]
|
||||
txo_message.nout, = struct.unpack('<I', txo['txo_hash'][32:])
|
||||
|
||||
@@ -13,3 +13,11 @@ class Support(Signable):
|
||||
@emoji.setter
|
||||
def emoji(self, emoji: str):
|
||||
self.message.emoji = emoji
|
||||
|
||||
@property
|
||||
def comment(self) -> str:
|
||||
return self.message.comment
|
||||
|
||||
@comment.setter
|
||||
def comment(self, comment: str):
|
||||
self.message.comment = comment
|
||||
@@ -0,0 +1,740 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: hub.proto
|
||||
"""Generated protocol buffer code."""
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import message as _message
|
||||
from google.protobuf import reflection as _reflection
|
||||
from google.protobuf import symbol_database as _symbol_database
|
||||
# @@protoc_insertion_point(imports)
|
||||
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
import lbry.schema.types.v2.result_pb2 as result__pb2
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='hub.proto',
|
||||
package='pb',
|
||||
syntax='proto3',
|
||||
serialized_options=b'Z$github.com/lbryio/hub/protobuf/go/pb',
|
||||
create_key=_descriptor._internal_create_key,
|
||||
serialized_pb=b'\n\thub.proto\x12\x02pb\x1a\x0cresult.proto\"0\n\x0fInvertibleField\x12\x0e\n\x06invert\x18\x01 \x01(\x08\x12\r\n\x05value\x18\x02 \x03(\t\"\x1a\n\tBoolValue\x12\r\n\x05value\x18\x01 \x01(\x08\"\x1c\n\x0bUInt32Value\x12\r\n\x05value\x18\x01 \x01(\r\"j\n\nRangeField\x12\x1d\n\x02op\x18\x01 \x01(\x0e\x32\x11.pb.RangeField.Op\x12\r\n\x05value\x18\x02 \x03(\t\".\n\x02Op\x12\x06\n\x02\x45Q\x10\x00\x12\x07\n\x03LTE\x10\x01\x12\x07\n\x03GTE\x10\x02\x12\x06\n\x02LT\x10\x03\x12\x06\n\x02GT\x10\x04\"\x9c\r\n\rSearchRequest\x12%\n\x08\x63laim_id\x18\x01 \x01(\x0b\x32\x13.pb.InvertibleField\x12\'\n\nchannel_id\x18\x02 \x01(\x0b\x32\x13.pb.InvertibleField\x12\x0c\n\x04text\x18\x03 \x01(\t\x12\r\n\x05limit\x18\x04 \x01(\r\x12\x10\n\x08order_by\x18\x05 \x03(\t\x12\x0e\n\x06offset\x18\x06 \x01(\r\x12\x16\n\x0eis_controlling\x18\x07 \x01(\x08\x12\x1d\n\x15last_take_over_height\x18\x08 \x01(\t\x12\x12\n\nclaim_name\x18\t \x01(\t\x12\x17\n\x0fnormalized_name\x18\n \x01(\t\x12#\n\x0btx_position\x18\x0b \x01(\x0b\x32\x0e.pb.RangeField\x12\x1e\n\x06\x61mount\x18\x0c \x01(\x0b\x32\x0e.pb.RangeField\x12!\n\ttimestamp\x18\r \x01(\x0b\x32\x0e.pb.RangeField\x12*\n\x12\x63reation_timestamp\x18\x0e \x01(\x0b\x32\x0e.pb.RangeField\x12\x1e\n\x06height\x18\x0f \x01(\x0b\x32\x0e.pb.RangeField\x12\'\n\x0f\x63reation_height\x18\x10 \x01(\x0b\x32\x0e.pb.RangeField\x12)\n\x11\x61\x63tivation_height\x18\x11 \x01(\x0b\x32\x0e.pb.RangeField\x12)\n\x11\x65xpiration_height\x18\x12 \x01(\x0b\x32\x0e.pb.RangeField\x12$\n\x0crelease_time\x18\x13 \x01(\x0b\x32\x0e.pb.RangeField\x12\x11\n\tshort_url\x18\x14 \x01(\t\x12\x15\n\rcanonical_url\x18\x15 \x01(\t\x12\r\n\x05title\x18\x16 \x01(\t\x12\x0e\n\x06\x61uthor\x18\x17 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x18 \x01(\t\x12\x12\n\nclaim_type\x18\x19 \x03(\t\x12$\n\x0crepost_count\x18\x1a \x01(\x0b\x32\x0e.pb.RangeField\x12\x13\n\x0bstream_type\x18\x1b \x03(\t\x12\x12\n\nmedia_type\x18\x1c \x03(\t\x12\"\n\nfee_amount\x18\x1d \x01(\x0b\x32\x0e.pb.RangeField\x12\x14\n\x0c\x66\x65\x65_currency\x18\x1e \x01(\t\x12 \n\x08\x64uration\x18\x1f \x01(\x0b\x32\x0e.pb.RangeField\x12\x19\n\x11reposted_claim_id\x18 \x01(\t\x12#\n\x0b\x63\x65nsor_type\x18! \x01(\x0b\x32\x0e.pb.RangeField\x12\x19\n\x11\x63laims_in_channel\x18\" \x01(\t\x12$\n\x0c\x63hannel_join\x18# \x01(\x0b\x32\x0e.pb.RangeField\x12)\n\x12is_signature_valid\x18$ \x01(\x0b\x32\r.pb.BoolValue\x12(\n\x10\x65\x66\x66\x65\x63tive_amount\x18% \x01(\x0b\x32\x0e.pb.RangeField\x12&\n\x0esupport_amount\x18& \x01(\x0b\x32\x0e.pb.RangeField\x12&\n\x0etrending_group\x18\' \x01(\x0b\x32\x0e.pb.RangeField\x12&\n\x0etrending_mixed\x18( \x01(\x0b\x32\x0e.pb.RangeField\x12&\n\x0etrending_local\x18) \x01(\x0b\x32\x0e.pb.RangeField\x12\'\n\x0ftrending_global\x18* \x01(\x0b\x32\x0e.pb.RangeField\x12\r\n\x05tx_id\x18+ \x01(\t\x12 \n\x07tx_nout\x18, \x01(\x0b\x32\x0f.pb.UInt32Value\x12\x11\n\tsignature\x18- \x01(\t\x12\x18\n\x10signature_digest\x18. \x01(\t\x12\x18\n\x10public_key_bytes\x18/ \x01(\t\x12\x15\n\rpublic_key_id\x18\x30 \x01(\t\x12\x10\n\x08\x61ny_tags\x18\x31 \x03(\t\x12\x10\n\x08\x61ll_tags\x18\x32 \x03(\t\x12\x10\n\x08not_tags\x18\x33 \x03(\t\x12\x1d\n\x15has_channel_signature\x18\x34 \x01(\x08\x12!\n\nhas_source\x18\x35 \x01(\x0b\x32\r.pb.BoolValue\x12 \n\x18limit_claims_per_channel\x18\x36 \x01(\r\x12\x15\n\rany_languages\x18\x37 \x03(\t\x12\x15\n\rall_languages\x18\x38 \x03(\t\x12\x19\n\x11remove_duplicates\x18\x39 \x01(\x08\x12\x11\n\tno_totals\x18: \x01(\x08\x32\x31\n\x03Hub\x12*\n\x06Search\x12\x11.pb.SearchRequest\x1a\x0b.pb.Outputs\"\x00\x42&Z$github.com/lbryio/hub/protobuf/go/pbb\x06proto3'
|
||||
,
|
||||
dependencies=[result__pb2.DESCRIPTOR,])
|
||||
|
||||
|
||||
|
||||
_RANGEFIELD_OP = _descriptor.EnumDescriptor(
|
||||
name='Op',
|
||||
full_name='pb.RangeField.Op',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
values=[
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='EQ', index=0, number=0,
|
||||
serialized_options=None,
|
||||
type=None,
|
||||
create_key=_descriptor._internal_create_key),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='LTE', index=1, number=1,
|
||||
serialized_options=None,
|
||||
type=None,
|
||||
create_key=_descriptor._internal_create_key),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='GTE', index=2, number=2,
|
||||
serialized_options=None,
|
||||
type=None,
|
||||
create_key=_descriptor._internal_create_key),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='LT', index=3, number=3,
|
||||
serialized_options=None,
|
||||
type=None,
|
||||
create_key=_descriptor._internal_create_key),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='GT', index=4, number=4,
|
||||
serialized_options=None,
|
||||
type=None,
|
||||
create_key=_descriptor._internal_create_key),
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=199,
|
||||
serialized_end=245,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_RANGEFIELD_OP)
|
||||
|
||||
|
||||
_INVERTIBLEFIELD = _descriptor.Descriptor(
|
||||
name='InvertibleField',
|
||||
full_name='pb.InvertibleField',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='invert', full_name='pb.InvertibleField.invert', index=0,
|
||||
number=1, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='value', full_name='pb.InvertibleField.value', index=1,
|
||||
number=2, type=9, cpp_type=9, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=31,
|
||||
serialized_end=79,
|
||||
)
|
||||
|
||||
|
||||
_BOOLVALUE = _descriptor.Descriptor(
|
||||
name='BoolValue',
|
||||
full_name='pb.BoolValue',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='value', full_name='pb.BoolValue.value', index=0,
|
||||
number=1, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=81,
|
||||
serialized_end=107,
|
||||
)
|
||||
|
||||
|
||||
_UINT32VALUE = _descriptor.Descriptor(
|
||||
name='UInt32Value',
|
||||
full_name='pb.UInt32Value',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='value', full_name='pb.UInt32Value.value', index=0,
|
||||
number=1, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=109,
|
||||
serialized_end=137,
|
||||
)
|
||||
|
||||
|
||||
_RANGEFIELD = _descriptor.Descriptor(
|
||||
name='RangeField',
|
||||
full_name='pb.RangeField',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='op', full_name='pb.RangeField.op', index=0,
|
||||
number=1, type=14, cpp_type=8, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='value', full_name='pb.RangeField.value', index=1,
|
||||
number=2, type=9, cpp_type=9, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
_RANGEFIELD_OP,
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=139,
|
||||
serialized_end=245,
|
||||
)
|
||||
|
||||
|
||||
_SEARCHREQUEST = _descriptor.Descriptor(
|
||||
name='SearchRequest',
|
||||
full_name='pb.SearchRequest',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='claim_id', full_name='pb.SearchRequest.claim_id', index=0,
|
||||
number=1, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='channel_id', full_name='pb.SearchRequest.channel_id', index=1,
|
||||
number=2, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='text', full_name='pb.SearchRequest.text', index=2,
|
||||
number=3, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='limit', full_name='pb.SearchRequest.limit', index=3,
|
||||
number=4, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='order_by', full_name='pb.SearchRequest.order_by', index=4,
|
||||
number=5, type=9, cpp_type=9, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='offset', full_name='pb.SearchRequest.offset', index=5,
|
||||
number=6, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='is_controlling', full_name='pb.SearchRequest.is_controlling', index=6,
|
||||
number=7, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='last_take_over_height', full_name='pb.SearchRequest.last_take_over_height', index=7,
|
||||
number=8, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='claim_name', full_name='pb.SearchRequest.claim_name', index=8,
|
||||
number=9, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='normalized_name', full_name='pb.SearchRequest.normalized_name', index=9,
|
||||
number=10, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='tx_position', full_name='pb.SearchRequest.tx_position', index=10,
|
||||
number=11, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='amount', full_name='pb.SearchRequest.amount', index=11,
|
||||
number=12, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='timestamp', full_name='pb.SearchRequest.timestamp', index=12,
|
||||
number=13, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='creation_timestamp', full_name='pb.SearchRequest.creation_timestamp', index=13,
|
||||
number=14, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='height', full_name='pb.SearchRequest.height', index=14,
|
||||
number=15, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='creation_height', full_name='pb.SearchRequest.creation_height', index=15,
|
||||
number=16, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='activation_height', full_name='pb.SearchRequest.activation_height', index=16,
|
||||
number=17, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='expiration_height', full_name='pb.SearchRequest.expiration_height', index=17,
|
||||
number=18, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='release_time', full_name='pb.SearchRequest.release_time', index=18,
|
||||
number=19, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='short_url', full_name='pb.SearchRequest.short_url', index=19,
|
||||
number=20, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='canonical_url', full_name='pb.SearchRequest.canonical_url', index=20,
|
||||
number=21, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='title', full_name='pb.SearchRequest.title', index=21,
|
||||
number=22, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='author', full_name='pb.SearchRequest.author', index=22,
|
||||
number=23, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='description', full_name='pb.SearchRequest.description', index=23,
|
||||
number=24, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='claim_type', full_name='pb.SearchRequest.claim_type', index=24,
|
||||
number=25, type=9, cpp_type=9, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='repost_count', full_name='pb.SearchRequest.repost_count', index=25,
|
||||
number=26, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='stream_type', full_name='pb.SearchRequest.stream_type', index=26,
|
||||
number=27, type=9, cpp_type=9, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='media_type', full_name='pb.SearchRequest.media_type', index=27,
|
||||
number=28, type=9, cpp_type=9, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='fee_amount', full_name='pb.SearchRequest.fee_amount', index=28,
|
||||
number=29, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='fee_currency', full_name='pb.SearchRequest.fee_currency', index=29,
|
||||
number=30, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='duration', full_name='pb.SearchRequest.duration', index=30,
|
||||
number=31, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='reposted_claim_id', full_name='pb.SearchRequest.reposted_claim_id', index=31,
|
||||
number=32, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='censor_type', full_name='pb.SearchRequest.censor_type', index=32,
|
||||
number=33, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='claims_in_channel', full_name='pb.SearchRequest.claims_in_channel', index=33,
|
||||
number=34, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='channel_join', full_name='pb.SearchRequest.channel_join', index=34,
|
||||
number=35, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='is_signature_valid', full_name='pb.SearchRequest.is_signature_valid', index=35,
|
||||
number=36, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='effective_amount', full_name='pb.SearchRequest.effective_amount', index=36,
|
||||
number=37, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='support_amount', full_name='pb.SearchRequest.support_amount', index=37,
|
||||
number=38, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='trending_group', full_name='pb.SearchRequest.trending_group', index=38,
|
||||
number=39, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='trending_mixed', full_name='pb.SearchRequest.trending_mixed', index=39,
|
||||
number=40, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='trending_local', full_name='pb.SearchRequest.trending_local', index=40,
|
||||
number=41, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='trending_global', full_name='pb.SearchRequest.trending_global', index=41,
|
||||
number=42, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='tx_id', full_name='pb.SearchRequest.tx_id', index=42,
|
||||
number=43, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='tx_nout', full_name='pb.SearchRequest.tx_nout', index=43,
|
||||
number=44, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='signature', full_name='pb.SearchRequest.signature', index=44,
|
||||
number=45, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='signature_digest', full_name='pb.SearchRequest.signature_digest', index=45,
|
||||
number=46, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='public_key_bytes', full_name='pb.SearchRequest.public_key_bytes', index=46,
|
||||
number=47, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='public_key_id', full_name='pb.SearchRequest.public_key_id', index=47,
|
||||
number=48, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=b"".decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='any_tags', full_name='pb.SearchRequest.any_tags', index=48,
|
||||
number=49, type=9, cpp_type=9, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='all_tags', full_name='pb.SearchRequest.all_tags', index=49,
|
||||
number=50, type=9, cpp_type=9, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='not_tags', full_name='pb.SearchRequest.not_tags', index=50,
|
||||
number=51, type=9, cpp_type=9, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='has_channel_signature', full_name='pb.SearchRequest.has_channel_signature', index=51,
|
||||
number=52, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='has_source', full_name='pb.SearchRequest.has_source', index=52,
|
||||
number=53, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='limit_claims_per_channel', full_name='pb.SearchRequest.limit_claims_per_channel', index=53,
|
||||
number=54, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='any_languages', full_name='pb.SearchRequest.any_languages', index=54,
|
||||
number=55, type=9, cpp_type=9, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='all_languages', full_name='pb.SearchRequest.all_languages', index=55,
|
||||
number=56, type=9, cpp_type=9, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='remove_duplicates', full_name='pb.SearchRequest.remove_duplicates', index=56,
|
||||
number=57, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='no_totals', full_name='pb.SearchRequest.no_totals', index=57,
|
||||
number=58, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=248,
|
||||
serialized_end=1940,
|
||||
)
|
||||
|
||||
_RANGEFIELD.fields_by_name['op'].enum_type = _RANGEFIELD_OP
|
||||
_RANGEFIELD_OP.containing_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['claim_id'].message_type = _INVERTIBLEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['channel_id'].message_type = _INVERTIBLEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['tx_position'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['amount'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['timestamp'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['creation_timestamp'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['height'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['creation_height'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['activation_height'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['expiration_height'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['release_time'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['repost_count'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['fee_amount'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['duration'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['censor_type'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['channel_join'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['is_signature_valid'].message_type = _BOOLVALUE
|
||||
_SEARCHREQUEST.fields_by_name['effective_amount'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['support_amount'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['trending_group'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['trending_mixed'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['trending_local'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['trending_global'].message_type = _RANGEFIELD
|
||||
_SEARCHREQUEST.fields_by_name['tx_nout'].message_type = _UINT32VALUE
|
||||
_SEARCHREQUEST.fields_by_name['has_source'].message_type = _BOOLVALUE
|
||||
DESCRIPTOR.message_types_by_name['InvertibleField'] = _INVERTIBLEFIELD
|
||||
DESCRIPTOR.message_types_by_name['BoolValue'] = _BOOLVALUE
|
||||
DESCRIPTOR.message_types_by_name['UInt32Value'] = _UINT32VALUE
|
||||
DESCRIPTOR.message_types_by_name['RangeField'] = _RANGEFIELD
|
||||
DESCRIPTOR.message_types_by_name['SearchRequest'] = _SEARCHREQUEST
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
InvertibleField = _reflection.GeneratedProtocolMessageType('InvertibleField', (_message.Message,), {
|
||||
'DESCRIPTOR' : _INVERTIBLEFIELD,
|
||||
'__module__' : 'hub_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.InvertibleField)
|
||||
})
|
||||
_sym_db.RegisterMessage(InvertibleField)
|
||||
|
||||
BoolValue = _reflection.GeneratedProtocolMessageType('BoolValue', (_message.Message,), {
|
||||
'DESCRIPTOR' : _BOOLVALUE,
|
||||
'__module__' : 'hub_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.BoolValue)
|
||||
})
|
||||
_sym_db.RegisterMessage(BoolValue)
|
||||
|
||||
UInt32Value = _reflection.GeneratedProtocolMessageType('UInt32Value', (_message.Message,), {
|
||||
'DESCRIPTOR' : _UINT32VALUE,
|
||||
'__module__' : 'hub_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.UInt32Value)
|
||||
})
|
||||
_sym_db.RegisterMessage(UInt32Value)
|
||||
|
||||
RangeField = _reflection.GeneratedProtocolMessageType('RangeField', (_message.Message,), {
|
||||
'DESCRIPTOR' : _RANGEFIELD,
|
||||
'__module__' : 'hub_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.RangeField)
|
||||
})
|
||||
_sym_db.RegisterMessage(RangeField)
|
||||
|
||||
SearchRequest = _reflection.GeneratedProtocolMessageType('SearchRequest', (_message.Message,), {
|
||||
'DESCRIPTOR' : _SEARCHREQUEST,
|
||||
'__module__' : 'hub_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.SearchRequest)
|
||||
})
|
||||
_sym_db.RegisterMessage(SearchRequest)
|
||||
|
||||
|
||||
DESCRIPTOR._options = None
|
||||
|
||||
_HUB = _descriptor.ServiceDescriptor(
|
||||
name='Hub',
|
||||
full_name='pb.Hub',
|
||||
file=DESCRIPTOR,
|
||||
index=0,
|
||||
serialized_options=None,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
serialized_start=1942,
|
||||
serialized_end=1991,
|
||||
methods=[
|
||||
_descriptor.MethodDescriptor(
|
||||
name='Search',
|
||||
full_name='pb.Hub.Search',
|
||||
index=0,
|
||||
containing_service=None,
|
||||
input_type=_SEARCHREQUEST,
|
||||
output_type=result__pb2._OUTPUTS,
|
||||
serialized_options=None,
|
||||
create_key=_descriptor._internal_create_key,
|
||||
),
|
||||
])
|
||||
_sym_db.RegisterServiceDescriptor(_HUB)
|
||||
|
||||
DESCRIPTOR.services_by_name['Hub'] = _HUB
|
||||
|
||||
# @@protoc_insertion_point(module_scope)
|
||||
@@ -0,0 +1,67 @@
|
||||
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
|
||||
"""Client and server classes corresponding to protobuf-defined services."""
|
||||
import grpc
|
||||
|
||||
import lbry.schema.types.v2.hub_pb2 as hub__pb2
|
||||
import lbry.schema.types.v2.result_pb2 as result__pb2
|
||||
|
||||
|
||||
class HubStub(object):
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
|
||||
def __init__(self, channel):
|
||||
"""Constructor.
|
||||
|
||||
Args:
|
||||
channel: A grpc.Channel.
|
||||
"""
|
||||
self.Search = channel.unary_unary(
|
||||
'/pb.Hub/Search',
|
||||
request_serializer=hub__pb2.SearchRequest.SerializeToString,
|
||||
response_deserializer=result__pb2.Outputs.FromString,
|
||||
)
|
||||
|
||||
|
||||
class HubServicer(object):
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
|
||||
def Search(self, request, context):
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details('Method not implemented!')
|
||||
raise NotImplementedError('Method not implemented!')
|
||||
|
||||
|
||||
def add_HubServicer_to_server(servicer, server):
|
||||
rpc_method_handlers = {
|
||||
'Search': grpc.unary_unary_rpc_method_handler(
|
||||
servicer.Search,
|
||||
request_deserializer=hub__pb2.SearchRequest.FromString,
|
||||
response_serializer=result__pb2.Outputs.SerializeToString,
|
||||
),
|
||||
}
|
||||
generic_handler = grpc.method_handlers_generic_handler(
|
||||
'pb.Hub', rpc_method_handlers)
|
||||
server.add_generic_rpc_handlers((generic_handler,))
|
||||
|
||||
|
||||
# This class is part of an EXPERIMENTAL API.
|
||||
class Hub(object):
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
|
||||
@staticmethod
|
||||
def Search(request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
call_credentials=None,
|
||||
insecure=False,
|
||||
compression=None,
|
||||
wait_for_ready=None,
|
||||
timeout=None,
|
||||
metadata=None):
|
||||
return grpc.experimental.unary_unary(request, target, '/pb.Hub/Search',
|
||||
hub__pb2.SearchRequest.SerializeToString,
|
||||
result__pb2.Outputs.FromString,
|
||||
options, channel_credentials,
|
||||
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
|
||||
@@ -19,7 +19,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='support.proto',
|
||||
package='pb',
|
||||
syntax='proto3',
|
||||
serialized_pb=_b('\n\rsupport.proto\x12\x02pb\"\x18\n\x07Support\x12\r\n\x05\x65moji\x18\x01 \x01(\tb\x06proto3')
|
||||
serialized_pb=_b('\n\rsupport.proto\x12\x02pb\")\n\x07Support\x12\r\n\x05\x65moji\x18\x01 \x01(\t\x12\x0f\n\x07\x63omment\x18\x02 \x01(\tb\x06proto3')
|
||||
)
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
@@ -40,6 +40,13 @@ _SUPPORT = _descriptor.Descriptor(
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='comment', full_name='pb.Support.comment', index=1,
|
||||
number=2, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
@@ -53,7 +60,7 @@ _SUPPORT = _descriptor.Descriptor(
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=21,
|
||||
serialized_end=45,
|
||||
serialized_end=62,
|
||||
)
|
||||
|
||||
DESCRIPTOR.message_types_by_name['Support'] = _SUPPORT
|
||||
|
||||
@@ -4,6 +4,7 @@ import binascii
|
||||
import logging
|
||||
import typing
|
||||
import asyncio
|
||||
import time
|
||||
import re
|
||||
from collections import OrderedDict
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
@@ -152,15 +153,19 @@ class StreamDescriptor:
|
||||
h.update(self.old_sort_json())
|
||||
return h.hexdigest()
|
||||
|
||||
async def make_sd_blob(self, blob_file_obj: typing.Optional[AbstractBlob] = None,
|
||||
old_sort: typing.Optional[bool] = False,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None):
|
||||
async def make_sd_blob(
|
||||
self, blob_file_obj: typing.Optional[AbstractBlob] = None, old_sort: typing.Optional[bool] = False,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None,
|
||||
added_on: float = None, is_mine: bool = False
|
||||
):
|
||||
sd_hash = self.calculate_sd_hash() if not old_sort else self.calculate_old_sort_sd_hash()
|
||||
if not old_sort:
|
||||
sd_data = self.as_json()
|
||||
else:
|
||||
sd_data = self.old_sort_json()
|
||||
sd_blob = blob_file_obj or BlobFile(self.loop, sd_hash, len(sd_data), blob_completed_callback, self.blob_dir)
|
||||
sd_blob = blob_file_obj or BlobFile(
|
||||
self.loop, sd_hash, len(sd_data), blob_completed_callback, self.blob_dir, added_on, is_mine
|
||||
)
|
||||
if blob_file_obj:
|
||||
blob_file_obj.set_length(len(sd_data))
|
||||
if not sd_blob.get_is_verified():
|
||||
@@ -183,11 +188,11 @@ class StreamDescriptor:
|
||||
raise InvalidStreamDescriptorError("Does not decode as valid JSON")
|
||||
if decoded['blobs'][-1]['length'] != 0:
|
||||
raise InvalidStreamDescriptorError("Does not end with a zero-length blob.")
|
||||
if any([blob_info['length'] == 0 for blob_info in decoded['blobs'][:-1]]):
|
||||
if any(blob_info['length'] == 0 for blob_info in decoded['blobs'][:-1]):
|
||||
raise InvalidStreamDescriptorError("Contains zero-length data blob")
|
||||
if 'blob_hash' in decoded['blobs'][-1]:
|
||||
raise InvalidStreamDescriptorError("Stream terminator blob should not have a hash")
|
||||
if any([i != blob_info['blob_num'] for i, blob_info in enumerate(decoded['blobs'])]):
|
||||
if any(i != blob_info['blob_num'] for i, blob_info in enumerate(decoded['blobs'])):
|
||||
raise InvalidStreamDescriptorError("Stream contains out of order or skipped blobs")
|
||||
descriptor = cls(
|
||||
loop, blob_dir,
|
||||
@@ -252,20 +257,25 @@ class StreamDescriptor:
|
||||
iv_generator = iv_generator or random_iv_generator()
|
||||
key = key or os.urandom(AES.block_size // 8)
|
||||
blob_num = -1
|
||||
added_on = time.time()
|
||||
async for blob_bytes in file_reader(file_path):
|
||||
blob_num += 1
|
||||
blob_info = await BlobFile.create_from_unencrypted(
|
||||
loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num, blob_completed_callback
|
||||
loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num, added_on, True, blob_completed_callback
|
||||
)
|
||||
blobs.append(blob_info)
|
||||
blobs.append(
|
||||
BlobInfo(len(blobs), 0, binascii.hexlify(next(iv_generator)).decode())) # add the stream terminator
|
||||
# add the stream terminator
|
||||
BlobInfo(len(blobs), 0, binascii.hexlify(next(iv_generator)).decode(), None, added_on, True)
|
||||
)
|
||||
file_name = os.path.basename(file_path)
|
||||
suggested_file_name = sanitize_file_name(file_name)
|
||||
descriptor = cls(
|
||||
loop, blob_dir, file_name, binascii.hexlify(key).decode(), suggested_file_name, blobs
|
||||
)
|
||||
sd_blob = await descriptor.make_sd_blob(old_sort=old_sort, blob_completed_callback=blob_completed_callback)
|
||||
sd_blob = await descriptor.make_sd_blob(
|
||||
old_sort=old_sort, blob_completed_callback=blob_completed_callback, added_on=added_on, is_mine=True
|
||||
)
|
||||
descriptor.sd_hash = sd_blob.blob_hash
|
||||
return descriptor
|
||||
|
||||
|
||||
@@ -254,7 +254,7 @@ class ManagedStream(ManagedDownloadSource):
|
||||
self.finished_writing.clear()
|
||||
self.started_writing.clear()
|
||||
try:
|
||||
open(output_path, 'wb').close()
|
||||
open(output_path, 'wb').close() # pylint: disable=consider-using-with
|
||||
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
|
||||
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
|
||||
await self.loop.run_in_executor(None, self._write_decrypted_blob, output_path, decrypted)
|
||||
@@ -372,9 +372,6 @@ class ManagedStream(ManagedDownloadSource):
|
||||
protocol.transport.close()
|
||||
self.uploading_to_reflector = False
|
||||
|
||||
if not self.fully_reflected.is_set():
|
||||
self.fully_reflected.set()
|
||||
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
|
||||
return sent
|
||||
|
||||
async def update_content_claim(self, claim_info: Optional[typing.Dict] = None):
|
||||
|
||||
@@ -59,7 +59,7 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
return
|
||||
|
||||
async def send_request(self, request_dict: typing.Dict, timeout: int = 180):
|
||||
msg = json.dumps(request_dict)
|
||||
msg = json.dumps(request_dict, sort_keys=True)
|
||||
try:
|
||||
self.transport.write(msg.encode())
|
||||
self.pending_request = self.loop.create_task(asyncio.wait_for(self.response_queue.get(), timeout))
|
||||
|
||||
@@ -17,7 +17,7 @@ log = logging.getLogger(__name__)
|
||||
class ReflectorServerProtocol(asyncio.Protocol):
|
||||
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000,
|
||||
stop_event: asyncio.Event = None, incoming_event: asyncio.Event = None,
|
||||
not_incoming_event: asyncio.Event = None):
|
||||
not_incoming_event: asyncio.Event = None, partial_event: asyncio.Event = None):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
@@ -34,6 +34,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
self.stop_event = stop_event or asyncio.Event(loop=self.loop)
|
||||
self.chunk_size = response_chunk_size
|
||||
self.wait_for_stop_task: typing.Optional[asyncio.Task] = None
|
||||
self.partial_event = partial_event
|
||||
|
||||
async def wait_for_stop(self):
|
||||
await self.stop_event.wait()
|
||||
@@ -115,10 +116,14 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
if self.writer:
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
self.send_response({"send_sd_blob": False, 'needed': [
|
||||
blob.blob_hash for blob in self.descriptor.blobs[:-1]
|
||||
if not self.blob_manager.get_blob(blob.blob_hash).get_is_verified()
|
||||
]})
|
||||
|
||||
needs = [blob.blob_hash
|
||||
for blob in self.descriptor.blobs[:-1]
|
||||
if not self.blob_manager.get_blob(blob.blob_hash).get_is_verified()]
|
||||
if needs and not self.partial_event.is_set():
|
||||
needs = needs[:3]
|
||||
self.partial_event.set()
|
||||
self.send_response({"send_sd_blob": False, 'needed_blobs': needs})
|
||||
return
|
||||
return
|
||||
elif self.descriptor:
|
||||
@@ -153,7 +158,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
class ReflectorServer:
|
||||
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000,
|
||||
stop_event: asyncio.Event = None, incoming_event: asyncio.Event = None,
|
||||
not_incoming_event: asyncio.Event = None):
|
||||
not_incoming_event: asyncio.Event = None, partial_needs=False):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: typing.Optional[asyncio.Task] = None
|
||||
@@ -163,19 +168,19 @@ class ReflectorServer:
|
||||
self.not_incoming_event = not_incoming_event or asyncio.Event(loop=self.loop)
|
||||
self.response_chunk_size = response_chunk_size
|
||||
self.stop_event = stop_event
|
||||
self.partial_needs = partial_needs # for testing cases where it doesn't know what it wants
|
||||
|
||||
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
|
||||
if self.server_task is not None:
|
||||
raise Exception("already running")
|
||||
|
||||
async def _start_server():
|
||||
server = await self.loop.create_server(
|
||||
lambda: ReflectorServerProtocol(
|
||||
self.blob_manager, self.response_chunk_size, self.stop_event, self.incoming_event,
|
||||
self.not_incoming_event
|
||||
),
|
||||
interface, port
|
||||
)
|
||||
partial_event = asyncio.Event()
|
||||
if not self.partial_needs:
|
||||
partial_event.set()
|
||||
server = await self.loop.create_server(lambda: ReflectorServerProtocol(
|
||||
self.blob_manager, self.response_chunk_size, self.stop_event, self.incoming_event,
|
||||
self.not_incoming_event, partial_event), interface, port)
|
||||
self.started_listening.set()
|
||||
self.stopped_listening.clear()
|
||||
log.info("Reflector server listening on TCP %s:%i", interface, port)
|
||||
|
||||
@@ -215,7 +215,7 @@ class StreamManager(SourceManager):
|
||||
server, port = random.choice(self.config.reflector_servers)
|
||||
if stream.sd_hash in self.running_reflector_uploads:
|
||||
return self.running_reflector_uploads[stream.sd_hash]
|
||||
task = self.loop.create_task(stream.upload_to_reflector(server, port))
|
||||
task = self.loop.create_task(self._retriable_reflect_stream(stream, server, port))
|
||||
self.running_reflector_uploads[stream.sd_hash] = task
|
||||
task.add_done_callback(
|
||||
lambda _: None if stream.sd_hash not in self.running_reflector_uploads else
|
||||
@@ -223,6 +223,13 @@ class StreamManager(SourceManager):
|
||||
)
|
||||
return task
|
||||
|
||||
async def _retriable_reflect_stream(self, stream, host, port):
|
||||
sent = await stream.upload_to_reflector(host, port)
|
||||
while not stream.is_fully_reflected and stream.reflector_progress > 0 and len(sent) > 0:
|
||||
stream.reflector_progress = 0
|
||||
sent = await stream.upload_to_reflector(host, port)
|
||||
return sent
|
||||
|
||||
async def create(self, file_path: str, key: Optional[bytes] = None,
|
||||
iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
|
||||
descriptor = await StreamDescriptor.create_stream(
|
||||
@@ -230,7 +237,7 @@ class StreamManager(SourceManager):
|
||||
blob_completed_callback=self.blob_manager.blob_completed
|
||||
)
|
||||
await self.storage.store_stream(
|
||||
self.blob_manager.get_blob(descriptor.sd_hash), descriptor
|
||||
self.blob_manager.get_blob(descriptor.sd_hash, is_mine=True), descriptor
|
||||
)
|
||||
row_id = await self.storage.save_published_file(
|
||||
descriptor.stream_hash, os.path.basename(file_path), os.path.dirname(file_path), 0
|
||||
|
||||
+42
-10
@@ -18,7 +18,7 @@ from lbry.wallet import WalletManager, Wallet, Ledger, Account, Transaction
|
||||
from lbry.conf import Config
|
||||
from lbry.wallet.util import satoshis_to_coins
|
||||
from lbry.wallet.orchstr8 import Conductor
|
||||
from lbry.wallet.orchstr8.node import BlockchainNode, WalletNode
|
||||
from lbry.wallet.orchstr8.node import BlockchainNode, WalletNode, HubNode
|
||||
|
||||
from lbry.extras.daemon.daemon import Daemon, jsonrpc_dumps_pretty
|
||||
from lbry.extras.daemon.components import Component, WalletComponent
|
||||
@@ -223,6 +223,7 @@ class IntegrationTestCase(AsyncioTestCase):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.conductor: Optional[Conductor] = None
|
||||
self.blockchain: Optional[BlockchainNode] = None
|
||||
self.hub: Optional[HubNode] = None
|
||||
self.wallet_node: Optional[WalletNode] = None
|
||||
self.manager: Optional[WalletManager] = None
|
||||
self.ledger: Optional[Ledger] = None
|
||||
@@ -237,7 +238,10 @@ class IntegrationTestCase(AsyncioTestCase):
|
||||
self.addCleanup(self.conductor.stop_spv)
|
||||
await self.conductor.start_wallet()
|
||||
self.addCleanup(self.conductor.stop_wallet)
|
||||
await self.conductor.start_hub()
|
||||
self.addCleanup(self.conductor.stop_hub)
|
||||
self.blockchain = self.conductor.blockchain_node
|
||||
self.hub = self.conductor.hub_node
|
||||
self.wallet_node = self.conductor.wallet_node
|
||||
self.manager = self.wallet_node.manager
|
||||
self.ledger = self.wallet_node.ledger
|
||||
@@ -350,7 +354,12 @@ class CommandTestCase(IntegrationTestCase):
|
||||
|
||||
server_tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, server_tmp_dir)
|
||||
self.server_config = Config()
|
||||
self.server_config = Config(
|
||||
data_dir=server_tmp_dir,
|
||||
wallet_dir=server_tmp_dir,
|
||||
save_files=True,
|
||||
download_dir=server_tmp_dir
|
||||
)
|
||||
self.server_config.transaction_cache_size = 10000
|
||||
self.server_storage = SQLiteStorage(self.server_config, ':memory:')
|
||||
await self.server_storage.open()
|
||||
@@ -374,6 +383,7 @@ class CommandTestCase(IntegrationTestCase):
|
||||
await daemon.stop()
|
||||
|
||||
async def add_daemon(self, wallet_node=None, seed=None):
|
||||
start_wallet_node = False
|
||||
if wallet_node is None:
|
||||
wallet_node = WalletNode(
|
||||
self.wallet_node.manager_class,
|
||||
@@ -381,22 +391,24 @@ class CommandTestCase(IntegrationTestCase):
|
||||
port=self.extra_wallet_node_port
|
||||
)
|
||||
self.extra_wallet_node_port += 1
|
||||
await wallet_node.start(self.conductor.spv_node, seed=seed)
|
||||
self.extra_wallet_nodes.append(wallet_node)
|
||||
start_wallet_node = True
|
||||
|
||||
upload_dir = os.path.join(wallet_node.data_path, 'uploads')
|
||||
os.mkdir(upload_dir)
|
||||
|
||||
conf = Config()
|
||||
conf.data_dir = wallet_node.data_path
|
||||
conf.wallet_dir = wallet_node.data_path
|
||||
conf.download_dir = wallet_node.data_path
|
||||
conf = Config(
|
||||
# needed during instantiation to access known_hubs path
|
||||
data_dir=wallet_node.data_path,
|
||||
wallet_dir=wallet_node.data_path,
|
||||
save_files=True,
|
||||
download_dir=wallet_node.data_path
|
||||
)
|
||||
conf.upload_dir = upload_dir # not a real conf setting
|
||||
conf.share_usage_data = False
|
||||
conf.use_upnp = False
|
||||
conf.reflect_streams = True
|
||||
conf.blockchain_name = 'lbrycrd_regtest'
|
||||
conf.lbryum_servers = [('127.0.0.1', 50001)]
|
||||
conf.lbryum_servers = [(self.conductor.spv_node.hostname, self.conductor.spv_node.port)]
|
||||
conf.reflector_servers = [('127.0.0.1', 5566)]
|
||||
conf.fixed_peers = [('127.0.0.1', 5567)]
|
||||
conf.known_dht_nodes = []
|
||||
@@ -408,7 +420,13 @@ class CommandTestCase(IntegrationTestCase):
|
||||
]
|
||||
if self.skip_libtorrent:
|
||||
conf.components_to_skip.append(LIBTORRENT_COMPONENT)
|
||||
wallet_node.manager.config = conf
|
||||
|
||||
if start_wallet_node:
|
||||
await wallet_node.start(self.conductor.spv_node, seed=seed, config=conf)
|
||||
self.extra_wallet_nodes.append(wallet_node)
|
||||
else:
|
||||
wallet_node.manager.config = conf
|
||||
wallet_node.manager.ledger.config['known_hubs'] = conf.known_hubs
|
||||
|
||||
def wallet_maker(component_manager):
|
||||
wallet_component = WalletComponent(component_manager)
|
||||
@@ -577,6 +595,11 @@ class CommandTestCase(IntegrationTestCase):
|
||||
self.daemon.jsonrpc_support_abandon(*args, **kwargs), confirm
|
||||
)
|
||||
|
||||
async def account_send(self, *args, confirm=True, **kwargs):
|
||||
return await self.confirm_and_render(
|
||||
self.daemon.jsonrpc_account_send(*args, **kwargs), confirm
|
||||
)
|
||||
|
||||
async def wallet_send(self, *args, confirm=True, **kwargs):
|
||||
return await self.confirm_and_render(
|
||||
self.daemon.jsonrpc_wallet_send(*args, **kwargs), confirm
|
||||
@@ -590,6 +613,12 @@ class CommandTestCase(IntegrationTestCase):
|
||||
await asyncio.wait([self.ledger.wait(tx, self.blockchain.block_expected) for tx in txs])
|
||||
return self.sout(txs)
|
||||
|
||||
async def blob_clean(self):
|
||||
return await self.out(self.daemon.jsonrpc_blob_clean())
|
||||
|
||||
async def status(self):
|
||||
return await self.out(self.daemon.jsonrpc_status())
|
||||
|
||||
async def resolve(self, uri, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_resolve(uri, **kwargs)))[uri]
|
||||
|
||||
@@ -620,6 +649,9 @@ class CommandTestCase(IntegrationTestCase):
|
||||
async def transaction_list(self, *args, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_transaction_list(*args, **kwargs)))['items']
|
||||
|
||||
async def blob_list(self, *args, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_blob_list(*args, **kwargs)))['items']
|
||||
|
||||
@staticmethod
|
||||
def get_claim_id(tx):
|
||||
return tx['outputs'][0]['claim_id']
|
||||
|
||||
+12
-14
@@ -21,7 +21,6 @@ import pkg_resources
|
||||
import certifi
|
||||
import aiohttp
|
||||
from prometheus_client import Counter
|
||||
from prometheus_client.registry import REGISTRY
|
||||
from lbry.schema.claim import Claim
|
||||
|
||||
|
||||
@@ -156,7 +155,7 @@ def async_timed_cache(duration: int):
|
||||
async def _inner(*args, **kwargs) -> typing.Any:
|
||||
loop = asyncio.get_running_loop()
|
||||
time_now = loop.time()
|
||||
key = tuple([args, tuple([tuple([k, kwargs[k]]) for k in kwargs])])
|
||||
key = (args, tuple(kwargs.items()))
|
||||
if key in cache and (time_now - cache[key][1] < duration):
|
||||
return cache[key][0]
|
||||
to_cache = await func(*args, **kwargs)
|
||||
@@ -174,7 +173,7 @@ def cache_concurrent(async_fn):
|
||||
|
||||
@functools.wraps(async_fn)
|
||||
async def wrapper(*args, **kwargs):
|
||||
key = tuple([args, tuple([tuple([k, kwargs[k]]) for k in kwargs])])
|
||||
key = (args, tuple(kwargs.items()))
|
||||
cache[key] = cache.get(key) or asyncio.create_task(async_fn(*args, **kwargs))
|
||||
try:
|
||||
return await cache[key]
|
||||
@@ -276,12 +275,6 @@ class LRUCacheWithMetrics:
|
||||
|
||||
def __del__(self):
|
||||
self.clear()
|
||||
if self._track_metrics: # needed for tests
|
||||
try:
|
||||
REGISTRY.unregister(self.hits)
|
||||
REGISTRY.unregister(self.misses)
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
|
||||
class LRUCache:
|
||||
@@ -310,11 +303,14 @@ class LRUCache:
|
||||
self.cache.popitem(last=False)
|
||||
self.cache[key] = value
|
||||
|
||||
def items(self):
|
||||
return self.cache.items()
|
||||
|
||||
def clear(self):
|
||||
self.cache.clear()
|
||||
|
||||
def pop(self, key):
|
||||
return self.cache.pop(key)
|
||||
def pop(self, key, default=None):
|
||||
return self.cache.pop(key, default)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
return self.set(key, value)
|
||||
@@ -346,7 +342,7 @@ def lru_cache_concurrent(cache_size: typing.Optional[int] = None,
|
||||
|
||||
@functools.wraps(async_fn)
|
||||
async def _inner(*args, **kwargs):
|
||||
key = tuple([args, tuple([tuple([k, kwargs[k]]) for k in kwargs])])
|
||||
key = (args, tuple(kwargs.items()))
|
||||
if key in lru_cache:
|
||||
return lru_cache.get(key)
|
||||
|
||||
@@ -380,13 +376,15 @@ CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
|
||||
IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
|
||||
|
||||
|
||||
def is_valid_public_ipv4(address, allow_localhost: bool = False):
|
||||
def is_valid_public_ipv4(address, allow_localhost: bool = False, allow_lan: bool = False):
|
||||
try:
|
||||
parsed_ip = ipaddress.ip_address(address)
|
||||
if parsed_ip.is_loopback and allow_localhost:
|
||||
return True
|
||||
if allow_lan and parsed_ip.is_private:
|
||||
return True
|
||||
if any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local, parsed_ip.is_loopback,
|
||||
parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private, parsed_ip.is_reserved)):
|
||||
parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private)):
|
||||
return False
|
||||
else:
|
||||
return not any((CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
|
||||
|
||||
Binary file not shown.
+12
-33
@@ -5,7 +5,6 @@ import logging
|
||||
import typing
|
||||
import asyncio
|
||||
import random
|
||||
from functools import partial
|
||||
from hashlib import sha256
|
||||
from string import hexdigits
|
||||
from typing import Type, Dict, Tuple, Optional, Any, List
|
||||
@@ -16,7 +15,7 @@ from lbry.crypto.crypt import aes_encrypt, aes_decrypt
|
||||
|
||||
from .bip32 import PrivateKey, PubKey, from_extended_key_string
|
||||
from .mnemonic import Mnemonic
|
||||
from .constants import COIN, CLAIM_TYPES, TXO_TYPES
|
||||
from .constants import COIN, TXO_TYPES
|
||||
from .transaction import Transaction, Input, Output
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
@@ -97,7 +96,8 @@ class AddressManager:
|
||||
return [r['address'] for r in records]
|
||||
|
||||
async def get_or_create_usable_address(self) -> str:
|
||||
addresses = await self.get_addresses(only_usable=True, limit=10)
|
||||
async with self.address_generator_lock:
|
||||
addresses = await self.get_addresses(only_usable=True, limit=10)
|
||||
if addresses:
|
||||
return random.choice(addresses)
|
||||
addresses = await self.ensure_address_gap()
|
||||
@@ -250,7 +250,7 @@ class Account:
|
||||
generator_name = address_generator.get('name', HierarchicalDeterministic.name)
|
||||
self.address_generator = self.address_generators[generator_name]
|
||||
self.receiving, self.change = self.address_generator.from_dict(self, address_generator)
|
||||
self.address_managers = {am.chain_number: am for am in {self.receiving, self.change}}
|
||||
self.address_managers = {am.chain_number: am for am in (self.receiving, self.change)}
|
||||
self.channel_keys = channel_keys
|
||||
ledger.add_account(self)
|
||||
wallet.add_account(self)
|
||||
@@ -566,35 +566,14 @@ class Account:
|
||||
if gap_changed:
|
||||
self.wallet.save()
|
||||
|
||||
async def get_detailed_balance(self, confirmations=0, reserved_subtotals=False, read_only=False):
|
||||
tips_balance, supports_balance, claims_balance = 0, 0, 0
|
||||
get_total_balance = partial(self.get_balance, read_only=read_only, confirmations=confirmations,
|
||||
include_claims=True)
|
||||
total = await get_total_balance()
|
||||
if reserved_subtotals:
|
||||
claims_balance = await get_total_balance(txo_type__in=CLAIM_TYPES)
|
||||
for txo in await self.get_support_summary():
|
||||
if confirmations > 0 and not 0 < txo.tx_ref.height <= self.ledger.headers.height - (confirmations - 1):
|
||||
continue
|
||||
if txo.is_my_input:
|
||||
supports_balance += txo.amount
|
||||
else:
|
||||
tips_balance += txo.amount
|
||||
reserved = claims_balance + supports_balance + tips_balance
|
||||
else:
|
||||
reserved = await self.get_balance(
|
||||
confirmations=confirmations, include_claims=True, txo_type__gt=0
|
||||
)
|
||||
return {
|
||||
'total': total,
|
||||
'available': total - reserved,
|
||||
'reserved': reserved,
|
||||
'reserved_subtotals': {
|
||||
'claims': claims_balance,
|
||||
'supports': supports_balance,
|
||||
'tips': tips_balance
|
||||
} if reserved_subtotals else None
|
||||
}
|
||||
async def get_detailed_balance(self, confirmations=0, read_only=False):
|
||||
constraints = {}
|
||||
if confirmations > 0:
|
||||
height = self.ledger.headers.height - (confirmations-1)
|
||||
constraints.update({'height__lte': height, 'height__gt': 0})
|
||||
return await self.ledger.db.get_detailed_balance(
|
||||
accounts=[self], read_only=read_only, **constraints
|
||||
)
|
||||
|
||||
def get_transaction_history(self, read_only=False, **constraints):
|
||||
return self.ledger.get_transaction_history(
|
||||
|
||||
@@ -46,9 +46,11 @@ class _KeyBase:
|
||||
if len(raw_serkey) != 33:
|
||||
raise ValueError('raw_serkey must have length 33')
|
||||
|
||||
return (ver_bytes + bytes((self.depth,))
|
||||
+ self.parent_fingerprint() + self.n.to_bytes(4, 'big')
|
||||
+ self.chain_code + raw_serkey)
|
||||
return (
|
||||
ver_bytes + bytes((self.depth,))
|
||||
+ self.parent_fingerprint() + self.n.to_bytes(4, 'big')
|
||||
+ self.chain_code + raw_serkey
|
||||
)
|
||||
|
||||
def identifier(self):
|
||||
raise NotImplementedError
|
||||
|
||||
+55
-10
@@ -18,7 +18,7 @@ from .constants import TXO_TYPES, CLAIM_TYPES
|
||||
from .util import date_to_julian_day
|
||||
|
||||
from concurrent.futures.thread import ThreadPoolExecutor # pylint: disable=wrong-import-order
|
||||
if platform.system() == 'Windows' or 'ANDROID_ARGUMENT' or 'KIVY_BUILD' in os.environ:
|
||||
if platform.system() == 'Windows' or ({'ANDROID_ARGUMENT', 'KIVY_BUILD'} & os.environ.keys()):
|
||||
from concurrent.futures.thread import ThreadPoolExecutor as ReaderExecutorClass # pylint: disable=reimported
|
||||
else:
|
||||
from concurrent.futures.process import ProcessPoolExecutor as ReaderExecutorClass
|
||||
@@ -82,10 +82,10 @@ class AIOSQLite:
|
||||
"read_count", "Number of database reads", namespace="daemon_database"
|
||||
)
|
||||
acquire_write_lock_metric = Histogram(
|
||||
f'write_lock_acquired', 'Time to acquire the write lock', namespace="daemon_database", buckets=HISTOGRAM_BUCKETS
|
||||
'write_lock_acquired', 'Time to acquire the write lock', namespace="daemon_database", buckets=HISTOGRAM_BUCKETS
|
||||
)
|
||||
held_write_lock_metric = Histogram(
|
||||
f'write_lock_held', 'Length of time the write lock is held for', namespace="daemon_database",
|
||||
'write_lock_held', 'Length of time the write lock is held for', namespace="daemon_database",
|
||||
buckets=HISTOGRAM_BUCKETS
|
||||
)
|
||||
|
||||
@@ -446,6 +446,10 @@ class SQLiteMixin:
|
||||
version = await self.db.execute_fetchone("SELECT version FROM version LIMIT 1;")
|
||||
if version == (self.SCHEMA_VERSION,):
|
||||
return
|
||||
if version == ("1.5",) and self.SCHEMA_VERSION == "1.6":
|
||||
await self.db.execute("ALTER TABLE txo ADD COLUMN has_source bool DEFAULT 1;")
|
||||
await self.db.execute("UPDATE version SET version = ?", (self.SCHEMA_VERSION,))
|
||||
return
|
||||
await self.db.executescript('\n'.join(
|
||||
f"DROP TABLE {table};" for table in tables
|
||||
) + '\n' + 'PRAGMA WAL_CHECKPOINT(FULL);' + '\n' + 'VACUUM;')
|
||||
@@ -502,7 +506,7 @@ def _get_spendable_utxos(transaction: sqlite3.Connection, accounts: List, decode
|
||||
amount_to_reserve: int, reserved_amount: int, floor: int, ceiling: int,
|
||||
fee_per_byte: int) -> int:
|
||||
accounts_fmt = ",".join(["?"] * len(accounts))
|
||||
txo_query = f"""
|
||||
txo_query = """
|
||||
SELECT tx.txid, txo.txoid, tx.raw, tx.height, txo.position as nout, tx.is_verified, txo.amount FROM txo
|
||||
INNER JOIN account_address USING (address)
|
||||
LEFT JOIN txi USING (txoid)
|
||||
@@ -592,7 +596,7 @@ def get_and_reserve_spendable_utxos(transaction: sqlite3.Connection, accounts: L
|
||||
|
||||
class Database(SQLiteMixin):
|
||||
|
||||
SCHEMA_VERSION = "1.5"
|
||||
SCHEMA_VERSION = "1.6"
|
||||
|
||||
PRAGMAS = """
|
||||
pragma journal_mode=WAL;
|
||||
@@ -646,6 +650,7 @@ class Database(SQLiteMixin):
|
||||
txo_type integer not null default 0,
|
||||
claim_id text,
|
||||
claim_name text,
|
||||
has_source bool,
|
||||
|
||||
channel_id text,
|
||||
reposted_claim_id text
|
||||
@@ -690,7 +695,8 @@ class Database(SQLiteMixin):
|
||||
'address': txo.get_address(self.ledger),
|
||||
'position': txo.position,
|
||||
'amount': txo.amount,
|
||||
'script': sqlite3.Binary(txo.script.source)
|
||||
'script': sqlite3.Binary(txo.script.source),
|
||||
'has_source': False,
|
||||
}
|
||||
if txo.is_claim:
|
||||
if txo.can_decode_claim:
|
||||
@@ -698,8 +704,11 @@ class Database(SQLiteMixin):
|
||||
row['txo_type'] = TXO_TYPES.get(claim.claim_type, TXO_TYPES['stream'])
|
||||
if claim.is_repost:
|
||||
row['reposted_claim_id'] = claim.repost.reference.claim_id
|
||||
row['has_source'] = True
|
||||
if claim.is_signed:
|
||||
row['channel_id'] = claim.signing_channel_id
|
||||
if claim.is_stream:
|
||||
row['has_source'] = claim.stream.has_source
|
||||
else:
|
||||
row['txo_type'] = TXO_TYPES['stream']
|
||||
elif txo.is_support:
|
||||
@@ -760,9 +769,10 @@ class Database(SQLiteMixin):
|
||||
conn.execute(*self._insert_sql(
|
||||
"txo", self.txo_to_row(tx, txo), ignore_duplicate=True
|
||||
)).fetchall()
|
||||
elif txo.script.is_pay_script_hash:
|
||||
# TODO: implement script hash payments
|
||||
log.warning('Database.save_transaction_io: pay script hash is not implemented!')
|
||||
elif txo.script.is_pay_script_hash and is_my_input:
|
||||
conn.execute(*self._insert_sql(
|
||||
"txo", self.txo_to_row(tx, txo), ignore_duplicate=True
|
||||
)).fetchall()
|
||||
|
||||
def save_transaction_io(self, tx: Transaction, address, txhash, history):
|
||||
return self.save_transaction_io_batch([tx], address, txhash, history)
|
||||
@@ -1015,7 +1025,7 @@ class Database(SQLiteMixin):
|
||||
|
||||
if 'order_by' not in constraints or constraints['order_by'] == 'height':
|
||||
constraints['order_by'] = [
|
||||
"tx.height=0 DESC", "tx.height DESC", "tx.position DESC", "txo.position"
|
||||
"tx.height in (0, -1) DESC", "tx.height DESC", "tx.position DESC", "txo.position"
|
||||
]
|
||||
elif constraints.get('order_by', None) == 'none':
|
||||
del constraints['order_by']
|
||||
@@ -1142,6 +1152,41 @@ class Database(SQLiteMixin):
|
||||
)
|
||||
return balance[0]['total'] or 0
|
||||
|
||||
async def get_detailed_balance(self, accounts, read_only=False, **constraints):
|
||||
constraints['accounts'] = accounts
|
||||
result = (await self.select_txos(
|
||||
f"COALESCE(SUM(amount), 0) AS total,"
|
||||
f"COALESCE(SUM("
|
||||
f" CASE WHEN"
|
||||
f" txo_type NOT IN ({TXO_TYPES['other']}, {TXO_TYPES['purchase']})"
|
||||
f" THEN amount ELSE 0 END), 0) AS reserved,"
|
||||
f"COALESCE(SUM("
|
||||
f" CASE WHEN"
|
||||
f" txo_type IN ({','.join(map(str, CLAIM_TYPES))})"
|
||||
f" THEN amount ELSE 0 END), 0) AS claims,"
|
||||
f"COALESCE(SUM(CASE WHEN txo_type = {TXO_TYPES['support']} THEN amount ELSE 0 END), 0) AS supports,"
|
||||
f"COALESCE(SUM("
|
||||
f" CASE WHEN"
|
||||
f" txo_type = {TXO_TYPES['support']} AND"
|
||||
f" TXI.address IS NOT NULL AND"
|
||||
f" TXI.address IN (SELECT address FROM account_address WHERE account = :$account__in0)"
|
||||
f" THEN amount ELSE 0 END), 0) AS my_supports",
|
||||
is_spent=False,
|
||||
include_is_my_input=True,
|
||||
read_only=read_only,
|
||||
**constraints
|
||||
))[0]
|
||||
return {
|
||||
"total": result["total"],
|
||||
"available": result["total"] - result["reserved"],
|
||||
"reserved": result["reserved"],
|
||||
"reserved_subtotals": {
|
||||
"claims": result["claims"],
|
||||
"supports": result["my_supports"],
|
||||
"tips": result["supports"] - result["my_supports"]
|
||||
}
|
||||
}
|
||||
|
||||
async def select_addresses(self, cols, read_only=False, **constraints):
|
||||
return await self.db.execute_fetchall(*query(
|
||||
f"SELECT {cols} FROM pubkey_address JOIN account_address USING (address)",
|
||||
|
||||
+27
-8
@@ -178,15 +178,25 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
raw_address = cls.pubkey_address_prefix + h160
|
||||
return Base58.encode(bytearray(raw_address + double_sha256(raw_address)[0:4]))
|
||||
|
||||
@classmethod
|
||||
def hash160_to_script_address(cls, h160):
|
||||
raw_address = cls.script_address_prefix + h160
|
||||
return Base58.encode(bytearray(raw_address + double_sha256(raw_address)[0:4]))
|
||||
|
||||
@staticmethod
|
||||
def address_to_hash160(address):
|
||||
return Base58.decode(address)[1:21]
|
||||
|
||||
@classmethod
|
||||
def is_valid_address(cls, address):
|
||||
def is_pubkey_address(cls, address):
|
||||
decoded = Base58.decode_check(address)
|
||||
return decoded[0] == cls.pubkey_address_prefix[0]
|
||||
|
||||
@classmethod
|
||||
def is_script_address(cls, address):
|
||||
decoded = Base58.decode_check(address)
|
||||
return decoded[0] == cls.script_address_prefix[0]
|
||||
|
||||
@classmethod
|
||||
def public_key_to_address(cls, public_key):
|
||||
return cls.hash160_to_address(hash160(public_key))
|
||||
@@ -538,7 +548,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
"request %i transactions, %i/%i for %s are already synced", len(to_request), len(already_synced),
|
||||
len(remote_history), address
|
||||
)
|
||||
remote_history_txids = set(txid for txid, _ in remote_history)
|
||||
remote_history_txids = {txid for txid, _ in remote_history}
|
||||
async for tx in self.request_synced_transactions(to_request, remote_history_txids, address):
|
||||
pending_synced_history[tx_indexes[tx.id]] = f"{tx.id}:{tx.height}:"
|
||||
if len(pending_synced_history) % 100 == 0:
|
||||
@@ -713,8 +723,10 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
self.hash160_to_address(txi.txo_ref.txo.pubkey_hash)
|
||||
)
|
||||
for txo in tx.outputs:
|
||||
if txo.has_address:
|
||||
if txo.is_pubkey_hash:
|
||||
addresses.add(self.hash160_to_address(txo.pubkey_hash))
|
||||
elif txo.is_script_hash:
|
||||
addresses.add(self.hash160_to_script_address(txo.script_hash))
|
||||
start = int(time.perf_counter())
|
||||
while timeout and (int(time.perf_counter()) - start) <= timeout:
|
||||
if await self._wait_round(tx, height, addresses):
|
||||
@@ -738,7 +750,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
))[1] if record['history'] else []
|
||||
for txid, local_height in local_history:
|
||||
if txid == tx.id:
|
||||
if local_height >= height:
|
||||
if local_height >= height or (local_height == 0 and height > local_height):
|
||||
return True
|
||||
log.warning(
|
||||
"local history has higher height than remote for %s (%i vs %i)", txid,
|
||||
@@ -756,9 +768,13 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
include_is_my_output=False,
|
||||
include_sent_supports=False,
|
||||
include_sent_tips=False,
|
||||
include_received_tips=False) -> Tuple[List[Output], dict, int, int]:
|
||||
include_received_tips=False,
|
||||
hub_server=False) -> Tuple[List[Output], dict, int, int]:
|
||||
encoded_outputs = await query
|
||||
outputs = Outputs.from_base64(encoded_outputs or b'') # TODO: why is the server returning None?
|
||||
if hub_server:
|
||||
outputs = Outputs.from_grpc(encoded_outputs)
|
||||
else:
|
||||
outputs = Outputs.from_base64(encoded_outputs or b'') # TODO: why is the server returning None?
|
||||
txs: List[Transaction] = []
|
||||
if len(outputs.txs) > 0:
|
||||
async for tx in self.request_transactions(tuple(outputs.txs), cached=True):
|
||||
@@ -875,6 +891,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
claim_search(**kwargs), accounts,
|
||||
include_purchase_receipt=include_purchase_receipt,
|
||||
include_is_my_output=include_is_my_output,
|
||||
hub_server=new_sdk_server is not None
|
||||
)
|
||||
|
||||
async def get_claim_by_claim_id(self, accounts, claim_id, **kwargs) -> Output:
|
||||
@@ -1060,7 +1077,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
'abandon_info': [],
|
||||
'purchase_info': []
|
||||
}
|
||||
is_my_inputs = all([txi.is_my_input for txi in tx.inputs])
|
||||
is_my_inputs = all(txi.is_my_input for txi in tx.inputs)
|
||||
if is_my_inputs:
|
||||
# fees only matter if we are the ones paying them
|
||||
item['value'] = dewies_to_lbc(tx.net_account_balance + tx.fee)
|
||||
@@ -1171,7 +1188,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
balance = self._balance_cache.get(account.id)
|
||||
if not balance:
|
||||
balance = self._balance_cache[account.id] = \
|
||||
await account.get_detailed_balance(confirmations, reserved_subtotals=True)
|
||||
await account.get_detailed_balance(confirmations)
|
||||
for key, value in balance.items():
|
||||
if key == 'reserved_subtotals':
|
||||
for subkey, subvalue in value.items():
|
||||
@@ -1180,6 +1197,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
result[key] += value
|
||||
return result
|
||||
|
||||
|
||||
class TestNetLedger(Ledger):
|
||||
network_name = 'testnet'
|
||||
pubkey_address_prefix = bytes((111,))
|
||||
@@ -1188,6 +1206,7 @@ class TestNetLedger(Ledger):
|
||||
extended_private_key_prefix = unhexlify('04358394')
|
||||
checkpoints = {}
|
||||
|
||||
|
||||
class RegTestLedger(Ledger):
|
||||
network_name = 'regtest'
|
||||
headers_class = UnvalidatedHeaders
|
||||
|
||||
+24
-4
@@ -3,12 +3,14 @@ import json
|
||||
import typing
|
||||
import logging
|
||||
import asyncio
|
||||
from distutils.util import strtobool
|
||||
|
||||
from binascii import unhexlify
|
||||
from decimal import Decimal
|
||||
from typing import List, Type, MutableSequence, MutableMapping, Optional
|
||||
|
||||
from lbry.error import KeyFeeAboveMaxAllowedError
|
||||
from lbry.conf import Config
|
||||
from lbry.error import KeyFeeAboveMaxAllowedError, WalletNotLoadedError
|
||||
from lbry.conf import Config, NOT_SET
|
||||
|
||||
from .dewies import dewies_to_lbc
|
||||
from .account import Account
|
||||
@@ -95,7 +97,7 @@ class WalletManager:
|
||||
for wallet in self.wallets:
|
||||
if wallet.id == wallet_id:
|
||||
return wallet
|
||||
raise ValueError(f"Couldn't find wallet: {wallet_id}.")
|
||||
raise WalletNotLoadedError(wallet_id)
|
||||
|
||||
@staticmethod
|
||||
def get_balance(wallet):
|
||||
@@ -181,8 +183,14 @@ class WalletManager:
|
||||
}[config.blockchain_name]
|
||||
|
||||
ledger_config = {
|
||||
'use_go_hub': not strtobool(os.environ.get('ENABLE_LEGACY_SEARCH') or 'yes'),
|
||||
'auto_connect': True,
|
||||
'explicit_servers': [],
|
||||
'hub_timeout': config.hub_timeout,
|
||||
'default_servers': config.lbryum_servers,
|
||||
'known_hubs': config.known_hubs,
|
||||
'jurisdiction': config.jurisdiction,
|
||||
'concurrent_hub_requests': config.concurrent_hub_requests,
|
||||
'data_path': config.wallet_dir,
|
||||
'tx_cache_size': config.transaction_cache_size
|
||||
}
|
||||
@@ -195,6 +203,10 @@ class WalletManager:
|
||||
os.path.join(wallets_directory, 'default_wallet')
|
||||
)
|
||||
|
||||
if Config.lbryum_servers.is_set_to_default(config):
|
||||
with config.update_config() as c:
|
||||
c.lbryum_servers = NOT_SET
|
||||
|
||||
manager = cls.from_config({
|
||||
'ledgers': {ledger_id: ledger_config},
|
||||
'wallets': [
|
||||
@@ -224,10 +236,18 @@ class WalletManager:
|
||||
|
||||
async def reset(self):
|
||||
self.ledger.config = {
|
||||
'use_go_hub': not strtobool(os.environ.get('ENABLE_LEGACY_SEARCH') or 'yes'),
|
||||
'auto_connect': True,
|
||||
'default_servers': self.config.lbryum_servers,
|
||||
'explicit_servers': [],
|
||||
'default_servers': Config.lbryum_servers.default,
|
||||
'known_hubs': self.config.known_hubs,
|
||||
'jurisdiction': self.config.jurisdiction,
|
||||
'hub_timeout': self.config.hub_timeout,
|
||||
'concurrent_hub_requests': self.config.concurrent_hub_requests,
|
||||
'data_path': self.config.wallet_dir,
|
||||
}
|
||||
if Config.lbryum_servers.is_set(self.config):
|
||||
self.ledger.config['explicit_servers'] = self.config.lbryum_servers
|
||||
await self.ledger.stop()
|
||||
await self.ledger.start()
|
||||
|
||||
|
||||
+80
-30
@@ -7,6 +7,9 @@ from time import perf_counter
|
||||
from collections import defaultdict
|
||||
from typing import Dict, Optional, Tuple
|
||||
import aiohttp
|
||||
import grpc
|
||||
from lbry.schema.types.v2 import hub_pb2_grpc
|
||||
from lbry.schema.types.v2.hub_pb2 import SearchRequest
|
||||
|
||||
from lbry import __version__
|
||||
from lbry.utils import resolve_host
|
||||
@@ -14,12 +17,13 @@ from lbry.error import IncompatibleWalletServerError
|
||||
from lbry.wallet.rpc import RPCSession as BaseClientSession, Connector, RPCError, ProtocolError
|
||||
from lbry.wallet.stream import StreamController
|
||||
from lbry.wallet.server.udp import SPVStatusClientProtocol, SPVPong
|
||||
from lbry.conf import KnownHubsList
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ClientSession(BaseClientSession):
|
||||
def __init__(self, *args, network: 'Network', server, timeout=30, **kwargs):
|
||||
def __init__(self, *args, network: 'Network', server, timeout=30, concurrency=32, **kwargs):
|
||||
self.network = network
|
||||
self.server = server
|
||||
super().__init__(*args, **kwargs)
|
||||
@@ -29,7 +33,11 @@ class ClientSession(BaseClientSession):
|
||||
self.response_time: Optional[float] = None
|
||||
self.connection_latency: Optional[float] = None
|
||||
self._response_samples = 0
|
||||
self.pending_amount = 0
|
||||
self._concurrency = asyncio.Semaphore(concurrency)
|
||||
|
||||
@property
|
||||
def concurrency(self):
|
||||
return self._concurrency._value
|
||||
|
||||
@property
|
||||
def available(self):
|
||||
@@ -55,9 +63,9 @@ class ClientSession(BaseClientSession):
|
||||
return result
|
||||
|
||||
async def send_request(self, method, args=()):
|
||||
self.pending_amount += 1
|
||||
log.debug("send %s%s to %s:%i (%i timeout)", method, tuple(args), self.server[0], self.server[1], self.timeout)
|
||||
try:
|
||||
await self._concurrency.acquire()
|
||||
if method == 'server.version':
|
||||
return await self.send_timed_server_version_request(args, self.timeout)
|
||||
request = asyncio.ensure_future(super().send_request(method, args))
|
||||
@@ -91,7 +99,7 @@ class ClientSession(BaseClientSession):
|
||||
# self.synchronous_close()
|
||||
raise
|
||||
finally:
|
||||
self.pending_amount -= 1
|
||||
self._concurrency.release()
|
||||
|
||||
async def ensure_server_version(self, required=None, timeout=3):
|
||||
required = required or self.network.PROTOCOL_VERSION
|
||||
@@ -154,7 +162,6 @@ class Network:
|
||||
# self._switch_task: Optional[asyncio.Task] = None
|
||||
self.running = False
|
||||
self.remote_height: int = 0
|
||||
self._concurrency = asyncio.Semaphore(16)
|
||||
|
||||
self._on_connected_controller = StreamController()
|
||||
self.on_connected = self._on_connected_controller.stream
|
||||
@@ -165,9 +172,13 @@ class Network:
|
||||
self._on_status_controller = StreamController(merge_repeated_events=True)
|
||||
self.on_status = self._on_status_controller.stream
|
||||
|
||||
self._on_hub_controller = StreamController(merge_repeated_events=True)
|
||||
self.on_hub = self._on_hub_controller.stream
|
||||
|
||||
self.subscription_controllers = {
|
||||
'blockchain.headers.subscribe': self._on_header_controller,
|
||||
'blockchain.address.subscribe': self._on_status_controller,
|
||||
'blockchain.peers.subscribe': self._on_hub_controller,
|
||||
}
|
||||
|
||||
self.aiohttp_session: Optional[aiohttp.ClientSession] = None
|
||||
@@ -179,6 +190,16 @@ class Network:
|
||||
def config(self):
|
||||
return self.ledger.config
|
||||
|
||||
@property
|
||||
def known_hubs(self):
|
||||
if 'known_hubs' not in self.config:
|
||||
return KnownHubsList()
|
||||
return self.config['known_hubs']
|
||||
|
||||
@property
|
||||
def jurisdiction(self):
|
||||
return self.config.get("jurisdiction")
|
||||
|
||||
def disconnect(self):
|
||||
if self._keepalive_task and not self._keepalive_task.done():
|
||||
self._keepalive_task.cancel()
|
||||
@@ -189,6 +210,7 @@ class Network:
|
||||
self.running = True
|
||||
self.aiohttp_session = aiohttp.ClientSession()
|
||||
self.on_header.listen(self._update_remote_height)
|
||||
self.on_hub.listen(self._update_hubs)
|
||||
self._loop_task = asyncio.create_task(self.network_loop())
|
||||
self._urgent_need_reconnect.set()
|
||||
|
||||
@@ -216,18 +238,25 @@ class Network:
|
||||
log.exception("error looking up dns for spv server %s:%i", server, port)
|
||||
|
||||
# accumulate the dns results
|
||||
await asyncio.gather(*(resolve_spv(server, port) for (server, port) in self.config['default_servers']))
|
||||
if self.config['explicit_servers']:
|
||||
hubs = self.config['explicit_servers']
|
||||
elif self.known_hubs:
|
||||
hubs = self.known_hubs
|
||||
else:
|
||||
hubs = self.config['default_servers']
|
||||
await asyncio.gather(*(resolve_spv(server, port) for (server, port) in hubs))
|
||||
return hostname_to_ip, ip_to_hostnames
|
||||
|
||||
async def get_n_fastest_spvs(self, timeout=3.0) -> Dict[Tuple[str, int], SPVPong]:
|
||||
async def get_n_fastest_spvs(self, timeout=3.0) -> Dict[Tuple[str, int], Optional[SPVPong]]:
|
||||
loop = asyncio.get_event_loop()
|
||||
pong_responses = asyncio.Queue()
|
||||
connection = SPVStatusClientProtocol(pong_responses)
|
||||
sent_ping_timestamps = {}
|
||||
_, ip_to_hostnames = await self.resolve_spv_dns()
|
||||
n = len(ip_to_hostnames)
|
||||
log.info("%i possible spv servers to try (%i urls in config)", n, len(self.config['default_servers']))
|
||||
log.info("%i possible spv servers to try (%i urls in config)", n, len(self.config['explicit_servers']))
|
||||
pongs = {}
|
||||
known_hubs = self.known_hubs
|
||||
try:
|
||||
await loop.create_datagram_endpoint(lambda: connection, ('0.0.0.0', 0))
|
||||
# could raise OSError if it cant bind
|
||||
@@ -242,8 +271,11 @@ class Network:
|
||||
'/'.join(ip_to_hostnames[remote]), remote[1], round(latency * 1000, 2),
|
||||
pong.available, pong.height)
|
||||
|
||||
known_hubs.hubs.setdefault((ip_to_hostnames[remote][0], remote[1]), {}).update(
|
||||
{"country": pong.country_name}
|
||||
)
|
||||
if pong.available:
|
||||
pongs[remote] = pong
|
||||
pongs[(ip_to_hostnames[remote][0], remote[1])] = pong
|
||||
return pongs
|
||||
except asyncio.TimeoutError:
|
||||
if pongs:
|
||||
@@ -253,15 +285,22 @@ class Network:
|
||||
log.warning("%i spv status probes failed, retrying later. servers tried: %s",
|
||||
len(sent_ping_timestamps),
|
||||
', '.join('/'.join(hosts) + f' ({ip})' for ip, hosts in ip_to_hostnames.items()))
|
||||
return {random.choice(list(ip_to_hostnames)): None}
|
||||
random_server = random.choice(list(ip_to_hostnames.keys()))
|
||||
host, port = random_server
|
||||
log.warning("trying fallback to randomly selected spv: %s:%i", host, port)
|
||||
known_hubs.hubs.setdefault((host, port), {})
|
||||
return {(host, port): None}
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
async def connect_to_fastest(self) -> Optional[ClientSession]:
|
||||
fastest_spvs = await self.get_n_fastest_spvs()
|
||||
for (host, port) in fastest_spvs:
|
||||
|
||||
client = ClientSession(network=self, server=(host, port))
|
||||
for (host, port), pong in fastest_spvs.items():
|
||||
if (pong is not None and self.jurisdiction is not None) and \
|
||||
(pong.country_name != self.jurisdiction):
|
||||
continue
|
||||
client = ClientSession(network=self, server=(host, port), timeout=self.config['hub_timeout'],
|
||||
concurrency=self.config['concurrent_hub_requests'])
|
||||
try:
|
||||
await client.create_connection()
|
||||
log.warning("Connected to spv server %s:%i", host, port)
|
||||
@@ -291,6 +330,8 @@ class Network:
|
||||
log.debug("get spv server features %s:%i", *client.server)
|
||||
features = await client.send_request('server.features', [])
|
||||
self.client, self.server_features = client, features
|
||||
log.debug("discover other hubs %s:%i", *client.server)
|
||||
await self._update_hubs(await client.send_request('server.peers.subscribe', []))
|
||||
log.info("subscribe to headers %s:%i", *client.server)
|
||||
self._update_remote_height((await self.subscribe_headers(),))
|
||||
self._on_connected_controller.add(True)
|
||||
@@ -342,24 +383,31 @@ class Network:
|
||||
raise ConnectionError("Attempting to send rpc request when connection is not available.")
|
||||
|
||||
async def retriable_call(self, function, *args, **kwargs):
|
||||
async with self._concurrency:
|
||||
while self.running:
|
||||
if not self.is_connected:
|
||||
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
|
||||
self._urgent_need_reconnect.set()
|
||||
await self.on_connected.first
|
||||
try:
|
||||
return await function(*args, **kwargs)
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("Wallet server call timed out, retrying.")
|
||||
except ConnectionError:
|
||||
log.warning("connection error")
|
||||
while self.running:
|
||||
if not self.is_connected:
|
||||
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
|
||||
self._urgent_need_reconnect.set()
|
||||
await self.on_connected.first
|
||||
try:
|
||||
return await function(*args, **kwargs)
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("Wallet server call timed out, retrying.")
|
||||
except ConnectionError:
|
||||
log.warning("connection error")
|
||||
|
||||
raise asyncio.CancelledError() # if we got here, we are shutting down
|
||||
|
||||
def _update_remote_height(self, header_args):
|
||||
self.remote_height = header_args[0]["height"]
|
||||
|
||||
async def _update_hubs(self, hubs):
|
||||
if hubs and hubs != ['']:
|
||||
try:
|
||||
if self.known_hubs.add_hubs(hubs):
|
||||
self.known_hubs.save()
|
||||
except Exception:
|
||||
log.exception("could not add hubs: %s", hubs)
|
||||
|
||||
def get_transaction(self, tx_hash, known_height=None):
|
||||
# use any server if its old, otherwise restrict to who gave us the history
|
||||
restricted = known_height in (None, -1, 0) or 0 > known_height > self.remote_height - 10
|
||||
@@ -430,11 +478,13 @@ class Network:
|
||||
return result['result']
|
||||
|
||||
async def new_claim_search(self, server, **kwargs):
|
||||
kwargs['protobuf'] = True
|
||||
message = {"method": "claim_search", "params": kwargs}
|
||||
async with self.aiohttp_session.post(server, json=message) as r:
|
||||
result = await r.json()
|
||||
return result['result']
|
||||
async with grpc.aio.insecure_channel(server) as channel:
|
||||
stub = hub_pb2_grpc.HubStub(channel)
|
||||
try:
|
||||
response = await stub.Search(SearchRequest(**kwargs))
|
||||
except grpc.aio.AioRpcError as error:
|
||||
raise RPCError(error.code(), error.details())
|
||||
return response
|
||||
|
||||
async def sum_supports(self, server, **kwargs):
|
||||
message = {"method": "support_sum", "params": kwargs}
|
||||
|
||||
@@ -1,2 +1,5 @@
|
||||
__hub_url__ = (
|
||||
"https://github.com/lbryio/hub/releases/download/v0.2021.08.24-beta/hub"
|
||||
)
|
||||
from .node import Conductor
|
||||
from .service import ConductorService
|
||||
@@ -8,6 +8,8 @@ import logging
|
||||
import tempfile
|
||||
import subprocess
|
||||
import importlib
|
||||
from distutils.util import strtobool
|
||||
|
||||
from binascii import hexlify
|
||||
from typing import Type, Optional
|
||||
import urllib.request
|
||||
@@ -17,7 +19,8 @@ import lbry
|
||||
from lbry.wallet.server.server import Server
|
||||
from lbry.wallet.server.env import Env
|
||||
from lbry.wallet import Wallet, Ledger, RegTestLedger, WalletManager, Account, BlockHeightEvent
|
||||
|
||||
from lbry.conf import KnownHubsList, Config
|
||||
from lbry.wallet.orchstr8 import __hub_url__
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -47,10 +50,12 @@ class Conductor:
|
||||
self.wallet_node = WalletNode(
|
||||
self.manager_module, RegTestLedger, default_seed=seed
|
||||
)
|
||||
self.hub_node = HubNode(__hub_url__, "hub", self.spv_node)
|
||||
|
||||
self.blockchain_started = False
|
||||
self.spv_started = False
|
||||
self.wallet_started = False
|
||||
self.hub_started = False
|
||||
|
||||
self.log = log.getChild('conductor')
|
||||
|
||||
@@ -66,6 +71,17 @@ class Conductor:
|
||||
await self.blockchain_node.stop(cleanup=True)
|
||||
self.blockchain_started = False
|
||||
|
||||
async def start_hub(self):
|
||||
if not self.hub_started:
|
||||
asyncio.create_task(self.hub_node.start())
|
||||
await self.blockchain_node.running.wait()
|
||||
self.hub_started = True
|
||||
|
||||
async def stop_hub(self):
|
||||
if self.hub_started:
|
||||
await self.hub_node.stop(cleanup=True)
|
||||
self.hub_started = False
|
||||
|
||||
async def start_spv(self):
|
||||
if not self.spv_started:
|
||||
await self.spv_node.start(self.blockchain_node)
|
||||
@@ -107,7 +123,8 @@ class Conductor:
|
||||
class WalletNode:
|
||||
|
||||
def __init__(self, manager_class: Type[WalletManager], ledger_class: Type[Ledger],
|
||||
verbose: bool = False, port: int = 5280, default_seed: str = None) -> None:
|
||||
verbose: bool = False, port: int = 5280, default_seed: str = None,
|
||||
data_path: str = None) -> None:
|
||||
self.manager_class = manager_class
|
||||
self.ledger_class = ledger_class
|
||||
self.verbose = verbose
|
||||
@@ -115,12 +132,12 @@ class WalletNode:
|
||||
self.ledger: Optional[Ledger] = None
|
||||
self.wallet: Optional[Wallet] = None
|
||||
self.account: Optional[Account] = None
|
||||
self.data_path: Optional[str] = None
|
||||
self.data_path: str = data_path or tempfile.mkdtemp()
|
||||
self.port = port
|
||||
self.default_seed = default_seed
|
||||
self.known_hubs = KnownHubsList()
|
||||
|
||||
async def start(self, spv_node: 'SPVNode', seed=None, connect=True):
|
||||
self.data_path = tempfile.mkdtemp()
|
||||
async def start(self, spv_node: 'SPVNode', seed=None, connect=True, config=None):
|
||||
wallets_dir = os.path.join(self.data_path, 'wallets')
|
||||
os.mkdir(wallets_dir)
|
||||
wallet_file_name = os.path.join(wallets_dir, 'my_wallet.json')
|
||||
@@ -129,13 +146,19 @@ class WalletNode:
|
||||
self.manager = self.manager_class.from_config({
|
||||
'ledgers': {
|
||||
self.ledger_class.get_id(): {
|
||||
'use_go_hub': not strtobool(os.environ.get('ENABLE_LEGACY_SEARCH') or 'yes'),
|
||||
'api_port': self.port,
|
||||
'default_servers': [(spv_node.hostname, spv_node.port)],
|
||||
'data_path': self.data_path
|
||||
'explicit_servers': [(spv_node.hostname, spv_node.port)],
|
||||
'default_servers': Config.lbryum_servers.default,
|
||||
'data_path': self.data_path,
|
||||
'known_hubs': config.known_hubs if config else KnownHubsList(),
|
||||
'hub_timeout': 30,
|
||||
'concurrent_hub_requests': 32,
|
||||
}
|
||||
},
|
||||
'wallets': [wallet_file_name]
|
||||
})
|
||||
self.manager.config = config
|
||||
self.ledger = self.manager.ledgers[self.ledger_class]
|
||||
self.wallet = self.manager.default_wallet
|
||||
if not self.wallet:
|
||||
@@ -172,9 +195,12 @@ class SPVNode:
|
||||
self.udp_port = self.port
|
||||
self.session_timeout = 600
|
||||
self.rpc_port = '0' # disabled by default
|
||||
self.stopped = False
|
||||
self.index_name = None
|
||||
|
||||
async def start(self, blockchain_node: 'BlockchainNode', extraconf=None):
|
||||
self.data_path = tempfile.mkdtemp()
|
||||
self.index_name = uuid4().hex
|
||||
conf = {
|
||||
'DESCRIPTION': '',
|
||||
'PAYMENT_ADDRESS': '',
|
||||
@@ -189,7 +215,7 @@ class SPVNode:
|
||||
'MAX_QUERY_WORKERS': '0',
|
||||
'INDIVIDUAL_TAG_INDEXES': '',
|
||||
'RPC_PORT': self.rpc_port,
|
||||
'ES_INDEX_PREFIX': uuid4().hex,
|
||||
'ES_INDEX_PREFIX': self.index_name,
|
||||
'ES_MODE': 'writer',
|
||||
}
|
||||
if extraconf:
|
||||
@@ -201,10 +227,13 @@ class SPVNode:
|
||||
await self.server.start()
|
||||
|
||||
async def stop(self, cleanup=True):
|
||||
if self.stopped:
|
||||
return
|
||||
try:
|
||||
await self.server.db.search_index.delete_index()
|
||||
await self.server.db.search_index.stop()
|
||||
await self.server.stop()
|
||||
self.stopped = True
|
||||
finally:
|
||||
cleanup and self.cleanup()
|
||||
|
||||
@@ -410,7 +439,7 @@ class BlockchainNode:
|
||||
return self._cli_cmnd('getnewaddress', "", address_type)
|
||||
|
||||
async def get_balance(self):
|
||||
return float(await self._cli_cmnd('getbalance'))
|
||||
return await self._cli_cmnd('getbalance')
|
||||
|
||||
def send_to_address(self, address, amount):
|
||||
return self._cli_cmnd('sendtoaddress', address, str(amount))
|
||||
@@ -429,3 +458,129 @@ class BlockchainNode:
|
||||
|
||||
def get_raw_transaction(self, txid):
|
||||
return self._cli_cmnd('getrawtransaction', txid, '1')
|
||||
|
||||
|
||||
class HubProcess(asyncio.SubprocessProtocol):
|
||||
def __init__(self):
|
||||
self.ready = asyncio.Event()
|
||||
self.stopped = asyncio.Event()
|
||||
self.log = log.getChild('hub')
|
||||
|
||||
def pipe_data_received(self, fd, data):
|
||||
if self.log:
|
||||
self.log.info(data.decode())
|
||||
if b'error' in data.lower():
|
||||
self.ready.set()
|
||||
raise SystemError(data.decode())
|
||||
if b'listening on' in data:
|
||||
self.ready.set()
|
||||
|
||||
def process_exited(self):
|
||||
self.stopped.set()
|
||||
self.ready.set()
|
||||
|
||||
|
||||
class HubNode:
|
||||
|
||||
def __init__(self, url, daemon, spv_node):
|
||||
self.spv_node = spv_node
|
||||
self.debug = False
|
||||
|
||||
self.latest_release_url = url
|
||||
self.project_dir = os.path.dirname(os.path.dirname(__file__))
|
||||
self.bin_dir = os.path.join(self.project_dir, 'bin')
|
||||
self.daemon_bin = os.path.join(self.bin_dir, daemon)
|
||||
self.cli_bin = os.path.join(self.bin_dir, daemon)
|
||||
self.log = log.getChild('hub')
|
||||
self.transport = None
|
||||
self.protocol = None
|
||||
self.hostname = 'localhost'
|
||||
self.rpcport = 50051 # avoid conflict with default rpc port
|
||||
self.stopped = False
|
||||
self.restart_ready = asyncio.Event()
|
||||
self.restart_ready.set()
|
||||
self.running = asyncio.Event()
|
||||
|
||||
@property
|
||||
def exists(self):
|
||||
return (
|
||||
os.path.exists(self.cli_bin) and
|
||||
os.path.exists(self.daemon_bin)
|
||||
)
|
||||
|
||||
def download(self):
|
||||
downloaded_file = os.path.join(
|
||||
self.bin_dir,
|
||||
self.latest_release_url[self.latest_release_url.rfind('/')+1:]
|
||||
)
|
||||
|
||||
if not os.path.exists(self.bin_dir):
|
||||
os.mkdir(self.bin_dir)
|
||||
|
||||
if not os.path.exists(downloaded_file):
|
||||
self.log.info('Downloading: %s', self.latest_release_url)
|
||||
with urllib.request.urlopen(self.latest_release_url) as response:
|
||||
with open(downloaded_file, 'wb') as out_file:
|
||||
shutil.copyfileobj(response, out_file)
|
||||
|
||||
self.log.info('Extracting: %s', downloaded_file)
|
||||
|
||||
if downloaded_file.endswith('.zip'):
|
||||
with zipfile.ZipFile(downloaded_file) as dotzip:
|
||||
dotzip.extractall(self.bin_dir)
|
||||
# zipfile bug https://bugs.python.org/issue15795
|
||||
os.chmod(self.cli_bin, 0o755)
|
||||
os.chmod(self.daemon_bin, 0o755)
|
||||
|
||||
elif downloaded_file.endswith('.tar.gz'):
|
||||
with tarfile.open(downloaded_file) as tar:
|
||||
tar.extractall(self.bin_dir)
|
||||
|
||||
os.chmod(self.daemon_bin, 0o755)
|
||||
|
||||
return self.exists
|
||||
|
||||
def ensure(self):
|
||||
return self.exists or self.download()
|
||||
|
||||
async def start(self):
|
||||
assert self.ensure()
|
||||
loop = asyncio.get_event_loop()
|
||||
asyncio.get_child_watcher().attach_loop(loop)
|
||||
command = [
|
||||
self.daemon_bin, 'serve', '--esindex', self.spv_node.index_name + 'claims', '--debug'
|
||||
]
|
||||
self.log.info(' '.join(command))
|
||||
while not self.stopped:
|
||||
if self.running.is_set():
|
||||
await asyncio.sleep(1)
|
||||
continue
|
||||
await self.restart_ready.wait()
|
||||
try:
|
||||
if not self.debug:
|
||||
self.transport, self.protocol = await loop.subprocess_exec(
|
||||
HubProcess, *command
|
||||
)
|
||||
await self.protocol.ready.wait()
|
||||
assert not self.protocol.stopped.is_set()
|
||||
self.running.set()
|
||||
except asyncio.CancelledError:
|
||||
self.running.clear()
|
||||
raise
|
||||
except Exception as e:
|
||||
self.running.clear()
|
||||
log.exception('failed to start hub', exc_info=e)
|
||||
|
||||
async def stop(self, cleanup=True):
|
||||
self.stopped = True
|
||||
try:
|
||||
if not self.debug:
|
||||
self.transport.terminate()
|
||||
await self.protocol.stopped.wait()
|
||||
self.transport.close()
|
||||
finally:
|
||||
if cleanup:
|
||||
self.cleanup()
|
||||
|
||||
def cleanup(self):
|
||||
pass
|
||||
@@ -108,9 +108,6 @@ class Response:
|
||||
|
||||
class CodeMessageError(Exception):
|
||||
|
||||
def __init__(self, code, message):
|
||||
super().__init__(code, message)
|
||||
|
||||
@property
|
||||
def code(self):
|
||||
return self.args[0]
|
||||
|
||||
@@ -436,7 +436,8 @@ class RPCSession(SessionBase):
|
||||
except CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
self.logger.exception(f'exception handling {request}')
|
||||
reqstr = str(request)
|
||||
self.logger.exception(f'exception handling {reqstr[:16_000]}')
|
||||
result = RPCError(JSONRPC.INTERNAL_ERROR,
|
||||
'internal server error')
|
||||
if isinstance(request, Request):
|
||||
|
||||
@@ -605,8 +605,7 @@ class BlockProcessor:
|
||||
# Key: b'h' + compressed_tx_hash + tx_idx + tx_num
|
||||
# Value: hashX
|
||||
prefix = b'h' + tx_hash[:4] + idx_packed
|
||||
candidates = {db_key: hashX for db_key, hashX
|
||||
in self.db.utxo_db.iterator(prefix=prefix)}
|
||||
candidates = dict(self.db.utxo_db.iterator(prefix=prefix))
|
||||
for hdb_key, hashX in candidates.items():
|
||||
tx_num_packed = hdb_key[-4:]
|
||||
if len(candidates) > 1:
|
||||
@@ -799,11 +798,6 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
finally:
|
||||
if self.sql:
|
||||
self.sql.commit()
|
||||
if self.sql and self.db.first_sync and self.height == self.daemon.cached_height():
|
||||
self.timer.run(self.sql.execute, self.sql.SEARCH_INDEXES, timer_name='executing SEARCH_INDEXES')
|
||||
if self.env.individual_tag_indexes:
|
||||
self.timer.run(self.sql.execute, self.sql.TAG_INDEXES, timer_name='executing TAG_INDEXES')
|
||||
self.timer.run(self.sql.execute, self.sql.LANGUAGE_INDEXES, timer_name='executing LANGUAGE_INDEXES')
|
||||
|
||||
def advance_txs(self, height, txs, header, block_hash):
|
||||
timer = self.timer.sub_timers['advance_blocks']
|
||||
|
||||
@@ -25,7 +25,7 @@ def main():
|
||||
parser = get_argument_parser()
|
||||
args = parser.parse_args()
|
||||
coin_class = get_coin_class(args.spvserver)
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-4s %(name)s:%(lineno)d: %(message)s")
|
||||
logging.info('lbry.server starting')
|
||||
logging.getLogger('aiohttp').setLevel(logging.WARNING)
|
||||
logging.getLogger('elasticsearch').setLevel(logging.WARNING)
|
||||
|
||||
@@ -17,10 +17,6 @@ class FindShortestID:
|
||||
if self.short_id:
|
||||
return '#'+self.short_id
|
||||
|
||||
@classmethod
|
||||
def factory(cls):
|
||||
return cls(), cls.step, cls.finalize
|
||||
|
||||
|
||||
def register_canonical_functions(connection):
|
||||
connection.createaggregatefunction("shortest_id", FindShortestID.factory, 2)
|
||||
connection.create_aggregate("shortest_id", 2, FindShortestID)
|
||||
@@ -35,27 +35,48 @@ INDEX_DEFAULT_SETTINGS = {
|
||||
}
|
||||
}
|
||||
}
|
||||
FIELDS = {'is_controlling', 'last_take_over_height', 'claim_id', 'claim_name', 'normalized', 'tx_position', 'amount',
|
||||
'timestamp', 'creation_timestamp', 'height', 'creation_height', 'activation_height', 'expiration_height',
|
||||
'release_time', 'short_url', 'canonical_url', 'title', 'author', 'description', 'claim_type', 'reposted',
|
||||
'stream_type', 'media_type', 'fee_amount', 'fee_currency', 'duration', 'reposted_claim_hash', 'censor_type',
|
||||
'claims_in_channel', 'channel_join', 'signature_valid', 'effective_amount', 'support_amount',
|
||||
'trending_group', 'trending_mixed', 'trending_local', 'trending_global', 'channel_id', 'tx_id', 'tx_nout',
|
||||
'signature', 'signature_digest', 'public_key_bytes', 'public_key_hash', 'public_key_id', '_id', 'tags',
|
||||
'reposted_claim_id'}
|
||||
TEXT_FIELDS = {'author', 'canonical_url', 'channel_id', 'claim_name', 'description', 'claim_id',
|
||||
'media_type', 'normalized', 'public_key_bytes', 'public_key_hash', 'short_url', 'signature',
|
||||
'signature_digest', 'stream_type', 'title', 'tx_id', 'fee_currency', 'reposted_claim_id', 'tags'}
|
||||
|
||||
FIELDS = {
|
||||
'_id',
|
||||
'claim_id', 'claim_type', 'claim_name', 'normalized_name',
|
||||
'tx_id', 'tx_nout', 'tx_position',
|
||||
'short_url', 'canonical_url',
|
||||
'is_controlling', 'last_take_over_height',
|
||||
'public_key_bytes', 'public_key_id', 'claims_in_channel', 'channel_join_height',
|
||||
'channel_id', 'signature', 'signature_digest', 'is_signature_valid',
|
||||
'amount', 'effective_amount', 'support_amount',
|
||||
'fee_amount', 'fee_currency',
|
||||
'height', 'creation_height', 'activation_height', 'expiration_height',
|
||||
'stream_type', 'media_type', 'censor_type',
|
||||
'title', 'author', 'description',
|
||||
'timestamp', 'creation_timestamp',
|
||||
'duration', 'release_time',
|
||||
'tags', 'languages', 'has_source', 'reposted_claim_type',
|
||||
'reposted_claim_id', 'repost_count',
|
||||
'trending_group', 'trending_mixed', 'trending_local', 'trending_global',
|
||||
}
|
||||
|
||||
TEXT_FIELDS = {'author', 'canonical_url', 'channel_id', 'claim_name', 'description', 'claim_id', 'censoring_channel_id',
|
||||
'media_type', 'normalized_name', 'public_key_bytes', 'public_key_id', 'short_url', 'signature',
|
||||
'signature_digest', 'title', 'tx_id', 'fee_currency', 'reposted_claim_id', 'tags'}
|
||||
|
||||
RANGE_FIELDS = {
|
||||
'height', 'creation_height', 'activation_height', 'expiration_height',
|
||||
'timestamp', 'creation_timestamp', 'duration', 'release_time', 'fee_amount',
|
||||
'tx_position', 'channel_join', 'reposted', 'limit_claims_per_channel',
|
||||
'tx_position', 'channel_join', 'repost_count', 'limit_claims_per_channel',
|
||||
'amount', 'effective_amount', 'support_amount',
|
||||
'trending_group', 'trending_mixed', 'censor_type',
|
||||
'trending_local', 'trending_global',
|
||||
}
|
||||
|
||||
ALL_FIELDS = RANGE_FIELDS | TEXT_FIELDS | FIELDS
|
||||
|
||||
REPLACEMENTS = {
|
||||
'name': 'normalized',
|
||||
'name': 'normalized_name',
|
||||
'txid': 'tx_id',
|
||||
'claim_hash': '_id'
|
||||
'nout': 'tx_nout',
|
||||
'valid_channel_signature': 'is_signature_valid',
|
||||
'stream_types': 'stream_type',
|
||||
'media_types': 'media_type',
|
||||
'reposted': 'repost_count'
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import asyncio
|
||||
import struct
|
||||
from binascii import unhexlify
|
||||
from collections import Counter, deque
|
||||
from decimal import Decimal
|
||||
from operator import itemgetter
|
||||
from typing import Optional, List, Iterable, Union
|
||||
@@ -9,14 +10,14 @@ from elasticsearch import AsyncElasticsearch, NotFoundError, ConnectionError
|
||||
from elasticsearch.helpers import async_streaming_bulk
|
||||
|
||||
from lbry.crypto.base58 import Base58
|
||||
from lbry.error import ResolveCensoredError, claim_id as parse_claim_id
|
||||
from lbry.error import ResolveCensoredError, TooManyClaimSearchParametersError
|
||||
from lbry.schema.result import Outputs, Censor
|
||||
from lbry.schema.tags import clean_tags
|
||||
from lbry.schema.url import URL, normalize_name
|
||||
from lbry.utils import LRUCache
|
||||
from lbry.wallet.server.db.common import CLAIM_TYPES, STREAM_TYPES
|
||||
from lbry.wallet.server.db.elasticsearch.constants import INDEX_DEFAULT_SETTINGS, REPLACEMENTS, FIELDS, TEXT_FIELDS, \
|
||||
RANGE_FIELDS
|
||||
RANGE_FIELDS, ALL_FIELDS
|
||||
from lbry.wallet.server.util import class_logger
|
||||
|
||||
|
||||
@@ -32,7 +33,15 @@ class StreamResolution(str):
|
||||
return LookupError(f'Could not find claim at "{url}".')
|
||||
|
||||
|
||||
class IndexVersionMismatch(Exception):
|
||||
def __init__(self, got_version, expected_version):
|
||||
self.got_version = got_version
|
||||
self.expected_version = expected_version
|
||||
|
||||
|
||||
class SearchIndex:
|
||||
VERSION = 1
|
||||
|
||||
def __init__(self, index_prefix: str, search_timeout=3.0, elastic_host='localhost', elastic_port=9200):
|
||||
self.search_timeout = search_timeout
|
||||
self.sync_timeout = 600 # wont hit that 99% of the time, but can hit on a fresh import
|
||||
@@ -41,28 +50,48 @@ class SearchIndex:
|
||||
self.index = index_prefix + 'claims'
|
||||
self.logger = class_logger(__name__, self.__class__.__name__)
|
||||
self.claim_cache = LRUCache(2 ** 15)
|
||||
self.short_id_cache = LRUCache(2 ** 17) # never invalidated, since short ids are forever
|
||||
self.short_id_cache = LRUCache(2 ** 17)
|
||||
self.search_cache = LRUCache(2 ** 17)
|
||||
self.resolution_cache = LRUCache(2 ** 17)
|
||||
self._elastic_host = elastic_host
|
||||
self._elastic_port = elastic_port
|
||||
|
||||
async def start(self):
|
||||
async def get_index_version(self) -> int:
|
||||
try:
|
||||
template = await self.sync_client.indices.get_template(self.index)
|
||||
return template[self.index]['version']
|
||||
except NotFoundError:
|
||||
return 0
|
||||
|
||||
async def set_index_version(self, version):
|
||||
await self.sync_client.indices.put_template(
|
||||
self.index, body={'version': version, 'index_patterns': ['ignored']}, ignore=400
|
||||
)
|
||||
|
||||
async def start(self) -> bool:
|
||||
if self.sync_client:
|
||||
return
|
||||
return False
|
||||
hosts = [{'host': self._elastic_host, 'port': self._elastic_port}]
|
||||
self.sync_client = AsyncElasticsearch(hosts, timeout=self.sync_timeout)
|
||||
self.search_client = AsyncElasticsearch(hosts, timeout=self.search_timeout)
|
||||
while True:
|
||||
self.logger.info("DOITDOITDOIT\n\n\n\n%s:%i\n\n\n", self._elastic_host, self._elastic_port)
|
||||
try:
|
||||
await self.sync_client.cluster.health(wait_for_status='yellow')
|
||||
break
|
||||
except ConnectionError:
|
||||
self.logger.warning("Failed to connect to Elasticsearch. Waiting for it!")
|
||||
await asyncio.sleep(1)
|
||||
|
||||
res = await self.sync_client.indices.create(self.index, INDEX_DEFAULT_SETTINGS, ignore=400)
|
||||
return res.get('acknowledged', False)
|
||||
acked = res.get('acknowledged', False)
|
||||
if acked:
|
||||
await self.set_index_version(self.VERSION)
|
||||
return acked
|
||||
index_version = await self.get_index_version()
|
||||
if index_version != self.VERSION:
|
||||
self.logger.error("es search index has an incompatible version: %s vs %s", index_version, self.VERSION)
|
||||
raise IndexVersionMismatch(index_version, self.VERSION)
|
||||
return acked
|
||||
|
||||
def stop(self):
|
||||
clients = [self.sync_client, self.search_client]
|
||||
@@ -104,7 +133,7 @@ class SearchIndex:
|
||||
update = expand_query(claim_id__in=list(blockdict.keys()), censor_type=f"<{censor_type}")
|
||||
key = 'channel_id' if channels else 'claim_id'
|
||||
update['script'] = {
|
||||
"source": f"ctx._source.censor_type={censor_type}; ctx._source.censoring_channel_hash=params[ctx._source.{key}]",
|
||||
"source": f"ctx._source.censor_type={censor_type}; ctx._source.censoring_channel_id=params[ctx._source.{key}]",
|
||||
"lang": "painless",
|
||||
"params": blockdict
|
||||
}
|
||||
@@ -133,7 +162,11 @@ class SearchIndex:
|
||||
await self.sync_client.update_by_query(
|
||||
self.index, body=self.update_filter_query(Censor.RESOLVE, blocked_channels, True), slices=4)
|
||||
await self.sync_client.indices.refresh(self.index)
|
||||
self.clear_caches()
|
||||
|
||||
def clear_caches(self):
|
||||
self.search_cache.clear()
|
||||
self.short_id_cache.clear()
|
||||
self.claim_cache.clear()
|
||||
self.resolution_cache.clear()
|
||||
|
||||
@@ -175,7 +208,7 @@ class SearchIndex:
|
||||
|
||||
censored = [
|
||||
result if not isinstance(result, dict) or not censor.censor(result)
|
||||
else ResolveCensoredError(url, result['censoring_channel_hash'])
|
||||
else ResolveCensoredError(url, result['censoring_channel_id'])
|
||||
for url, result in zip(urls, results)
|
||||
]
|
||||
return results, censored, censor
|
||||
@@ -198,7 +231,7 @@ class SearchIndex:
|
||||
self.claim_cache.set(result['claim_id'], result)
|
||||
|
||||
async def full_id_from_short_id(self, name, short_id, channel_id=None):
|
||||
key = (channel_id or '') + name + short_id
|
||||
key = '#'.join((channel_id or '', name, short_id))
|
||||
if key not in self.short_id_cache:
|
||||
query = {'name': name, 'claim_id': short_id}
|
||||
if channel_id:
|
||||
@@ -219,13 +252,89 @@ class SearchIndex:
|
||||
if not kwargs['channel_id'] or not isinstance(kwargs['channel_id'], str):
|
||||
return [], 0, 0
|
||||
try:
|
||||
result = (await self.search_client.search(
|
||||
expand_query(**kwargs), index=self.index, track_total_hits=False if kwargs.get('no_totals') else 10_000
|
||||
))['hits']
|
||||
return await self.search_ahead(**kwargs)
|
||||
except NotFoundError:
|
||||
return [], 0, 0
|
||||
return expand_result(result['hits']), 0, result.get('total', {}).get('value', 0)
|
||||
|
||||
async def search_ahead(self, **kwargs):
|
||||
# 'limit_claims_per_channel' case. Fetch 1000 results, reorder, slice, inflate and return
|
||||
per_channel_per_page = kwargs.pop('limit_claims_per_channel', 0) or 0
|
||||
remove_duplicates = kwargs.pop('remove_duplicates', False)
|
||||
page_size = kwargs.pop('limit', 10)
|
||||
offset = kwargs.pop('offset', 0)
|
||||
kwargs['limit'] = 1000
|
||||
cache_item = ResultCacheItem.from_cache(f"ahead{per_channel_per_page}{kwargs}", self.search_cache)
|
||||
if cache_item.result is not None:
|
||||
reordered_hits = cache_item.result
|
||||
else:
|
||||
async with cache_item.lock:
|
||||
if cache_item.result:
|
||||
reordered_hits = cache_item.result
|
||||
else:
|
||||
query = expand_query(**kwargs)
|
||||
search_hits = deque((await self.search_client.search(
|
||||
query, index=self.index, track_total_hits=False,
|
||||
_source_includes=['_id', 'channel_id', 'reposted_claim_id', 'creation_height']
|
||||
))['hits']['hits'])
|
||||
if remove_duplicates:
|
||||
search_hits = self.__remove_duplicates(search_hits)
|
||||
if per_channel_per_page > 0:
|
||||
reordered_hits = self.__search_ahead(search_hits, page_size, per_channel_per_page)
|
||||
else:
|
||||
reordered_hits = [(hit['_id'], hit['_source']['channel_id']) for hit in search_hits]
|
||||
cache_item.result = reordered_hits
|
||||
result = list(await self.get_many(*(claim_id for claim_id, _ in reordered_hits[offset:(offset + page_size)])))
|
||||
return result, 0, len(reordered_hits)
|
||||
|
||||
def __remove_duplicates(self, search_hits: deque) -> deque:
|
||||
known_ids = {} # claim_id -> (creation_height, hit_id), where hit_id is either reposted claim id or original
|
||||
dropped = set()
|
||||
for hit in search_hits:
|
||||
hit_height, hit_id = hit['_source']['creation_height'], hit['_source']['reposted_claim_id'] or hit['_id']
|
||||
if hit_id not in known_ids:
|
||||
known_ids[hit_id] = (hit_height, hit['_id'])
|
||||
else:
|
||||
previous_height, previous_id = known_ids[hit_id]
|
||||
if hit_height < previous_height:
|
||||
known_ids[hit_id] = (hit_height, hit['_id'])
|
||||
dropped.add(previous_id)
|
||||
else:
|
||||
dropped.add(hit['_id'])
|
||||
return deque(hit for hit in search_hits if hit['_id'] not in dropped)
|
||||
|
||||
def __search_ahead(self, search_hits: list, page_size: int, per_channel_per_page: int):
|
||||
reordered_hits = []
|
||||
channel_counters = Counter()
|
||||
next_page_hits_maybe_check_later = deque()
|
||||
while search_hits or next_page_hits_maybe_check_later:
|
||||
if reordered_hits and len(reordered_hits) % page_size == 0:
|
||||
channel_counters.clear()
|
||||
elif not reordered_hits:
|
||||
pass
|
||||
else:
|
||||
break # means last page was incomplete and we are left with bad replacements
|
||||
for _ in range(len(next_page_hits_maybe_check_later)):
|
||||
claim_id, channel_id = next_page_hits_maybe_check_later.popleft()
|
||||
if per_channel_per_page > 0 and channel_counters[channel_id] < per_channel_per_page:
|
||||
reordered_hits.append((claim_id, channel_id))
|
||||
channel_counters[channel_id] += 1
|
||||
else:
|
||||
next_page_hits_maybe_check_later.append((claim_id, channel_id))
|
||||
while search_hits:
|
||||
hit = search_hits.popleft()
|
||||
hit_id, hit_channel_id = hit['_id'], hit['_source']['channel_id']
|
||||
if hit_channel_id is None or per_channel_per_page <= 0:
|
||||
reordered_hits.append((hit_id, hit_channel_id))
|
||||
elif channel_counters[hit_channel_id] < per_channel_per_page:
|
||||
reordered_hits.append((hit_id, hit_channel_id))
|
||||
channel_counters[hit_channel_id] += 1
|
||||
if len(reordered_hits) % page_size == 0:
|
||||
break
|
||||
else:
|
||||
next_page_hits_maybe_check_later.append((hit_id, hit_channel_id))
|
||||
return reordered_hits
|
||||
|
||||
async def resolve_url(self, raw_url):
|
||||
if raw_url not in self.resolution_cache:
|
||||
self.resolution_cache[raw_url] = await self._resolve_url(raw_url)
|
||||
@@ -302,7 +411,7 @@ class SearchIndex:
|
||||
txo_rows = [row for row in txo_rows if isinstance(row, dict)]
|
||||
referenced_ids = set(filter(None, map(itemgetter('reposted_claim_id'), txo_rows)))
|
||||
referenced_ids |= set(filter(None, (row['channel_id'] for row in txo_rows)))
|
||||
referenced_ids |= set(map(parse_claim_id, filter(None, (row['censoring_channel_hash'] for row in txo_rows))))
|
||||
referenced_ids |= set(filter(None, (row['censoring_channel_id'] for row in txo_rows)))
|
||||
|
||||
referenced_txos = []
|
||||
if referenced_ids:
|
||||
@@ -323,20 +432,22 @@ def extract_doc(doc, index):
|
||||
doc['reposted_claim_id'] = None
|
||||
channel_hash = doc.pop('channel_hash')
|
||||
doc['channel_id'] = channel_hash[::-1].hex() if channel_hash else channel_hash
|
||||
channel_hash = doc.pop('censoring_channel_hash')
|
||||
doc['censoring_channel_hash'] = channel_hash[::-1].hex() if channel_hash else channel_hash
|
||||
doc['censoring_channel_id'] = doc.get('censoring_channel_id')
|
||||
txo_hash = doc.pop('txo_hash')
|
||||
doc['tx_id'] = txo_hash[:32][::-1].hex()
|
||||
doc['tx_nout'] = struct.unpack('<I', txo_hash[32:])[0]
|
||||
doc['repost_count'] = doc.pop('reposted')
|
||||
doc['is_controlling'] = bool(doc['is_controlling'])
|
||||
doc['signature'] = (doc.pop('signature') or b'').hex() or None
|
||||
doc['signature_digest'] = (doc.pop('signature_digest') or b'').hex() or None
|
||||
doc['public_key_bytes'] = (doc.pop('public_key_bytes') or b'').hex() or None
|
||||
doc['public_key_hash'] = (doc.pop('public_key_hash') or b'').hex() or None
|
||||
doc['signature_valid'] = bool(doc['signature_valid'])
|
||||
doc['public_key_id'] = (doc.pop('public_key_hash') or b'').hex() or None
|
||||
doc['is_signature_valid'] = bool(doc['signature_valid'])
|
||||
doc['claim_type'] = doc.get('claim_type', 0) or 0
|
||||
doc['stream_type'] = int(doc.get('stream_type', 0) or 0)
|
||||
doc['has_source'] = bool(doc['has_source'])
|
||||
doc['normalized_name'] = doc.pop('normalized')
|
||||
doc = {key: value for key, value in doc.items() if key in ALL_FIELDS}
|
||||
return {'doc': doc, '_id': doc['claim_id'], '_index': index, '_op_type': 'update', 'doc_as_upsert': True}
|
||||
|
||||
|
||||
@@ -354,6 +465,8 @@ def expand_query(**kwargs):
|
||||
for key, value in kwargs.items():
|
||||
key = key.replace('claim.', '')
|
||||
many = key.endswith('__in') or isinstance(value, list)
|
||||
if many and len(value) > 2048:
|
||||
raise TooManyClaimSearchParametersError(key, 2048)
|
||||
if many:
|
||||
key = key.replace('__in', '')
|
||||
value = list(filter(None, value))
|
||||
@@ -367,6 +480,8 @@ def expand_query(**kwargs):
|
||||
value = CLAIM_TYPES[value]
|
||||
else:
|
||||
value = [CLAIM_TYPES[claim_type] for claim_type in value]
|
||||
elif key == 'stream_type':
|
||||
value = STREAM_TYPES[value] if isinstance(value, str) else list(map(STREAM_TYPES.get, value))
|
||||
if key == '_id':
|
||||
if isinstance(value, Iterable):
|
||||
value = [item[::-1].hex() for item in value]
|
||||
@@ -375,9 +490,8 @@ def expand_query(**kwargs):
|
||||
if not many and key in ('_id', 'claim_id') and len(value) < 20:
|
||||
partial_id = True
|
||||
if key == 'public_key_id':
|
||||
key = 'public_key_hash'
|
||||
value = Base58.decode(value)[1:21].hex()
|
||||
if key == 'signature_valid':
|
||||
if key in ('signature_valid', 'has_source'):
|
||||
continue # handled later
|
||||
if key in TEXT_FIELDS:
|
||||
key += '.keyword'
|
||||
@@ -406,8 +520,6 @@ def expand_query(**kwargs):
|
||||
query['must'].append({"terms": {'claim_id.keyword': value}})
|
||||
elif key == 'media_types':
|
||||
query['must'].append({"terms": {'media_type.keyword': value}})
|
||||
elif key == 'stream_types':
|
||||
query['must'].append({"terms": {'stream_type': [STREAM_TYPES[stype] for stype in value]}})
|
||||
elif key == 'any_languages':
|
||||
query['must'].append({"terms": {'languages': clean_tags(value)}})
|
||||
elif key == 'any_languages':
|
||||
@@ -427,12 +539,12 @@ def expand_query(**kwargs):
|
||||
if kwargs.get('has_channel_signature'):
|
||||
query['must'].append({"exists": {"field": "signature_digest"}})
|
||||
if 'signature_valid' in kwargs:
|
||||
query['must'].append({"term": {"signature_valid": bool(kwargs["signature_valid"])}})
|
||||
query['must'].append({"term": {"is_signature_valid": bool(kwargs["signature_valid"])}})
|
||||
elif 'signature_valid' in kwargs:
|
||||
query.setdefault('should', [])
|
||||
query["minimum_should_match"] = 1
|
||||
query['should'].append({"bool": {"must_not": {"exists": {"field": "signature_digest"}}}})
|
||||
query['should'].append({"term": {"signature_valid": bool(kwargs["signature_valid"])}})
|
||||
query['should'].append({"term": {"is_signature_valid": bool(kwargs["signature_valid"])}})
|
||||
if 'has_source' in kwargs:
|
||||
query.setdefault('should', [])
|
||||
query["minimum_should_match"] = 1
|
||||
@@ -440,6 +552,7 @@ def expand_query(**kwargs):
|
||||
query['should'].append(
|
||||
{"bool": {"must": [{"match": {"has_source": kwargs['has_source']}}, is_stream_or_repost]}})
|
||||
query['should'].append({"bool": {"must_not": [is_stream_or_repost]}})
|
||||
query['should'].append({"bool": {"must": [{"term": {"reposted_claim_type": CLAIM_TYPES['channel']}}]}})
|
||||
if kwargs.get('text'):
|
||||
query['must'].append(
|
||||
{"simple_query_string":
|
||||
@@ -497,8 +610,9 @@ def expand_result(results):
|
||||
result['channel_hash'] = unhexlify(result['channel_id'])[::-1] if result['channel_id'] else None
|
||||
result['txo_hash'] = unhexlify(result['tx_id'])[::-1] + struct.pack('<I', result['tx_nout'])
|
||||
result['tx_hash'] = unhexlify(result['tx_id'])[::-1]
|
||||
if result['censoring_channel_hash']:
|
||||
result['censoring_channel_hash'] = unhexlify(result['censoring_channel_hash'])[::-1]
|
||||
result['reposted'] = result.pop('repost_count')
|
||||
result['signature_valid'] = result.pop('is_signature_valid')
|
||||
result['normalized'] = result.pop('normalized_name')
|
||||
expanded.append(result)
|
||||
if inner_hits:
|
||||
return expand_result(inner_hits)
|
||||
|
||||
@@ -5,84 +5,96 @@ import os
|
||||
from collections import namedtuple
|
||||
from multiprocessing import Process
|
||||
|
||||
import apsw
|
||||
import sqlite3
|
||||
from elasticsearch import AsyncElasticsearch
|
||||
from elasticsearch.helpers import async_bulk
|
||||
from lbry.wallet.server.env import Env
|
||||
from lbry.wallet.server.coin import LBC
|
||||
from lbry.wallet.server.db.elasticsearch.search import extract_doc, SearchIndex
|
||||
|
||||
INDEX = 'claims'
|
||||
from lbry.wallet.server.db.elasticsearch.search import extract_doc, SearchIndex, IndexVersionMismatch
|
||||
|
||||
|
||||
async def get_all(db, shard_num, shards_total, limit=0):
|
||||
async def get_all(db, shard_num, shards_total, limit=0, index_name='claims'):
|
||||
logging.info("shard %d starting", shard_num)
|
||||
def exec_factory(cursor, statement, bindings):
|
||||
tpl = namedtuple('row', (d[0] for d in cursor.getdescription()))
|
||||
cursor.setrowtrace(lambda cursor, row: tpl(*row))
|
||||
return True
|
||||
|
||||
db.setexectrace(exec_factory)
|
||||
def namedtuple_factory(cursor, row):
|
||||
Row = namedtuple('Row', (d[0] for d in cursor.description))
|
||||
return Row(*row)
|
||||
db.row_factory = namedtuple_factory
|
||||
total = db.execute(f"select count(*) as total from claim where height % {shards_total} = {shard_num};").fetchone()[0]
|
||||
for num, claim in enumerate(db.execute(f"""
|
||||
SELECT claimtrie.claim_hash as is_controlling,
|
||||
claimtrie.last_take_over_height,
|
||||
(select group_concat(tag, ',,') from tag where tag.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as tags,
|
||||
(select group_concat(language, ' ') from language where language.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as languages,
|
||||
(select cr.has_source from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_has_source,
|
||||
claim.*
|
||||
FROM claim LEFT JOIN claimtrie USING (claim_hash)
|
||||
WHERE claim.height % {shards_total} = {shard_num}
|
||||
ORDER BY claim.height desc
|
||||
SELECT claimtrie.claim_hash as is_controlling,
|
||||
claimtrie.last_take_over_height,
|
||||
(select group_concat(tag, ',,') from tag where tag.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as tags,
|
||||
(select group_concat(language, ' ') from language where language.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as languages,
|
||||
cr.has_source as reposted_has_source,
|
||||
cr.claim_type as reposted_claim_type,
|
||||
cr.stream_type as reposted_stream_type,
|
||||
cr.media_type as reposted_media_type,
|
||||
cr.duration as reposted_duration,
|
||||
cr.fee_amount as reposted_fee_amount,
|
||||
cr.fee_currency as reposted_fee_currency,
|
||||
claim.*
|
||||
FROM claim LEFT JOIN claimtrie USING (claim_hash) LEFT JOIN claim cr ON cr.claim_hash=claim.reposted_claim_hash
|
||||
WHERE claim.height % {shards_total} = {shard_num}
|
||||
ORDER BY claim.height desc
|
||||
""")):
|
||||
claim = dict(claim._asdict())
|
||||
claim['has_source'] = bool(claim.pop('reposted_has_source') or claim['has_source'])
|
||||
claim['stream_type'] = claim.pop('reposted_stream_type') or claim['stream_type']
|
||||
claim['media_type'] = claim.pop('reposted_media_type') or claim['media_type']
|
||||
claim['fee_amount'] = claim.pop('reposted_fee_amount') or claim['fee_amount']
|
||||
claim['fee_currency'] = claim.pop('reposted_fee_currency') or claim['fee_currency']
|
||||
claim['duration'] = claim.pop('reposted_duration') or claim['duration']
|
||||
claim['censor_type'] = 0
|
||||
claim['censoring_channel_hash'] = None
|
||||
claim['censoring_channel_id'] = None
|
||||
claim['tags'] = claim['tags'].split(',,') if claim['tags'] else []
|
||||
claim['languages'] = claim['languages'].split(' ') if claim['languages'] else []
|
||||
if num % 10_000 == 0:
|
||||
logging.info("%d/%d", num, total)
|
||||
yield extract_doc(claim, INDEX)
|
||||
yield extract_doc(claim, index_name)
|
||||
if 0 < limit <= num:
|
||||
break
|
||||
|
||||
|
||||
async def consume(producer):
|
||||
async def consume(producer, index_name):
|
||||
env = Env(LBC)
|
||||
logging.info("ES sync host: %s:%i", env.elastic_host, env.elastic_port)
|
||||
es = AsyncElasticsearch([{'host': env.elastic_host, 'port': env.elastic_port}])
|
||||
try:
|
||||
await async_bulk(es, producer, request_timeout=120)
|
||||
await es.indices.refresh(index=INDEX)
|
||||
await es.indices.refresh(index=index_name)
|
||||
finally:
|
||||
await es.close()
|
||||
|
||||
|
||||
async def make_es_index():
|
||||
async def make_es_index(index=None):
|
||||
env = Env(LBC)
|
||||
index = SearchIndex('', elastic_host=env.elastic_host, elastic_port=env.elastic_port)
|
||||
if index is None:
|
||||
index = SearchIndex('', elastic_host=env.elastic_host, elastic_port=env.elastic_port)
|
||||
|
||||
try:
|
||||
return await index.start()
|
||||
except IndexVersionMismatch as err:
|
||||
logging.info(
|
||||
"dropping ES search index (version %s) for upgrade to version %s", err.got_version, err.expected_version
|
||||
)
|
||||
await index.delete_index()
|
||||
await index.stop()
|
||||
return await index.start()
|
||||
finally:
|
||||
index.stop()
|
||||
|
||||
|
||||
async def run(args, shard):
|
||||
def itsbusy(*_):
|
||||
logging.info("shard %d: db is busy, retry", shard)
|
||||
return True
|
||||
db = apsw.Connection(args.db_path, flags=apsw.SQLITE_OPEN_READONLY | apsw.SQLITE_OPEN_URI)
|
||||
db.setbusyhandler(itsbusy)
|
||||
db.cursor().execute('pragma journal_mode=wal;')
|
||||
db.cursor().execute('pragma temp_store=memory;')
|
||||
|
||||
producer = get_all(db.cursor(), shard, args.clients, limit=args.blocks)
|
||||
await asyncio.gather(*(consume(producer) for _ in range(min(8, args.clients))))
|
||||
async def run(db_path, clients, blocks, shard, index_name='claims'):
|
||||
db = sqlite3.connect(db_path, isolation_level=None, check_same_thread=False, uri=True)
|
||||
db.execute('pragma journal_mode=wal;')
|
||||
db.execute('pragma temp_store=memory;')
|
||||
producer = get_all(db, shard, clients, limit=blocks, index_name=index_name)
|
||||
await asyncio.gather(*(consume(producer, index_name=index_name) for _ in range(min(8, clients))))
|
||||
|
||||
|
||||
def __run(args, shard):
|
||||
asyncio.run(run(args, shard))
|
||||
asyncio.run(run(args.db_path, args.clients, args.blocks, shard))
|
||||
|
||||
|
||||
def run_elastic_sync():
|
||||
|
||||
@@ -231,7 +231,7 @@ def run(db, height, final_height, recalculate_claim_hashes):
|
||||
FROM claim
|
||||
WHERE claim_hash IN
|
||||
({','.join('?' for _ in recalculate_claim_hashes)});
|
||||
""", recalculate_claim_hashes):
|
||||
""", list(recalculate_claim_hashes)):
|
||||
trending_data.update_claim(row[0], 1E-8*row[1], time_boost)
|
||||
|
||||
trending_log("done.\n")
|
||||
|
||||
@@ -5,7 +5,7 @@ decay rate for high valued claims.
|
||||
|
||||
import math
|
||||
import time
|
||||
import apsw
|
||||
import sqlite3
|
||||
|
||||
# Half life in blocks *for lower LBC claims* (it's shorter for whale claims)
|
||||
HALF_LIFE = 200
|
||||
@@ -96,16 +96,16 @@ class TrendingDB:
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.conn = apsw.Connection(":memory:")
|
||||
self.conn = sqlite3.connect(":memory:", check_same_thread=False)
|
||||
self.cursor = self.conn.cursor()
|
||||
self.initialised = False
|
||||
self.write_needed = set()
|
||||
|
||||
def execute(self, query, *args, **kwargs):
|
||||
return self.cursor.execute(query, *args, **kwargs)
|
||||
return self.conn.execute(query, *args, **kwargs)
|
||||
|
||||
def executemany(self, query, *args, **kwargs):
|
||||
return self.cursor.executemany(query, *args, **kwargs)
|
||||
return self.conn.executemany(query, *args, **kwargs)
|
||||
|
||||
def begin(self):
|
||||
self.execute("BEGIN;")
|
||||
@@ -251,7 +251,7 @@ class TrendingDB:
|
||||
FROM claims
|
||||
WHERE claim_hash IN
|
||||
({','.join('?' for _ in self.write_needed)});
|
||||
""", self.write_needed).fetchall()
|
||||
""", list(self.write_needed)).fetchall()
|
||||
|
||||
db.executemany("""UPDATE claim SET trending_mixed = ?
|
||||
WHERE claim_hash = ?;""", rows)
|
||||
@@ -278,7 +278,7 @@ class TrendingDB:
|
||||
FROM claim
|
||||
WHERE claim_hash IN
|
||||
({','.join('?' for _ in recalculate_claim_hashes)});
|
||||
""", recalculate_claim_hashes):
|
||||
""", list(recalculate_claim_hashes)):
|
||||
claim_hash, lbc = row
|
||||
|
||||
# Insert into trending db if it does not exist
|
||||
|
||||
@@ -47,14 +47,10 @@ class ZScore:
|
||||
return self.last
|
||||
return (self.last - self.mean) / (self.standard_deviation or 1)
|
||||
|
||||
@classmethod
|
||||
def factory(cls):
|
||||
return cls(), cls.step, cls.finalize
|
||||
|
||||
|
||||
def install(connection):
|
||||
connection.createaggregatefunction("zscore", ZScore.factory, 1)
|
||||
connection.cursor().execute(CREATE_TREND_TABLE)
|
||||
connection.create_aggregate("zscore", 1, ZScore)
|
||||
connection.executescript(CREATE_TREND_TABLE)
|
||||
|
||||
|
||||
def run(db, height, final_height, affected_claims):
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
import os
|
||||
|
||||
import apsw
|
||||
import sqlite3
|
||||
from typing import Union, Tuple, Set, List
|
||||
from itertools import chain
|
||||
from decimal import Decimal
|
||||
from collections import namedtuple
|
||||
from multiprocessing import Manager
|
||||
from binascii import unhexlify, hexlify
|
||||
from lbry.wallet.server.leveldb import LevelDB
|
||||
from lbry.wallet.server.util import class_logger
|
||||
@@ -22,6 +21,7 @@ from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS, INDEXED_LANGUAGES
|
||||
from lbry.wallet.server.db.elasticsearch import SearchIndex
|
||||
|
||||
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
|
||||
sqlite3.enable_callback_tracebacks(True)
|
||||
|
||||
|
||||
class SQLDB:
|
||||
@@ -220,7 +220,6 @@ class SQLDB:
|
||||
self.db = None
|
||||
self.logger = class_logger(__name__, self.__class__.__name__)
|
||||
self.ledger = Ledger if main.coin.NET == 'mainnet' else RegTestLedger
|
||||
self.state_manager = None
|
||||
self.blocked_streams = None
|
||||
self.blocked_channels = None
|
||||
self.blocking_channel_hashes = {
|
||||
@@ -235,27 +234,19 @@ class SQLDB:
|
||||
self.pending_deletes = set()
|
||||
|
||||
def open(self):
|
||||
self.db = apsw.Connection(
|
||||
self._db_path,
|
||||
flags=(
|
||||
apsw.SQLITE_OPEN_READWRITE |
|
||||
apsw.SQLITE_OPEN_CREATE |
|
||||
apsw.SQLITE_OPEN_URI
|
||||
)
|
||||
)
|
||||
def exec_factory(cursor, statement, bindings):
|
||||
tpl = namedtuple('row', (d[0] for d in cursor.getdescription()))
|
||||
cursor.setrowtrace(lambda cursor, row: tpl(*row))
|
||||
return True
|
||||
self.db.setexectrace(exec_factory)
|
||||
self.execute(self.PRAGMAS)
|
||||
self.execute(self.CREATE_TABLES_QUERY)
|
||||
self.db = sqlite3.connect(self._db_path, isolation_level=None, check_same_thread=False, uri=True)
|
||||
|
||||
def namedtuple_factory(cursor, row):
|
||||
Row = namedtuple('Row', (d[0] for d in cursor.description))
|
||||
return Row(*row)
|
||||
self.db.row_factory = namedtuple_factory
|
||||
self.db.executescript(self.PRAGMAS)
|
||||
self.db.executescript(self.CREATE_TABLES_QUERY)
|
||||
register_canonical_functions(self.db)
|
||||
self.state_manager = Manager()
|
||||
self.blocked_streams = self.state_manager.dict()
|
||||
self.blocked_channels = self.state_manager.dict()
|
||||
self.filtered_streams = self.state_manager.dict()
|
||||
self.filtered_channels = self.state_manager.dict()
|
||||
self.blocked_streams = {}
|
||||
self.blocked_channels = {}
|
||||
self.filtered_streams = {}
|
||||
self.filtered_channels = {}
|
||||
self.update_blocked_and_filtered_claims()
|
||||
for algorithm in self.trending:
|
||||
algorithm.install(self.db)
|
||||
@@ -263,8 +254,6 @@ class SQLDB:
|
||||
def close(self):
|
||||
if self.db is not None:
|
||||
self.db.close()
|
||||
if self.state_manager is not None:
|
||||
self.state_manager.shutdown()
|
||||
|
||||
def update_blocked_and_filtered_claims(self):
|
||||
self.update_claims_from_channel_hashes(
|
||||
@@ -324,10 +313,10 @@ class SQLDB:
|
||||
return f"DELETE FROM {table} WHERE {where}", values
|
||||
|
||||
def execute(self, *args):
|
||||
return self.db.cursor().execute(*args)
|
||||
return self.db.execute(*args)
|
||||
|
||||
def executemany(self, *args):
|
||||
return self.db.cursor().executemany(*args)
|
||||
return self.db.executemany(*args)
|
||||
|
||||
def begin(self):
|
||||
self.execute('begin;')
|
||||
@@ -406,7 +395,8 @@ class SQLDB:
|
||||
if isinstance(fee.currency, str):
|
||||
claim_record['fee_currency'] = fee.currency.lower()
|
||||
if isinstance(fee.amount, Decimal):
|
||||
claim_record['fee_amount'] = int(fee.amount*1000)
|
||||
if fee.amount >= 0 and int(fee.amount*1000) < 9223372036854775807:
|
||||
claim_record['fee_amount'] = int(fee.amount*1000)
|
||||
elif claim.is_repost:
|
||||
claim_record['claim_type'] = CLAIM_TYPES['repost']
|
||||
claim_record['reposted_claim_hash'] = claim.repost.reference.claim_hash
|
||||
@@ -547,7 +537,7 @@ class SQLDB:
|
||||
WHERE claim_hash = ?
|
||||
""", targets
|
||||
)
|
||||
return set(target[0] for target in targets)
|
||||
return {target[0] for target in targets}
|
||||
|
||||
def validate_channel_signatures(self, height, new_claims, updated_claims, spent_claims, affected_channels, timer):
|
||||
if not new_claims and not updated_claims and not spent_claims:
|
||||
@@ -624,7 +614,7 @@ class SQLDB:
|
||||
channel_hash IN ({','.join('?' for _ in changed_channel_keys)}) AND
|
||||
signature IS NOT NULL
|
||||
"""
|
||||
for affected_claim in self.execute(sql, changed_channel_keys.keys()):
|
||||
for affected_claim in self.execute(sql, list(changed_channel_keys.keys())):
|
||||
if affected_claim.claim_hash not in signables:
|
||||
claim_updates.append({
|
||||
'claim_hash': affected_claim.claim_hash,
|
||||
@@ -681,7 +671,7 @@ class SQLDB:
|
||||
signature_valid=CASE WHEN signature IS NOT NULL THEN 0 END,
|
||||
channel_join=NULL, canonical_url=NULL
|
||||
WHERE channel_hash IN ({','.join('?' for _ in spent_claims)})
|
||||
""", spent_claims
|
||||
""", list(spent_claims)
|
||||
)
|
||||
sub_timer.stop()
|
||||
|
||||
@@ -714,7 +704,7 @@ class SQLDB:
|
||||
claim_in_channel.channel_hash=claim.claim_hash
|
||||
)
|
||||
WHERE claim_hash = ?
|
||||
""", [(channel_hash,) for channel_hash in all_channel_keys.keys()])
|
||||
""", [(channel_hash,) for channel_hash in all_channel_keys])
|
||||
sub_timer.stop()
|
||||
|
||||
sub_timer = timer.add_timer('update blocked claims list')
|
||||
@@ -806,12 +796,13 @@ class SQLDB:
|
||||
|
||||
def update_claimtrie(self, height, changed_claim_hashes, deleted_names, timer):
|
||||
r = timer.run
|
||||
binary_claim_hashes = list(changed_claim_hashes)
|
||||
|
||||
r(self._calculate_activation_height, height)
|
||||
r(self._update_support_amount, changed_claim_hashes)
|
||||
r(self._update_support_amount, binary_claim_hashes)
|
||||
|
||||
r(self._update_effective_amount, height, changed_claim_hashes)
|
||||
r(self._perform_overtake, height, changed_claim_hashes, list(deleted_names))
|
||||
r(self._update_effective_amount, height, binary_claim_hashes)
|
||||
r(self._perform_overtake, height, binary_claim_hashes, list(deleted_names))
|
||||
|
||||
r(self._update_effective_amount, height)
|
||||
r(self._perform_overtake, height, [], [])
|
||||
@@ -822,34 +813,47 @@ class SQLDB:
|
||||
)
|
||||
|
||||
def enqueue_changes(self):
|
||||
for claim in self.execute(f"""
|
||||
query = """
|
||||
SELECT claimtrie.claim_hash as is_controlling,
|
||||
claimtrie.last_take_over_height,
|
||||
(select group_concat(tag, ',,') from tag where tag.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as tags,
|
||||
(select group_concat(language, ' ') from language where language.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as languages,
|
||||
(select cr.has_source from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_has_source,
|
||||
cr.has_source as reposted_has_source,
|
||||
cr.claim_type as reposted_claim_type,
|
||||
cr.stream_type as reposted_stream_type,
|
||||
cr.media_type as reposted_media_type,
|
||||
cr.duration as reposted_duration,
|
||||
cr.fee_amount as reposted_fee_amount,
|
||||
cr.fee_currency as reposted_fee_currency,
|
||||
claim.*
|
||||
FROM claim LEFT JOIN claimtrie USING (claim_hash)
|
||||
FROM claim LEFT JOIN claimtrie USING (claim_hash) LEFT JOIN claim cr ON cr.claim_hash=claim.reposted_claim_hash
|
||||
WHERE claim.claim_hash in (SELECT claim_hash FROM changelog)
|
||||
"""):
|
||||
"""
|
||||
for claim in self.execute(query):
|
||||
claim = claim._asdict()
|
||||
id_set = set(filter(None, (claim['claim_hash'], claim['channel_hash'], claim['reposted_claim_hash'])))
|
||||
claim['censor_type'] = 0
|
||||
claim['censoring_channel_hash'] = None
|
||||
censoring_channel_hash = None
|
||||
claim['has_source'] = bool(claim.pop('reposted_has_source') or claim['has_source'])
|
||||
claim['stream_type'] = claim.pop('reposted_stream_type') or claim['stream_type']
|
||||
claim['media_type'] = claim.pop('reposted_media_type') or claim['media_type']
|
||||
claim['fee_amount'] = claim.pop('reposted_fee_amount') or claim['fee_amount']
|
||||
claim['fee_currency'] = claim.pop('reposted_fee_currency') or claim['fee_currency']
|
||||
claim['duration'] = claim.pop('reposted_duration') or claim['duration']
|
||||
for reason_id in id_set:
|
||||
if reason_id in self.blocked_streams:
|
||||
claim['censor_type'] = 2
|
||||
claim['censoring_channel_hash'] = self.blocked_streams.get(reason_id)
|
||||
censoring_channel_hash = self.blocked_streams.get(reason_id)
|
||||
elif reason_id in self.blocked_channels:
|
||||
claim['censor_type'] = 2
|
||||
claim['censoring_channel_hash'] = self.blocked_channels.get(reason_id)
|
||||
censoring_channel_hash = self.blocked_channels.get(reason_id)
|
||||
elif reason_id in self.filtered_streams:
|
||||
claim['censor_type'] = 1
|
||||
claim['censoring_channel_hash'] = self.filtered_streams.get(reason_id)
|
||||
censoring_channel_hash = self.filtered_streams.get(reason_id)
|
||||
elif reason_id in self.filtered_channels:
|
||||
claim['censor_type'] = 1
|
||||
claim['censoring_channel_hash'] = self.filtered_channels.get(reason_id)
|
||||
censoring_channel_hash = self.filtered_channels.get(reason_id)
|
||||
claim['censoring_channel_id'] = censoring_channel_hash[::-1].hex() if censoring_channel_hash else None
|
||||
|
||||
claim['tags'] = claim['tags'].split(',,') if claim['tags'] else []
|
||||
claim['languages'] = claim['languages'].split(' ') if claim['languages'] else []
|
||||
|
||||
@@ -73,9 +73,12 @@ class Env:
|
||||
self.tor_banner_file = self.default('TOR_BANNER_FILE', self.banner_file)
|
||||
self.anon_logs = self.boolean('ANON_LOGS', False)
|
||||
self.log_sessions = self.integer('LOG_SESSIONS', 3600)
|
||||
self.allow_lan_udp = self.boolean('ALLOW_LAN_UDP', False)
|
||||
self.country = self.default('COUNTRY', 'US')
|
||||
# Peer discovery
|
||||
self.peer_discovery = self.peer_discovery_enum()
|
||||
self.peer_announce = self.boolean('PEER_ANNOUNCE', True)
|
||||
self.peer_hubs = self.extract_peer_hubs()
|
||||
self.force_proxy = self.boolean('FORCE_PROXY', False)
|
||||
self.tor_proxy_host = self.default('TOR_PROXY_HOST', 'localhost')
|
||||
self.tor_proxy_port = self.integer('TOR_PROXY_PORT', None)
|
||||
@@ -268,3 +271,6 @@ class Env:
|
||||
return self.PD_SELF
|
||||
else:
|
||||
return self.PD_ON
|
||||
|
||||
def extract_peer_hubs(self):
|
||||
return [hub.strip() for hub in self.default('PEER_HUBS', '').split(',') if hub.strip()]
|
||||
@@ -16,13 +16,13 @@ from collections import defaultdict
|
||||
from functools import partial
|
||||
|
||||
from lbry.wallet.server import util
|
||||
from lbry.wallet.server.util import pack_be_uint16, unpack_be_uint16_from
|
||||
from lbry.wallet.server.util import pack_be_uint32, unpack_be_uint32_from, unpack_be_uint16_from
|
||||
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
|
||||
|
||||
|
||||
class History:
|
||||
|
||||
DB_VERSIONS = [0]
|
||||
DB_VERSIONS = [0, 1]
|
||||
|
||||
def __init__(self):
|
||||
self.logger = util.class_logger(__name__, self.__class__.__name__)
|
||||
@@ -32,9 +32,34 @@ class History:
|
||||
self.unflushed_count = 0
|
||||
self.db = None
|
||||
|
||||
@property
|
||||
def needs_migration(self):
|
||||
return self.db_version != max(self.DB_VERSIONS)
|
||||
|
||||
def migrate(self):
|
||||
# 0 -> 1: flush_count from 16 to 32 bits
|
||||
self.logger.warning("HISTORY MIGRATION IN PROGRESS. Please avoid shutting down before it finishes.")
|
||||
with self.db.write_batch() as batch:
|
||||
for key, value in self.db.iterator(prefix=b''):
|
||||
if len(key) != 13:
|
||||
continue
|
||||
flush_id, = unpack_be_uint16_from(key[-2:])
|
||||
new_key = key[:-2] + pack_be_uint32(flush_id)
|
||||
batch.put(new_key, value)
|
||||
self.logger.warning("history migration: new keys added, removing old ones.")
|
||||
for key, value in self.db.iterator(prefix=b''):
|
||||
if len(key) == 13:
|
||||
batch.delete(key)
|
||||
self.logger.warning("history migration: writing new state.")
|
||||
self.db_version = 1
|
||||
self.write_state(batch)
|
||||
self.logger.warning("history migration: done.")
|
||||
|
||||
def open_db(self, db_class, for_sync, utxo_flush_count, compacting):
|
||||
self.db = db_class('hist', for_sync)
|
||||
self.read_state()
|
||||
if self.needs_migration:
|
||||
self.migrate()
|
||||
self.clear_excess(utxo_flush_count)
|
||||
# An incomplete compaction needs to be cancelled otherwise
|
||||
# restarting it will corrupt the history
|
||||
@@ -81,7 +106,7 @@ class History:
|
||||
|
||||
keys = []
|
||||
for key, hist in self.db.iterator(prefix=b''):
|
||||
flush_id, = unpack_be_uint16_from(key[-2:])
|
||||
flush_id, = unpack_be_uint32_from(key[-4:])
|
||||
if flush_id > utxo_flush_count:
|
||||
keys.append(key)
|
||||
|
||||
@@ -126,7 +151,7 @@ class History:
|
||||
def flush(self):
|
||||
start_time = time.time()
|
||||
self.flush_count += 1
|
||||
flush_id = pack_be_uint16(self.flush_count)
|
||||
flush_id = pack_be_uint32(self.flush_count)
|
||||
unflushed = self.unflushed
|
||||
|
||||
with self.db.write_batch() as batch:
|
||||
@@ -250,7 +275,7 @@ class History:
|
||||
write_size = 0
|
||||
keys_to_delete.update(hist_map)
|
||||
for n, chunk in enumerate(util.chunks(full_hist, max_row_size)):
|
||||
key = hashX + pack_be_uint16(n)
|
||||
key = hashX + pack_be_uint32(n)
|
||||
if hist_map.get(key) == chunk:
|
||||
keys_to_delete.remove(key)
|
||||
else:
|
||||
@@ -301,8 +326,8 @@ class History:
|
||||
|
||||
# Loop over 2-byte prefixes
|
||||
cursor = self.comp_cursor
|
||||
while write_size < limit and cursor < 65536:
|
||||
prefix = pack_be_uint16(cursor)
|
||||
while write_size < limit and cursor < (1 << 32):
|
||||
prefix = pack_be_uint32(cursor)
|
||||
write_size += self._compact_prefix(prefix, write_items,
|
||||
keys_to_delete)
|
||||
cursor += 1
|
||||
|
||||
@@ -137,9 +137,7 @@ class LevelDB:
|
||||
return
|
||||
|
||||
def get_headers():
|
||||
return [
|
||||
header for header in self.headers_db.iterator(prefix=HEADER_PREFIX, include_key=False)
|
||||
]
|
||||
return list(self.headers_db.iterator(prefix=HEADER_PREFIX, include_key=False))
|
||||
|
||||
headers = await asyncio.get_event_loop().run_in_executor(self.executor, get_headers)
|
||||
assert len(headers) - 1 == self.db_height, f"{len(headers)} vs {self.db_height}"
|
||||
|
||||
@@ -324,7 +324,7 @@ class MemPool:
|
||||
for prevout in tx.prevouts
|
||||
if prevout[0] not in all_hashes)
|
||||
utxos = await self.api.lookup_utxos(prevouts)
|
||||
utxo_map = {prevout: utxo for prevout, utxo in zip(prevouts, utxos)}
|
||||
utxo_map = dict(zip(prevouts, utxos))
|
||||
|
||||
return self._accept_transactions(tx_map, utxo_map, touched)
|
||||
|
||||
|
||||
@@ -1,302 +0,0 @@
|
||||
# Copyright (c) 2017, Neil Booth
|
||||
#
|
||||
# All rights reserved.
|
||||
#
|
||||
# The MIT License (MIT)
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining
|
||||
# a copy of this software and associated documentation files (the
|
||||
# "Software"), to deal in the Software without restriction, including
|
||||
# without limitation the rights to use, copy, modify, merge, publish,
|
||||
# distribute, sublicense, and/or sell copies of the Software, and to
|
||||
# permit persons to whom the Software is furnished to do so, subject to
|
||||
# the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be
|
||||
# included in all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
||||
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
"""Representation of a peer server."""
|
||||
|
||||
from ipaddress import ip_address
|
||||
|
||||
from lbry.wallet.server import util
|
||||
from lbry.wallet.server.util import cachedproperty
|
||||
|
||||
from typing import Dict
|
||||
|
||||
|
||||
class Peer:
|
||||
|
||||
# Protocol version
|
||||
ATTRS = ('host', 'features',
|
||||
# metadata
|
||||
'source', 'ip_addr',
|
||||
'last_good', 'last_try', 'try_count')
|
||||
FEATURES = ('pruning', 'server_version', 'protocol_min', 'protocol_max',
|
||||
'ssl_port', 'tcp_port')
|
||||
# This should be set by the application
|
||||
DEFAULT_PORTS: Dict[str, int] = {}
|
||||
|
||||
def __init__(self, host, features, source='unknown', ip_addr=None,
|
||||
last_good=0, last_try=0, try_count=0):
|
||||
"""Create a peer given a host name (or IP address as a string),
|
||||
a dictionary of features, and a record of the source."""
|
||||
assert isinstance(host, str)
|
||||
assert isinstance(features, dict)
|
||||
assert host in features.get('hosts', {})
|
||||
self.host = host
|
||||
self.features = features.copy()
|
||||
# Canonicalize / clean-up
|
||||
for feature in self.FEATURES:
|
||||
self.features[feature] = getattr(self, feature)
|
||||
# Metadata
|
||||
self.source = source
|
||||
self.ip_addr = ip_addr
|
||||
# last_good represents the last connection that was
|
||||
# successful *and* successfully verified, at which point
|
||||
# try_count is set to 0. Failure to connect or failure to
|
||||
# verify increment the try_count.
|
||||
self.last_good = last_good
|
||||
self.last_try = last_try
|
||||
self.try_count = try_count
|
||||
# Transient, non-persisted metadata
|
||||
self.bad = False
|
||||
self.other_port_pairs = set()
|
||||
self.status = 2
|
||||
|
||||
@classmethod
|
||||
def peers_from_features(cls, features, source):
|
||||
peers = []
|
||||
if isinstance(features, dict):
|
||||
hosts = features.get('hosts')
|
||||
if isinstance(hosts, dict):
|
||||
peers = [Peer(host, features, source=source)
|
||||
for host in hosts if isinstance(host, str)]
|
||||
return peers
|
||||
|
||||
@classmethod
|
||||
def deserialize(cls, item):
|
||||
"""Deserialize from a dictionary."""
|
||||
return cls(**item)
|
||||
|
||||
def matches(self, peers):
|
||||
"""Return peers whose host matches our hostname or IP address.
|
||||
Additionally include all peers whose IP address matches our
|
||||
hostname if that is an IP address.
|
||||
"""
|
||||
candidates = (self.host.lower(), self.ip_addr)
|
||||
return [peer for peer in peers
|
||||
if peer.host.lower() in candidates
|
||||
or peer.ip_addr == self.host]
|
||||
|
||||
def __str__(self):
|
||||
return self.host
|
||||
|
||||
def update_features(self, features):
|
||||
"""Update features in-place."""
|
||||
try:
|
||||
tmp = Peer(self.host, features)
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
self.update_features_from_peer(tmp)
|
||||
|
||||
def update_features_from_peer(self, peer):
|
||||
if peer != self:
|
||||
self.features = peer.features
|
||||
for feature in self.FEATURES:
|
||||
setattr(self, feature, getattr(peer, feature))
|
||||
|
||||
def connection_port_pairs(self):
|
||||
"""Return a list of (kind, port) pairs to try when making a
|
||||
connection."""
|
||||
# Use a list not a set - it's important to try the registered
|
||||
# ports first.
|
||||
pairs = [('SSL', self.ssl_port), ('TCP', self.tcp_port)]
|
||||
while self.other_port_pairs:
|
||||
pairs.append(self.other_port_pairs.pop())
|
||||
return [pair for pair in pairs if pair[1]]
|
||||
|
||||
def mark_bad(self):
|
||||
"""Mark as bad to avoid reconnects but also to remember for a
|
||||
while."""
|
||||
self.bad = True
|
||||
|
||||
def check_ports(self, other):
|
||||
"""Remember differing ports in case server operator changed them
|
||||
or removed one."""
|
||||
if other.ssl_port != self.ssl_port:
|
||||
self.other_port_pairs.add(('SSL', other.ssl_port))
|
||||
if other.tcp_port != self.tcp_port:
|
||||
self.other_port_pairs.add(('TCP', other.tcp_port))
|
||||
return bool(self.other_port_pairs)
|
||||
|
||||
@cachedproperty
|
||||
def is_tor(self):
|
||||
return self.host.endswith('.onion')
|
||||
|
||||
@cachedproperty
|
||||
def is_valid(self):
|
||||
ip = self.ip_address
|
||||
if ip:
|
||||
return ((ip.is_global or ip.is_private)
|
||||
and not (ip.is_multicast or ip.is_unspecified))
|
||||
return util.is_valid_hostname(self.host)
|
||||
|
||||
@cachedproperty
|
||||
def is_public(self):
|
||||
ip = self.ip_address
|
||||
if ip:
|
||||
return self.is_valid and not ip.is_private
|
||||
else:
|
||||
return self.is_valid and self.host != 'localhost'
|
||||
|
||||
@cachedproperty
|
||||
def ip_address(self):
|
||||
"""The host as a python ip_address object, or None."""
|
||||
try:
|
||||
return ip_address(self.host)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
def bucket(self):
|
||||
if self.is_tor:
|
||||
return 'onion'
|
||||
if not self.ip_addr:
|
||||
return ''
|
||||
return tuple(self.ip_addr.split('.')[:2])
|
||||
|
||||
def serialize(self):
|
||||
"""Serialize to a dictionary."""
|
||||
return {attr: getattr(self, attr) for attr in self.ATTRS}
|
||||
|
||||
def _port(self, key):
|
||||
hosts = self.features.get('hosts')
|
||||
if isinstance(hosts, dict):
|
||||
host = hosts.get(self.host)
|
||||
port = self._integer(key, host)
|
||||
if port and 0 < port < 65536:
|
||||
return port
|
||||
return None
|
||||
|
||||
def _integer(self, key, d=None):
|
||||
d = d or self.features
|
||||
result = d.get(key) if isinstance(d, dict) else None
|
||||
if isinstance(result, str):
|
||||
try:
|
||||
result = int(result)
|
||||
except ValueError:
|
||||
pass
|
||||
return result if isinstance(result, int) else None
|
||||
|
||||
def _string(self, key):
|
||||
result = self.features.get(key)
|
||||
return result if isinstance(result, str) else None
|
||||
|
||||
@cachedproperty
|
||||
def genesis_hash(self):
|
||||
"""Returns None if no SSL port, otherwise the port as an integer."""
|
||||
return self._string('genesis_hash')
|
||||
|
||||
@cachedproperty
|
||||
def ssl_port(self):
|
||||
"""Returns None if no SSL port, otherwise the port as an integer."""
|
||||
return self._port('ssl_port')
|
||||
|
||||
@cachedproperty
|
||||
def tcp_port(self):
|
||||
"""Returns None if no TCP port, otherwise the port as an integer."""
|
||||
return self._port('tcp_port')
|
||||
|
||||
@cachedproperty
|
||||
def server_version(self):
|
||||
"""Returns the server version as a string if known, otherwise None."""
|
||||
return self._string('server_version')
|
||||
|
||||
@cachedproperty
|
||||
def pruning(self):
|
||||
"""Returns the pruning level as an integer. None indicates no
|
||||
pruning."""
|
||||
pruning = self._integer('pruning')
|
||||
if pruning and pruning > 0:
|
||||
return pruning
|
||||
return None
|
||||
|
||||
def _protocol_version_string(self, key):
|
||||
version_str = self.features.get(key)
|
||||
ptuple = util.protocol_tuple(version_str)
|
||||
return util.version_string(ptuple)
|
||||
|
||||
@cachedproperty
|
||||
def protocol_min(self):
|
||||
"""Minimum protocol version as a string, e.g., 1.0"""
|
||||
return self._protocol_version_string('protocol_min')
|
||||
|
||||
@cachedproperty
|
||||
def protocol_max(self):
|
||||
"""Maximum protocol version as a string, e.g., 1.1"""
|
||||
return self._protocol_version_string('protocol_max')
|
||||
|
||||
def to_tuple(self):
|
||||
"""The tuple ((ip, host, details) expected in response
|
||||
to a peers subscription."""
|
||||
details = self.real_name().split()[1:]
|
||||
return (self.ip_addr or self.host, self.host, details)
|
||||
|
||||
def real_name(self):
|
||||
"""Real name of this peer as used on IRC."""
|
||||
def port_text(letter, port):
|
||||
if port == self.DEFAULT_PORTS.get(letter):
|
||||
return letter
|
||||
else:
|
||||
return letter + str(port)
|
||||
|
||||
parts = [self.host, 'v' + self.protocol_max]
|
||||
if self.pruning:
|
||||
parts.append(f'p{self.pruning:d}')
|
||||
for letter, port in (('s', self.ssl_port), ('t', self.tcp_port)):
|
||||
if port:
|
||||
parts.append(port_text(letter, port))
|
||||
return ' '.join(parts)
|
||||
|
||||
@classmethod
|
||||
def from_real_name(cls, real_name, source):
|
||||
"""Real name is a real name as on IRC, such as
|
||||
|
||||
"erbium1.sytes.net v1.0 s t"
|
||||
|
||||
Returns an instance of this Peer class.
|
||||
"""
|
||||
host = 'nohost'
|
||||
features = {}
|
||||
ports = {}
|
||||
for n, part in enumerate(real_name.split()):
|
||||
if n == 0:
|
||||
host = part
|
||||
continue
|
||||
if part[0] in ('s', 't'):
|
||||
if len(part) == 1:
|
||||
port = cls.DEFAULT_PORTS[part[0]]
|
||||
else:
|
||||
port = part[1:]
|
||||
if part[0] == 's':
|
||||
ports['ssl_port'] = port
|
||||
else:
|
||||
ports['tcp_port'] = port
|
||||
elif part[0] == 'v':
|
||||
features['protocol_max'] = features['protocol_min'] = part[1:]
|
||||
elif part[0] == 'p':
|
||||
features['pruning'] = part[1:]
|
||||
|
||||
features.update(ports)
|
||||
features['hosts'] = {host: ports}
|
||||
|
||||
return cls(host, features, source)
|
||||
@@ -1,506 +0,0 @@
|
||||
# Copyright (c) 2017-2018, Neil Booth
|
||||
#
|
||||
# All rights reserved.
|
||||
#
|
||||
# See the file "LICENCE" for information about the copyright
|
||||
# and warranty status of this software.
|
||||
|
||||
"""Peer management."""
|
||||
|
||||
import asyncio
|
||||
import random
|
||||
import socket
|
||||
import ssl
|
||||
import time
|
||||
import typing
|
||||
from asyncio import Event, sleep
|
||||
from collections import defaultdict, Counter
|
||||
|
||||
from lbry.wallet.tasks import TaskGroup
|
||||
from lbry.wallet.rpc import (
|
||||
Connector, RPCSession, SOCKSProxy, Notification, handler_invocation,
|
||||
SOCKSError, RPCError
|
||||
)
|
||||
from lbry.wallet.server.peer import Peer
|
||||
from lbry.wallet.server.util import class_logger, protocol_tuple
|
||||
|
||||
PEER_GOOD, PEER_STALE, PEER_NEVER, PEER_BAD = range(4)
|
||||
STALE_SECS = 24 * 3600
|
||||
WAKEUP_SECS = 300
|
||||
|
||||
|
||||
class BadPeerError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def assert_good(message, result, instance):
|
||||
if not isinstance(result, instance):
|
||||
raise BadPeerError(f'{message} returned bad result type '
|
||||
f'{type(result).__name__}')
|
||||
|
||||
|
||||
class PeerSession(RPCSession):
|
||||
"""An outgoing session to a peer."""
|
||||
|
||||
async def handle_request(self, request):
|
||||
# We subscribe so might be unlucky enough to get a notification...
|
||||
if (isinstance(request, Notification) and
|
||||
request.method == 'blockchain.headers.subscribe'):
|
||||
pass
|
||||
else:
|
||||
await handler_invocation(None, request) # Raises
|
||||
|
||||
|
||||
class PeerManager:
|
||||
"""Looks after the DB of peer network servers.
|
||||
|
||||
Attempts to maintain a connection with up to 8 peers.
|
||||
Issues a 'peers.subscribe' RPC to them and tells them our data.
|
||||
"""
|
||||
def __init__(self, env, db):
|
||||
self.logger = class_logger(__name__, self.__class__.__name__)
|
||||
# Initialise the Peer class
|
||||
Peer.DEFAULT_PORTS = env.coin.PEER_DEFAULT_PORTS
|
||||
self.env = env
|
||||
self.db = db
|
||||
|
||||
# Our clearnet and Tor Peers, if any
|
||||
sclass = env.coin.SESSIONCLS
|
||||
self.myselves = [Peer(ident.host, sclass.server_features(env), 'env')
|
||||
for ident in env.identities]
|
||||
self.server_version_args = sclass.server_version_args()
|
||||
# Peers have one entry per hostname. Once connected, the
|
||||
# ip_addr property is either None, an onion peer, or the
|
||||
# IP address that was connected to. Adding a peer will evict
|
||||
# any other peers with the same host name or IP address.
|
||||
self.peers: typing.Set[Peer] = set()
|
||||
self.permit_onion_peer_time = time.time()
|
||||
self.proxy = None
|
||||
self.group = TaskGroup()
|
||||
|
||||
def _my_clearnet_peer(self):
|
||||
"""Returns the clearnet peer representing this server, if any."""
|
||||
clearnet = [peer for peer in self.myselves if not peer.is_tor]
|
||||
return clearnet[0] if clearnet else None
|
||||
|
||||
def _set_peer_statuses(self):
|
||||
"""Set peer statuses."""
|
||||
cutoff = time.time() - STALE_SECS
|
||||
for peer in self.peers:
|
||||
if peer.bad:
|
||||
peer.status = PEER_BAD
|
||||
elif peer.last_good > cutoff:
|
||||
peer.status = PEER_GOOD
|
||||
elif peer.last_good:
|
||||
peer.status = PEER_STALE
|
||||
else:
|
||||
peer.status = PEER_NEVER
|
||||
|
||||
def _features_to_register(self, peer, remote_peers):
|
||||
"""If we should register ourselves to the remote peer, which has
|
||||
reported the given list of known peers, return the clearnet
|
||||
identity features to register, otherwise None.
|
||||
"""
|
||||
# Announce ourself if not present. Don't if disabled, we
|
||||
# are a non-public IP address, or to ourselves.
|
||||
if not self.env.peer_announce or peer in self.myselves:
|
||||
return None
|
||||
my = self._my_clearnet_peer()
|
||||
if not my or not my.is_public:
|
||||
return None
|
||||
# Register if no matches, or ports have changed
|
||||
for peer in my.matches(remote_peers):
|
||||
if peer.tcp_port == my.tcp_port and peer.ssl_port == my.ssl_port:
|
||||
return None
|
||||
return my.features
|
||||
|
||||
def _permit_new_onion_peer(self):
|
||||
"""Accept a new onion peer only once per random time interval."""
|
||||
now = time.time()
|
||||
if now < self.permit_onion_peer_time:
|
||||
return False
|
||||
self.permit_onion_peer_time = now + random.randrange(0, 1200)
|
||||
return True
|
||||
|
||||
async def _import_peers(self):
|
||||
"""Import hard-coded peers from a file or the coin defaults."""
|
||||
imported_peers = self.myselves.copy()
|
||||
# Add the hard-coded ones unless only reporting ourself
|
||||
if self.env.peer_discovery != self.env.PD_SELF:
|
||||
imported_peers.extend(Peer.from_real_name(real_name, 'coins.py')
|
||||
for real_name in self.env.coin.PEERS)
|
||||
await self._note_peers(imported_peers, limit=None)
|
||||
|
||||
async def _detect_proxy(self):
|
||||
"""Detect a proxy if we don't have one and some time has passed since
|
||||
the last attempt.
|
||||
|
||||
If found self.proxy is set to a SOCKSProxy instance, otherwise
|
||||
None.
|
||||
"""
|
||||
host = self.env.tor_proxy_host
|
||||
if self.env.tor_proxy_port is None:
|
||||
ports = [9050, 9150, 1080]
|
||||
else:
|
||||
ports = [self.env.tor_proxy_port]
|
||||
while True:
|
||||
self.logger.info(f'trying to detect proxy on "{host}" '
|
||||
f'ports {ports}')
|
||||
proxy = await SOCKSProxy.auto_detect_host(host, ports, None)
|
||||
if proxy:
|
||||
self.proxy = proxy
|
||||
self.logger.info(f'detected {proxy}')
|
||||
return
|
||||
self.logger.info('no proxy detected, will try later')
|
||||
await sleep(900)
|
||||
|
||||
async def _note_peers(self, peers, limit=2, check_ports=False,
|
||||
source=None):
|
||||
"""Add a limited number of peers that are not already present."""
|
||||
new_peers = []
|
||||
for peer in peers:
|
||||
if not peer.is_public or (peer.is_tor and not self.proxy):
|
||||
continue
|
||||
|
||||
matches = peer.matches(self.peers)
|
||||
if not matches:
|
||||
new_peers.append(peer)
|
||||
elif check_ports:
|
||||
for match in matches:
|
||||
if match.check_ports(peer):
|
||||
self.logger.info(f'ports changed for {peer}')
|
||||
match.retry_event.set()
|
||||
|
||||
if new_peers:
|
||||
source = source or new_peers[0].source
|
||||
if limit:
|
||||
random.shuffle(new_peers)
|
||||
use_peers = new_peers[:limit]
|
||||
else:
|
||||
use_peers = new_peers
|
||||
for peer in use_peers:
|
||||
self.logger.info(f'accepted new peer {peer} from {source}')
|
||||
peer.retry_event = Event()
|
||||
self.peers.add(peer)
|
||||
await self.group.add(self._monitor_peer(peer))
|
||||
|
||||
async def _monitor_peer(self, peer):
|
||||
# Stop monitoring if we were dropped (a duplicate peer)
|
||||
while peer in self.peers:
|
||||
if await self._should_drop_peer(peer):
|
||||
self.peers.discard(peer)
|
||||
break
|
||||
# Figure out how long to sleep before retrying. Retry a
|
||||
# good connection when it is about to turn stale, otherwise
|
||||
# exponentially back off retries.
|
||||
if peer.try_count == 0:
|
||||
pause = STALE_SECS - WAKEUP_SECS * 2
|
||||
else:
|
||||
pause = WAKEUP_SECS * 2 ** peer.try_count
|
||||
pending, done = await asyncio.wait([peer.retry_event.wait()], timeout=pause)
|
||||
if done:
|
||||
peer.retry_event.clear()
|
||||
|
||||
async def _should_drop_peer(self, peer):
|
||||
peer.try_count += 1
|
||||
is_good = False
|
||||
for kind, port in peer.connection_port_pairs():
|
||||
peer.last_try = time.time()
|
||||
|
||||
kwargs = {}
|
||||
if kind == 'SSL':
|
||||
kwargs['ssl'] = ssl.SSLContext(ssl.PROTOCOL_TLS)
|
||||
|
||||
host = self.env.cs_host(for_rpc=False)
|
||||
if isinstance(host, list):
|
||||
host = host[0]
|
||||
|
||||
if self.env.force_proxy or peer.is_tor:
|
||||
if not self.proxy:
|
||||
return
|
||||
kwargs['proxy'] = self.proxy
|
||||
kwargs['resolve'] = not peer.is_tor
|
||||
elif host:
|
||||
# Use our listening Host/IP for outgoing non-proxy
|
||||
# connections so our peers see the correct source.
|
||||
kwargs['local_addr'] = (host, None)
|
||||
|
||||
peer_text = f'[{peer}:{port} {kind}]'
|
||||
try:
|
||||
async with Connector(PeerSession, peer.host, port,
|
||||
**kwargs) as session:
|
||||
await asyncio.wait_for(
|
||||
self._verify_peer(session, peer),
|
||||
120 if peer.is_tor else 30
|
||||
)
|
||||
is_good = True
|
||||
break
|
||||
except BadPeerError as e:
|
||||
self.logger.error(f'{peer_text} marking bad: ({e})')
|
||||
peer.mark_bad()
|
||||
break
|
||||
except RPCError as e:
|
||||
self.logger.error(f'{peer_text} RPC error: {e.message} '
|
||||
f'({e.code})')
|
||||
except (OSError, SOCKSError, ConnectionError, asyncio.TimeoutError) as e:
|
||||
self.logger.info(f'{peer_text} {e}')
|
||||
|
||||
if is_good:
|
||||
now = time.time()
|
||||
elapsed = now - peer.last_try
|
||||
self.logger.info(f'{peer_text} verified in {elapsed:.1f}s')
|
||||
peer.try_count = 0
|
||||
peer.last_good = now
|
||||
peer.source = 'peer'
|
||||
# At most 2 matches if we're a host name, potentially
|
||||
# several if we're an IP address (several instances
|
||||
# can share a NAT).
|
||||
matches = peer.matches(self.peers)
|
||||
for match in matches:
|
||||
if match.ip_address:
|
||||
if len(matches) > 1:
|
||||
self.peers.remove(match)
|
||||
# Force the peer's monitoring task to exit
|
||||
match.retry_event.set()
|
||||
elif peer.host in match.features['hosts']:
|
||||
match.update_features_from_peer(peer)
|
||||
else:
|
||||
# Forget the peer if long-term unreachable
|
||||
if peer.last_good and not peer.bad:
|
||||
try_limit = 10
|
||||
else:
|
||||
try_limit = 3
|
||||
if peer.try_count >= try_limit:
|
||||
desc = 'bad' if peer.bad else 'unreachable'
|
||||
self.logger.info(f'forgetting {desc} peer: {peer}')
|
||||
return True
|
||||
return False
|
||||
|
||||
async def _verify_peer(self, session, peer):
|
||||
if not peer.is_tor:
|
||||
address = session.peer_address()
|
||||
if address:
|
||||
peer.ip_addr = address[0]
|
||||
|
||||
# server.version goes first
|
||||
message = 'server.version'
|
||||
result = await session.send_request(message, self.server_version_args)
|
||||
assert_good(message, result, list)
|
||||
|
||||
# Protocol version 1.1 returns a pair with the version first
|
||||
if len(result) != 2 or not all(isinstance(x, str) for x in result):
|
||||
raise BadPeerError(f'bad server.version result: {result}')
|
||||
server_version, protocol_version = result
|
||||
peer.server_version = server_version
|
||||
peer.features['server_version'] = server_version
|
||||
ptuple = protocol_tuple(protocol_version)
|
||||
|
||||
await asyncio.wait([
|
||||
self._send_headers_subscribe(session, peer, ptuple),
|
||||
self._send_server_features(session, peer),
|
||||
self._send_peers_subscribe(session, peer)
|
||||
])
|
||||
|
||||
async def _send_headers_subscribe(self, session, peer, ptuple):
|
||||
message = 'blockchain.headers.subscribe'
|
||||
result = await session.send_request(message)
|
||||
assert_good(message, result, dict)
|
||||
|
||||
our_height = self.db.db_height
|
||||
if ptuple < (1, 3):
|
||||
their_height = result.get('block_height')
|
||||
else:
|
||||
their_height = result.get('height')
|
||||
if not isinstance(their_height, int):
|
||||
raise BadPeerError(f'invalid height {their_height}')
|
||||
if abs(our_height - their_height) > 5:
|
||||
raise BadPeerError(f'bad height {their_height:,d} '
|
||||
f'(ours: {our_height:,d})')
|
||||
|
||||
# Check prior header too in case of hard fork.
|
||||
check_height = min(our_height, their_height)
|
||||
raw_header = self.db.raw_header(check_height)
|
||||
if ptuple >= (1, 4):
|
||||
ours = raw_header.hex()
|
||||
message = 'blockchain.block.header'
|
||||
theirs = await session.send_request(message, [check_height])
|
||||
assert_good(message, theirs, str)
|
||||
if ours != theirs:
|
||||
raise BadPeerError(f'our header {ours} and '
|
||||
f'theirs {theirs} differ')
|
||||
else:
|
||||
ours = self.env.coin.electrum_header(raw_header, check_height)
|
||||
ours = ours.get('prev_block_hash')
|
||||
message = 'blockchain.block.get_header'
|
||||
theirs = await session.send_request(message, [check_height])
|
||||
assert_good(message, theirs, dict)
|
||||
theirs = theirs.get('prev_block_hash')
|
||||
if ours != theirs:
|
||||
raise BadPeerError(f'our header hash {ours} and '
|
||||
f'theirs {theirs} differ')
|
||||
|
||||
async def _send_server_features(self, session, peer):
|
||||
message = 'server.features'
|
||||
features = await session.send_request(message)
|
||||
assert_good(message, features, dict)
|
||||
hosts = [host.lower() for host in features.get('hosts', {})]
|
||||
if self.env.coin.GENESIS_HASH != features.get('genesis_hash'):
|
||||
raise BadPeerError('incorrect genesis hash')
|
||||
elif peer.host.lower() in hosts:
|
||||
peer.update_features(features)
|
||||
else:
|
||||
raise BadPeerError(f'not listed in own hosts list {hosts}')
|
||||
|
||||
async def _send_peers_subscribe(self, session, peer):
|
||||
message = 'server.peers.subscribe'
|
||||
raw_peers = await session.send_request(message)
|
||||
assert_good(message, raw_peers, list)
|
||||
|
||||
# Check the peers list we got from a remote peer.
|
||||
# Each is expected to be of the form:
|
||||
# [ip_addr, hostname, ['v1.0', 't51001', 's51002']]
|
||||
# Call add_peer if the remote doesn't appear to know about us.
|
||||
try:
|
||||
real_names = [' '.join([u[1]] + u[2]) for u in raw_peers]
|
||||
peers = [Peer.from_real_name(real_name, str(peer))
|
||||
for real_name in real_names]
|
||||
except Exception:
|
||||
raise BadPeerError('bad server.peers.subscribe response')
|
||||
|
||||
await self._note_peers(peers)
|
||||
features = self._features_to_register(peer, peers)
|
||||
if not features:
|
||||
return
|
||||
self.logger.info(f'registering ourself with {peer}')
|
||||
# We only care to wait for the response
|
||||
await session.send_request('server.add_peer', [features])
|
||||
|
||||
#
|
||||
# External interface
|
||||
#
|
||||
async def discover_peers(self):
|
||||
"""Perform peer maintenance. This includes
|
||||
|
||||
1) Forgetting unreachable peers.
|
||||
2) Verifying connectivity of new peers.
|
||||
3) Retrying old peers at regular intervals.
|
||||
"""
|
||||
if self.env.peer_discovery != self.env.PD_ON:
|
||||
self.logger.info('peer discovery is disabled')
|
||||
return
|
||||
|
||||
self.logger.info(f'beginning peer discovery. Force use of '
|
||||
f'proxy: {self.env.force_proxy}')
|
||||
|
||||
self.group.add(self._detect_proxy())
|
||||
self.group.add(self._import_peers())
|
||||
|
||||
def info(self) -> typing.Dict[str, int]:
|
||||
"""The number of peers."""
|
||||
self._set_peer_statuses()
|
||||
counter = Counter(peer.status for peer in self.peers)
|
||||
return {
|
||||
'bad': counter[PEER_BAD],
|
||||
'good': counter[PEER_GOOD],
|
||||
'never': counter[PEER_NEVER],
|
||||
'stale': counter[PEER_STALE],
|
||||
'total': len(self.peers),
|
||||
}
|
||||
|
||||
async def add_localRPC_peer(self, real_name):
|
||||
"""Add a peer passed by the admin over LocalRPC."""
|
||||
await self._note_peers([Peer.from_real_name(real_name, 'RPC')])
|
||||
|
||||
async def on_add_peer(self, features, source_info):
|
||||
"""Add a peer (but only if the peer resolves to the source)."""
|
||||
if not source_info:
|
||||
self.logger.info('ignored add_peer request: no source info')
|
||||
return False
|
||||
source = source_info[0]
|
||||
peers = Peer.peers_from_features(features, source)
|
||||
if not peers:
|
||||
self.logger.info('ignored add_peer request: no peers given')
|
||||
return False
|
||||
|
||||
# Just look at the first peer, require it
|
||||
peer = peers[0]
|
||||
host = peer.host
|
||||
if peer.is_tor:
|
||||
permit = self._permit_new_onion_peer()
|
||||
reason = 'rate limiting'
|
||||
else:
|
||||
getaddrinfo = asyncio.get_event_loop().getaddrinfo
|
||||
try:
|
||||
infos = await getaddrinfo(host, 80, type=socket.SOCK_STREAM)
|
||||
except socket.gaierror:
|
||||
permit = False
|
||||
reason = 'address resolution failure'
|
||||
else:
|
||||
permit = any(source == info[-1][0] for info in infos)
|
||||
reason = 'source-destination mismatch'
|
||||
|
||||
if permit:
|
||||
self.logger.info(f'accepted add_peer request from {source} '
|
||||
f'for {host}')
|
||||
await self._note_peers([peer], check_ports=True)
|
||||
else:
|
||||
self.logger.warning(f'rejected add_peer request from {source} '
|
||||
f'for {host} ({reason})')
|
||||
|
||||
return permit
|
||||
|
||||
def on_peers_subscribe(self, is_tor):
|
||||
"""Returns the server peers as a list of (ip, host, details) tuples.
|
||||
|
||||
We return all peers we've connected to in the last day.
|
||||
Additionally, if we don't have onion routing, we return a few
|
||||
hard-coded onion servers.
|
||||
"""
|
||||
cutoff = time.time() - STALE_SECS
|
||||
recent = [peer for peer in self.peers
|
||||
if peer.last_good > cutoff and
|
||||
not peer.bad and peer.is_public]
|
||||
onion_peers = []
|
||||
|
||||
# Always report ourselves if valid (even if not public)
|
||||
peers = {myself for myself in self.myselves
|
||||
if myself.last_good > cutoff}
|
||||
|
||||
# Bucket the clearnet peers and select up to two from each
|
||||
buckets = defaultdict(list)
|
||||
for peer in recent:
|
||||
if peer.is_tor:
|
||||
onion_peers.append(peer)
|
||||
else:
|
||||
buckets[peer.bucket()].append(peer)
|
||||
for bucket_peers in buckets.values():
|
||||
random.shuffle(bucket_peers)
|
||||
peers.update(bucket_peers[:2])
|
||||
|
||||
# Add up to 20% onion peers (but up to 10 is OK anyway)
|
||||
random.shuffle(onion_peers)
|
||||
max_onion = 50 if is_tor else max(10, len(peers) // 4)
|
||||
|
||||
peers.update(onion_peers[:max_onion])
|
||||
|
||||
return [peer.to_tuple() for peer in peers]
|
||||
|
||||
def proxy_peername(self):
|
||||
"""Return the peername of the proxy, if there is a proxy, otherwise
|
||||
None."""
|
||||
return self.proxy.peername if self.proxy else None
|
||||
|
||||
def rpc_data(self):
|
||||
"""Peer data for the peers RPC method."""
|
||||
self._set_peer_statuses()
|
||||
descs = ['good', 'stale', 'never', 'bad']
|
||||
|
||||
def peer_data(peer):
|
||||
data = peer.serialize()
|
||||
data['status'] = descs[peer.status]
|
||||
return data
|
||||
|
||||
def peer_key(peer):
|
||||
return (peer.bad, -peer.last_good)
|
||||
|
||||
return [peer_data(peer) for peer in sorted(self.peers, key=peer_key)]
|
||||
@@ -114,8 +114,8 @@ class Server:
|
||||
await self.start_prometheus()
|
||||
if self.env.udp_port:
|
||||
await self.bp.status_server.start(
|
||||
0, bytes.fromhex(self.bp.coin.GENESIS_HASH)[::-1],
|
||||
self.env.host, self.env.udp_port
|
||||
0, bytes.fromhex(self.bp.coin.GENESIS_HASH)[::-1], self.env.country,
|
||||
self.env.host, self.env.udp_port, self.env.allow_lan_udp
|
||||
)
|
||||
await _start_cancellable(self.bp.fetch_and_process_blocks)
|
||||
|
||||
|
||||
@@ -15,14 +15,13 @@ from asyncio import Event, sleep
|
||||
from collections import defaultdict
|
||||
from functools import partial
|
||||
|
||||
from binascii import hexlify
|
||||
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
|
||||
|
||||
from elasticsearch import ConnectionTimeout
|
||||
from prometheus_client import Counter, Info, Histogram, Gauge
|
||||
|
||||
import lbry
|
||||
from lbry.utils import LRUCacheWithMetrics
|
||||
from lbry.error import TooManyClaimSearchParametersError
|
||||
from lbry.build_info import BUILD, COMMIT_HASH, DOCKER_TAG
|
||||
from lbry.wallet.server.block_processor import LBRYBlockProcessor
|
||||
from lbry.wallet.server.db.writer import LBRYLevelDB
|
||||
@@ -39,7 +38,6 @@ from lbry.wallet.server import text
|
||||
from lbry.wallet.server import util
|
||||
from lbry.wallet.server.hash import sha256, hash_to_hex_str, hex_str_to_hash, HASHX_LEN, Base58Error
|
||||
from lbry.wallet.server.daemon import DaemonError
|
||||
from lbry.wallet.server.peers import PeerManager
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.wallet.server.env import Env
|
||||
from lbry.wallet.server.mempool import MemPool
|
||||
@@ -185,7 +183,6 @@ class SessionManager:
|
||||
self.bp = bp
|
||||
self.daemon = daemon
|
||||
self.mempool = mempool
|
||||
self.peer_mgr = PeerManager(env, db)
|
||||
self.shutdown_event = shutdown_event
|
||||
self.logger = util.class_logger(__name__, self.__class__.__name__)
|
||||
self.servers: typing.Dict[str, asyncio.AbstractServer] = {}
|
||||
@@ -209,7 +206,7 @@ class SessionManager:
|
||||
else:
|
||||
protocol_class = self.env.coin.SESSIONCLS
|
||||
protocol_factory = partial(protocol_class, self, self.db,
|
||||
self.mempool, self.peer_mgr, kind)
|
||||
self.mempool, kind)
|
||||
|
||||
host, port = args[:2]
|
||||
try:
|
||||
@@ -369,7 +366,7 @@ class SessionManager:
|
||||
'logged': logged,
|
||||
'paused': paused,
|
||||
'pid': os.getpid(),
|
||||
'peers': self.peer_mgr.info(),
|
||||
'peers': [],
|
||||
'requests': pending_requests,
|
||||
'method_counts': method_counts,
|
||||
'sessions': self.session_count(),
|
||||
@@ -436,7 +433,7 @@ class SessionManager:
|
||||
|
||||
real_name: "bch.electrumx.cash t50001 s50002" for example
|
||||
"""
|
||||
await self.peer_mgr.add_localRPC_peer(real_name)
|
||||
await self._notify_peer(real_name)
|
||||
return f"peer '{real_name}' added"
|
||||
|
||||
async def rpc_disconnect(self, session_ids):
|
||||
@@ -487,7 +484,7 @@ class SessionManager:
|
||||
|
||||
async def rpc_peers(self):
|
||||
"""Return a list of data about server peers."""
|
||||
return self.peer_mgr.rpc_data()
|
||||
return self.env.peer_hubs
|
||||
|
||||
async def rpc_query(self, items, limit):
|
||||
"""Return a list of data about server peers."""
|
||||
@@ -578,7 +575,6 @@ class SessionManager:
|
||||
# Peer discovery should start after the external servers
|
||||
# because we connect to ourself
|
||||
await asyncio.wait([
|
||||
self.peer_mgr.discover_peers(),
|
||||
self._clear_stale_sessions(),
|
||||
self._log_sessions(),
|
||||
self._manage_servers()
|
||||
@@ -638,6 +634,15 @@ class SessionManager:
|
||||
self.history_cache[hashX] = await self.db.limited_history(hashX, limit=limit)
|
||||
return self.history_cache[hashX]
|
||||
|
||||
def _notify_peer(self, peer):
|
||||
notify_tasks = [
|
||||
session.send_notification('blockchain.peers.subscribe', [peer])
|
||||
for session in self.sessions.values() if session.subscribe_peers
|
||||
]
|
||||
if notify_tasks:
|
||||
self.logger.info(f'notify {len(notify_tasks)} sessions of new peers')
|
||||
asyncio.create_task(asyncio.wait(notify_tasks))
|
||||
|
||||
async def _notify_sessions(self, height, touched, new_touched):
|
||||
"""Notify sessions about height changes and touched addresses."""
|
||||
height_changed = height != self.notified_height
|
||||
@@ -704,7 +709,7 @@ class SessionBase(RPCSession):
|
||||
request_handlers: typing.Dict[str, typing.Callable] = {}
|
||||
version = '0.5.7'
|
||||
|
||||
def __init__(self, session_mgr, db, mempool, peer_mgr, kind):
|
||||
def __init__(self, session_mgr, db, mempool, kind):
|
||||
connection = JSONRPCConnection(JSONRPCAutoDetect)
|
||||
self.env = session_mgr.env
|
||||
super().__init__(connection=connection)
|
||||
@@ -712,7 +717,6 @@ class SessionBase(RPCSession):
|
||||
self.session_mgr = session_mgr
|
||||
self.db = db
|
||||
self.mempool = mempool
|
||||
self.peer_mgr = peer_mgr
|
||||
self.kind = kind # 'RPC', 'TCP' etc.
|
||||
self.coin = self.env.coin
|
||||
self.anon_logs = self.env.anon_logs
|
||||
@@ -904,6 +908,7 @@ class LBRYElectrumX(SessionBase):
|
||||
LBRYElectrumX.set_server_features(self.env)
|
||||
self.subscribe_headers = False
|
||||
self.subscribe_headers_raw = False
|
||||
self.subscribe_peers = False
|
||||
self.connection.max_response_size = self.env.max_send
|
||||
self.hashX_subs = {}
|
||||
self.sv_seen = False
|
||||
@@ -1022,7 +1027,13 @@ class LBRYElectrumX(SessionBase):
|
||||
|
||||
async def claimtrie_search(self, **kwargs):
|
||||
if kwargs:
|
||||
return await self.run_and_cache_query('search', kwargs)
|
||||
try:
|
||||
return await self.run_and_cache_query('search', kwargs)
|
||||
except TooManyClaimSearchParametersError as err:
|
||||
await asyncio.sleep(2)
|
||||
self.logger.warning("Got an invalid query from %s, for %s with more than %d elements.",
|
||||
self.peer_address()[0], err.key, err.limit)
|
||||
return RPCError(1, str(err))
|
||||
|
||||
async def claimtrie_resolve(self, *urls):
|
||||
if urls:
|
||||
@@ -1084,7 +1095,8 @@ class LBRYElectrumX(SessionBase):
|
||||
|
||||
async def peers_subscribe(self):
|
||||
"""Return the server peers as a list of (ip, host, details) tuples."""
|
||||
return self.peer_mgr.on_peers_subscribe(self.is_tor())
|
||||
self.subscribe_peers = True
|
||||
return self.env.peer_hubs
|
||||
|
||||
async def address_status(self, hashX):
|
||||
"""Returns an address status.
|
||||
@@ -1327,11 +1339,7 @@ class LBRYElectrumX(SessionBase):
|
||||
async def banner(self):
|
||||
"""Return the server banner text."""
|
||||
banner = f'You are connected to an {self.version} server.'
|
||||
|
||||
if self.is_tor():
|
||||
banner_file = self.env.tor_banner_file
|
||||
else:
|
||||
banner_file = self.env.banner_file
|
||||
banner_file = self.env.banner_file
|
||||
if banner_file:
|
||||
try:
|
||||
with codecs.open(banner_file, 'r', 'utf-8') as f:
|
||||
|
||||
+56
-19
@@ -3,7 +3,8 @@ import struct
|
||||
from time import perf_counter
|
||||
import logging
|
||||
from typing import Optional, Tuple, NamedTuple
|
||||
from lbry.utils import LRUCache
|
||||
from lbry.utils import LRUCache, is_valid_public_ipv4
|
||||
from lbry.schema.attrs import country_str_to_int, country_int_to_str
|
||||
# from prometheus_client import Counter
|
||||
|
||||
|
||||
@@ -13,6 +14,9 @@ _MAGIC = 1446058291 # genesis blocktime (which is actually wrong)
|
||||
_PAD_BYTES = b'\x00' * 64
|
||||
|
||||
|
||||
PROTOCOL_VERSION = 1
|
||||
|
||||
|
||||
class SPVPing(NamedTuple):
|
||||
magic: int
|
||||
protocol_version: int
|
||||
@@ -22,8 +26,8 @@ class SPVPing(NamedTuple):
|
||||
return struct.pack(b'!lB64s', *self)
|
||||
|
||||
@staticmethod
|
||||
def make(protocol_version=1) -> bytes:
|
||||
return SPVPing(_MAGIC, protocol_version, _PAD_BYTES).encode()
|
||||
def make() -> bytes:
|
||||
return SPVPing(_MAGIC, PROTOCOL_VERSION, _PAD_BYTES).encode()
|
||||
|
||||
@classmethod
|
||||
def decode(cls, packet: bytes):
|
||||
@@ -33,24 +37,40 @@ class SPVPing(NamedTuple):
|
||||
return decoded
|
||||
|
||||
|
||||
PONG_ENCODING = b'!BBL32s4sH'
|
||||
|
||||
|
||||
class SPVPong(NamedTuple):
|
||||
protocol_version: int
|
||||
flags: int
|
||||
height: int
|
||||
tip: bytes
|
||||
source_address_raw: bytes
|
||||
country: int
|
||||
|
||||
def encode(self):
|
||||
return struct.pack(b'!BBl32s4s', *self)
|
||||
return struct.pack(PONG_ENCODING, *self)
|
||||
|
||||
@staticmethod
|
||||
def make(height: int, tip: bytes, flags: int, protocol_version: int = 1) -> bytes:
|
||||
# note: drops the last 4 bytes so the result can be cached and have addresses added to it as needed
|
||||
return SPVPong(protocol_version, flags, height, tip, b'\x00\x00\x00\x00').encode()[:38]
|
||||
def encode_address(address: str):
|
||||
return bytes(int(b) for b in address.split("."))
|
||||
|
||||
@classmethod
|
||||
def make(cls, flags: int, height: int, tip: bytes, source_address: str, country: str) -> bytes:
|
||||
return SPVPong(
|
||||
PROTOCOL_VERSION, flags, height, tip,
|
||||
cls.encode_address(source_address),
|
||||
country_str_to_int(country)
|
||||
).encode()
|
||||
|
||||
@classmethod
|
||||
def make_sans_source_address(cls, flags: int, height: int, tip: bytes, country: str) -> Tuple[bytes, bytes]:
|
||||
pong = cls.make(flags, height, tip, '0.0.0.0', country)
|
||||
return pong[:38], pong[42:]
|
||||
|
||||
@classmethod
|
||||
def decode(cls, packet: bytes):
|
||||
return cls(*struct.unpack(b'!BBl32s4s', packet[:42]))
|
||||
return cls(*struct.unpack(PONG_ENCODING, packet[:44]))
|
||||
|
||||
@property
|
||||
def available(self) -> bool:
|
||||
@@ -60,29 +80,41 @@ class SPVPong(NamedTuple):
|
||||
def ip_address(self) -> str:
|
||||
return ".".join(map(str, self.source_address_raw))
|
||||
|
||||
@property
|
||||
def country_name(self):
|
||||
return country_int_to_str(self.country)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"SPVPong(external_ip={self.ip_address}, version={self.protocol_version}, " \
|
||||
f"available={'True' if self.flags & 1 > 0 else 'False'}," \
|
||||
f" height={self.height}, tip={self.tip[::-1].hex()})"
|
||||
f" height={self.height}, tip={self.tip[::-1].hex()}, country={self.country_name})"
|
||||
|
||||
|
||||
class SPVServerStatusProtocol(asyncio.DatagramProtocol):
|
||||
PROTOCOL_VERSION = 1
|
||||
|
||||
def __init__(self, height: int, tip: bytes, throttle_cache_size: int = 1024, throttle_reqs_per_sec: int = 10):
|
||||
def __init__(
|
||||
self, height: int, tip: bytes, country: str,
|
||||
throttle_cache_size: int = 1024, throttle_reqs_per_sec: int = 10,
|
||||
allow_localhost: bool = False, allow_lan: bool = False
|
||||
):
|
||||
super().__init__()
|
||||
self.transport: Optional[asyncio.transports.DatagramTransport] = None
|
||||
self._height = height
|
||||
self._tip = tip
|
||||
self._flags = 0
|
||||
self._cached_response = None
|
||||
self._country = country
|
||||
self._left_cache = self._right_cache = None
|
||||
self.update_cached_response()
|
||||
self._throttle = LRUCache(throttle_cache_size)
|
||||
self._should_log = LRUCache(throttle_cache_size)
|
||||
self._min_delay = 1 / throttle_reqs_per_sec
|
||||
self._allow_localhost = allow_localhost
|
||||
self._allow_lan = allow_lan
|
||||
|
||||
def update_cached_response(self):
|
||||
self._cached_response = SPVPong.make(self._height, self._tip, self._flags, self.PROTOCOL_VERSION)
|
||||
self._left_cache, self._right_cache = SPVPong.make_sans_source_address(
|
||||
self._flags, max(0, self._height), self._tip, self._country
|
||||
)
|
||||
|
||||
def set_unavailable(self):
|
||||
self._flags &= 0b11111110
|
||||
@@ -109,7 +141,7 @@ class SPVServerStatusProtocol(asyncio.DatagramProtocol):
|
||||
return False
|
||||
|
||||
def make_pong(self, host):
|
||||
return self._cached_response + bytes(int(b) for b in host.split("."))
|
||||
return self._left_cache + SPVPong.encode_address(host) + self._right_cache
|
||||
|
||||
def datagram_received(self, data: bytes, addr: Tuple[str, int]):
|
||||
if self.should_throttle(addr[0]):
|
||||
@@ -119,7 +151,11 @@ class SPVServerStatusProtocol(asyncio.DatagramProtocol):
|
||||
except (ValueError, struct.error, AttributeError, TypeError):
|
||||
# log.exception("derp")
|
||||
return
|
||||
self.transport.sendto(self.make_pong(addr[0]), addr)
|
||||
if addr[1] >= 1024 and is_valid_public_ipv4(
|
||||
addr[0], allow_localhost=self._allow_localhost, allow_lan=self._allow_lan):
|
||||
self.transport.sendto(self.make_pong(addr[0]), addr)
|
||||
else:
|
||||
log.warning("odd packet from %s:%i", addr[0], addr[1])
|
||||
# ping_count_metric.inc()
|
||||
|
||||
def connection_made(self, transport) -> None:
|
||||
@@ -137,12 +173,14 @@ class StatusServer:
|
||||
def __init__(self):
|
||||
self._protocol: Optional[SPVServerStatusProtocol] = None
|
||||
|
||||
async def start(self, height: int, tip: bytes, interface: str, port: int):
|
||||
async def start(self, height: int, tip: bytes, country: str, interface: str, port: int, allow_lan: bool = False):
|
||||
if self.is_running:
|
||||
return
|
||||
loop = asyncio.get_event_loop()
|
||||
self._protocol = SPVServerStatusProtocol(height, tip)
|
||||
interface = interface if interface.lower() != 'localhost' else '127.0.0.1'
|
||||
self._protocol = SPVServerStatusProtocol(
|
||||
height, tip, country, allow_localhost=interface == '127.0.0.1', allow_lan=allow_lan
|
||||
)
|
||||
await loop.create_datagram_endpoint(lambda: self._protocol, (interface, port))
|
||||
log.info("started udp status server on %s:%i", interface, port)
|
||||
|
||||
@@ -169,13 +207,12 @@ class StatusServer:
|
||||
|
||||
|
||||
class SPVStatusClientProtocol(asyncio.DatagramProtocol):
|
||||
PROTOCOL_VERSION = 1
|
||||
|
||||
def __init__(self, responses: asyncio.Queue):
|
||||
super().__init__()
|
||||
self.transport: Optional[asyncio.transports.DatagramTransport] = None
|
||||
self.responses = responses
|
||||
self._ping_packet = SPVPing.make(self.PROTOCOL_VERSION)
|
||||
self._ping_packet = SPVPing.make()
|
||||
|
||||
def datagram_received(self, data: bytes, addr: Tuple[str, int]):
|
||||
try:
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
# need this to avoid circular import
|
||||
PROTOCOL_MIN = (0, 54, 0)
|
||||
PROTOCOL_MAX = (0, 99, 0)
|
||||
PROTOCOL_MAX = (0, 199, 0)
|
||||
@@ -272,16 +272,31 @@ class Output(InputOutput):
|
||||
def id(self):
|
||||
return self.ref.id
|
||||
|
||||
@property
|
||||
def is_pubkey_hash(self):
|
||||
return 'pubkey_hash' in self.script.values
|
||||
|
||||
@property
|
||||
def pubkey_hash(self):
|
||||
return self.script.values['pubkey_hash']
|
||||
|
||||
@property
|
||||
def is_script_hash(self):
|
||||
return 'script_hash' in self.script.values
|
||||
|
||||
@property
|
||||
def script_hash(self):
|
||||
return self.script.values['script_hash']
|
||||
|
||||
@property
|
||||
def has_address(self):
|
||||
return 'pubkey_hash' in self.script.values
|
||||
return self.is_pubkey_hash or self.is_script_hash
|
||||
|
||||
def get_address(self, ledger):
|
||||
return ledger.hash160_to_address(self.pubkey_hash)
|
||||
if self.is_pubkey_hash:
|
||||
return ledger.hash160_to_address(self.pubkey_hash)
|
||||
elif self.is_script_hash:
|
||||
return ledger.hash160_to_script_address(self.script_hash)
|
||||
|
||||
def get_estimator(self, ledger):
|
||||
return OutputEffectiveAmountEstimator(ledger, self)
|
||||
@@ -290,6 +305,10 @@ class Output(InputOutput):
|
||||
def pay_pubkey_hash(cls, amount, pubkey_hash):
|
||||
return cls(amount, OutputScript.pay_pubkey_hash(pubkey_hash))
|
||||
|
||||
@classmethod
|
||||
def pay_script_hash(cls, amount, pubkey_hash):
|
||||
return cls(amount, OutputScript.pay_script_hash(pubkey_hash))
|
||||
|
||||
@classmethod
|
||||
def deserialize_from(cls, stream):
|
||||
return cls(
|
||||
@@ -440,6 +459,12 @@ class Output(InputOutput):
|
||||
self.signable.signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
|
||||
self.script.generate()
|
||||
|
||||
def sign_data(self, data:bytes, timestamp:str) -> str:
|
||||
pieces = [timestamp.encode(), self.claim_hash, data]
|
||||
digest = sha256(b''.join(pieces))
|
||||
signature = self.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
|
||||
return hexlify(signature).decode()
|
||||
|
||||
def clear_signature(self):
|
||||
self.channel = None
|
||||
self.signable.clear_signature()
|
||||
@@ -906,15 +931,18 @@ class Transaction:
|
||||
|
||||
@classmethod
|
||||
def support(cls, claim_name: str, claim_id: str, amount: int, holding_address: str,
|
||||
funding_accounts: List['Account'], change_account: 'Account', signing_channel: Output = None):
|
||||
funding_accounts: List['Account'], change_account: 'Account', signing_channel: Output = None,
|
||||
comment: str = None):
|
||||
ledger, _ = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
|
||||
if signing_channel is not None:
|
||||
if signing_channel is not None or comment is not None:
|
||||
support = Support()
|
||||
support.emoji = '👍'
|
||||
if comment is not None:
|
||||
support.comment = comment
|
||||
support_output = Output.pay_support_data_pubkey_hash(
|
||||
amount, claim_name, claim_id, support, ledger.address_to_hash160(holding_address)
|
||||
)
|
||||
support_output.sign(signing_channel, b'placeholder txid:nout')
|
||||
if signing_channel is not None:
|
||||
support_output.sign(signing_channel, b'placeholder txid:nout')
|
||||
else:
|
||||
support_output = Output.pay_support_pubkey_hash(
|
||||
amount, claim_name, claim_id, ledger.address_to_hash160(holding_address)
|
||||
|
||||
@@ -35,7 +35,7 @@ class WalletServerPayer:
|
||||
if not address or not amount:
|
||||
continue
|
||||
|
||||
if not self.ledger.is_valid_address(address):
|
||||
if not self.ledger.is_pubkey_address(address):
|
||||
self._on_payment_controller.add_error(ServerPaymentInvalidAddressError(address))
|
||||
continue
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ import sys
|
||||
import asyncio
|
||||
from binascii import hexlify
|
||||
|
||||
from lbry.wallet.server.db import SQLDB
|
||||
from lbry.wallet.server.db.writer import SQLDB
|
||||
from lbry.wallet.server.coin import LBC
|
||||
from lbry.wallet.server.daemon import Daemon
|
||||
|
||||
@@ -64,7 +64,7 @@ async def checkcontrolling(daemon: Daemon, db: SQLDB):
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if not len(sys.argv) == 3:
|
||||
if len(sys.argv) != 3:
|
||||
print("usage: <db_file_path> <lbrycrd_url>")
|
||||
sys.exit(1)
|
||||
db_path, lbrycrd_url = sys.argv[1:] # pylint: disable=W0632
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
import asyncio
|
||||
import argparse
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from lbry.dht.constants import generate_id
|
||||
from lbry.dht.node import Node
|
||||
from lbry.dht.peer import PeerManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from lbry.conf import Config
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-4s %(name)s:%(lineno)d: %(message)s")
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def main(host: str, port: int, db_file_path: str, bootstrap_node: Optional[str]):
|
||||
loop = asyncio.get_event_loop()
|
||||
conf = Config()
|
||||
storage = SQLiteStorage(conf, db_file_path, loop, loop.time)
|
||||
if bootstrap_node:
|
||||
nodes = bootstrap_node.split(':')
|
||||
nodes = [(nodes[0], int(nodes[1]))]
|
||||
else:
|
||||
nodes = conf.known_dht_nodes
|
||||
await storage.open()
|
||||
node = Node(
|
||||
loop, PeerManager(loop), generate_id(), port, port, 3333, None,
|
||||
storage=storage
|
||||
)
|
||||
node.start(host, nodes)
|
||||
while True:
|
||||
await asyncio.sleep(10)
|
||||
log.info("Known peers: %d. Storing contact information for %d blobs from %d peers.",
|
||||
len(node.protocol.routing_table.get_peers()), len(node.protocol.data_store),
|
||||
len(node.protocol.data_store.get_storing_contacts()))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Starts a single DHT node, which then can be used as a seed node or just a contributing node.")
|
||||
parser.add_argument("--host", default='0.0.0.0', type=str, help="Host to listen for requests. Default: 0.0.0.0")
|
||||
parser.add_argument("--port", default=4444, type=int, help="Port to listen for requests. Default: 4444")
|
||||
parser.add_argument("--db_file", default='/tmp/dht.db', type=str, help="DB file to save peers. Default: /tmp/dht.db")
|
||||
parser.add_argument("--bootstrap_node", default=None, type=str,
|
||||
help="Node to connect for bootstraping this node. Leave unset to use the default ones. "
|
||||
"Format: host:port Example: lbrynet1.lbry.com:4444")
|
||||
args = parser.parse_args()
|
||||
asyncio.run(main(args.host, args.port, args.db_file, args.bootstrap_node))
|
||||
@@ -1,3 +1,19 @@
|
||||
"""A simple script that attempts to directly download a single blob.
|
||||
|
||||
To Do:
|
||||
------
|
||||
Currently `lbrynet blob get <hash>` does not work to download single blobs
|
||||
which are not already present in the system. The function locks up and
|
||||
never returns.
|
||||
It only works for blobs that are in the `blobfiles` directory already.
|
||||
|
||||
This bug is reported in lbryio/lbry-sdk, issue #2070.
|
||||
|
||||
Maybe this script can be investigated, and certain parts can be added to
|
||||
`lbry.extras.daemon.daemon.jsonrpc_blob_get`
|
||||
in order to solve the previous issue, and finally download single blobs
|
||||
from the network (peers or reflector servers).
|
||||
"""
|
||||
import sys
|
||||
import os
|
||||
import asyncio
|
||||
@@ -47,7 +63,11 @@ async def main(blob_hash: str, url: str):
|
||||
print(f"deleted {blob_hash}")
|
||||
|
||||
|
||||
if __name__ == "__main__": # usage: python download_blob_from_peer.py <blob_hash> [host url:port]
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2:
|
||||
print("usage: download_blob_from_peer.py <blob_hash> [host_url:port]")
|
||||
sys.exit(1)
|
||||
|
||||
url = 'reflector.lbry.com:5567'
|
||||
if len(sys.argv) > 2:
|
||||
url = sys.argv[2]
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import asyncio
|
||||
from pprint import pprint
|
||||
|
||||
from elasticsearch import AsyncElasticsearch
|
||||
from elasticsearch._async.helpers import async_scan, async_bulk
|
||||
|
||||
DB = {}
|
||||
INDEX = 'claims'
|
||||
|
||||
|
||||
async def generate_support_amounts(client: AsyncElasticsearch):
|
||||
async for doc in async_scan(client):
|
||||
DB[doc['_id']] = doc['_source']['support_amount']
|
||||
if len(DB) > 10:
|
||||
break
|
||||
pprint(DB)
|
||||
|
||||
|
||||
def generate_support_to_trending():
|
||||
for claim_id, amount in DB.items():
|
||||
yield {'doc': {"trending_mixed": amount}, '_id': claim_id, '_index': INDEX, '_op_type': 'update'}
|
||||
|
||||
|
||||
async def write_trending(client: AsyncElasticsearch):
|
||||
await async_bulk(client, generate_support_to_trending())
|
||||
|
||||
|
||||
def get_client(host='localhost', port=9201):
|
||||
hosts = [{'host': host, 'port': port}]
|
||||
return AsyncElasticsearch(hosts, timeout=port)
|
||||
|
||||
|
||||
async def run():
|
||||
client = get_client()
|
||||
await generate_support_amounts(client)
|
||||
await write_trending(client)
|
||||
for claim_id, value in DB.items():
|
||||
if value > 0:
|
||||
break
|
||||
doc = await client.get(INDEX, claim_id)
|
||||
pprint(doc)
|
||||
pprint(DB[claim_id])
|
||||
await client.close()
|
||||
|
||||
|
||||
asyncio.get_event_loop().run_until_complete(run())
|
||||
@@ -2,7 +2,7 @@ import time
|
||||
import asyncio
|
||||
import random
|
||||
from argparse import ArgumentParser
|
||||
from lbry.wallet.client.basenetwork import ClientSession
|
||||
from lbry.wallet.network import ClientSession
|
||||
|
||||
|
||||
class AgentSmith(ClientSession):
|
||||
|
||||
@@ -13,7 +13,6 @@ from lbry.extras.cli import set_kwargs, get_argument_parser
|
||||
from lbry.extras.daemon.daemon import (
|
||||
Daemon, jsonrpc_dumps_pretty, encode_pagination_doc
|
||||
)
|
||||
from tests.integration.other.test_comment_commands import MockedCommentServer
|
||||
from lbry.extras.daemon.json_response_encoder import (
|
||||
encode_tx_doc, encode_txo_doc, encode_account_doc, encode_file_doc,
|
||||
encode_wallet_doc
|
||||
@@ -67,10 +66,6 @@ class Examples(CommandTestCase):
|
||||
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
self.daemon.conf.comment_server = 'http://localhost:2903/api'
|
||||
self.comment_server = MockedCommentServer(2903)
|
||||
await self.comment_server.start()
|
||||
self.addCleanup(self.comment_server.stop)
|
||||
self.recorder = ExampleRecorder(self)
|
||||
|
||||
async def play(self):
|
||||
@@ -330,41 +325,6 @@ class Examples(CommandTestCase):
|
||||
await self.daemon.jsonrpc_channel_abandon(self.get_claim_id(big_stream))
|
||||
await self.generate(1)
|
||||
|
||||
# comments
|
||||
|
||||
comment = await r(
|
||||
'Posting a comment as your channel',
|
||||
'comment', 'create', '--comment="Thank you Based God"',
|
||||
'--channel_name=@channel', f'--claim_id={stream_id}'
|
||||
)
|
||||
|
||||
reply = await r(
|
||||
'Use the parent_id param to make replies',
|
||||
'comment', 'create',
|
||||
'--comment="I have photographic evidence confirming Sasquatch exists"',
|
||||
f'--channel_name=@channel', f'--parent_id={comment["comment_id"]}',
|
||||
f'--claim_id={stream_id}'
|
||||
)
|
||||
|
||||
await r(
|
||||
'List all comments on a claim',
|
||||
'comment', 'list', stream_id, '--include_replies'
|
||||
)
|
||||
|
||||
await r(
|
||||
'List a comment thread replying to a top level comment',
|
||||
'comment', 'list', stream_id,
|
||||
f'--parent_id={comment["comment_id"]}'
|
||||
)
|
||||
|
||||
await r(
|
||||
'Edit the contents of a comment',
|
||||
'comment', 'update', 'Where there was once sasquatch, there is not',
|
||||
f'--comment_id={comment["comment_id"]}'
|
||||
)
|
||||
|
||||
await self.daemon.jsonrpc_comment_abandon(reply['comment_id'])
|
||||
|
||||
# collections
|
||||
collection = await r(
|
||||
'Create a collection of one stream',
|
||||
@@ -431,11 +391,6 @@ class Examples(CommandTestCase):
|
||||
|
||||
# abandon all the things
|
||||
|
||||
await r(
|
||||
'Abandon a comment',
|
||||
'comment', 'abandon', comment['comment_id']
|
||||
)
|
||||
|
||||
abandon_stream = await r(
|
||||
'Abandon a stream claim',
|
||||
'stream', 'abandon', stream_id
|
||||
|
||||
+6
-4
@@ -151,19 +151,21 @@ def release(args):
|
||||
for pr in gh.search_issues(f"merged:>={previous_release._json_data['created_at']} repo:lbryio/lbry-sdk"):
|
||||
area_labels = list(get_labels(pr, 'area'))
|
||||
type_label = get_label(pr, 'type')
|
||||
pr_url = f'[#{pr.number}]({pr.html_url})'
|
||||
user_url = f'[{pr.user["login"]}]({pr.user["html_url"]})'
|
||||
if area_labels and type_label:
|
||||
for area_name in area_labels:
|
||||
for incompat in get_backwards_incompatible(pr.body or ""):
|
||||
incompats.append(f' * [{area_name}] {incompat.strip()} ({pr.html_url})')
|
||||
incompats.append(f' * [{area_name}] {incompat.strip()} ({pr_url})')
|
||||
for release_text in get_release_text(pr.body or ""):
|
||||
release_texts.append(release_text)
|
||||
if type_label == 'fixup':
|
||||
fixups.append(f' * {pr.title} ({pr.html_url}) by {pr.user["login"]}')
|
||||
fixups.append(f' * {pr.title} ({pr_url}) by {user_url}')
|
||||
else:
|
||||
area = areas.setdefault(area_name, [])
|
||||
area.append(f' * [{type_label}] {pr.title} ({pr.html_url}) by {pr.user["login"]}')
|
||||
area.append(f' * [{type_label}] {pr.title} ({pr_url}) by {user_url}')
|
||||
else:
|
||||
unlabeled.append(f' * {pr.title} ({pr.html_url}) by {pr.user["login"]}')
|
||||
unlabeled.append(f' * {pr.title} ({pr_url}) by {user_url}')
|
||||
|
||||
area_names = list(areas.keys())
|
||||
area_names.sort()
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import asyncio
|
||||
from lbry.wallet.client.basenetwork import ClientSession
|
||||
from lbry.wallet.network import ClientSession
|
||||
from lbry.wallet.rpc.jsonrpc import RPCError
|
||||
import logging
|
||||
import json
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import asyncio
|
||||
from aiohttp import web
|
||||
|
||||
from lbry.blob_exchange.serialization import BlobRequest, BlobResponse
|
||||
from lbry.dht.constants import generate_id
|
||||
from lbry.dht.node import Node
|
||||
from lbry.dht.peer import make_kademlia_peer, PeerManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
NODE = Node(
|
||||
loop, PeerManager(loop), generate_id(), 60600, 60600, 3333, None,
|
||||
storage=SQLiteStorage(None, ":memory:", loop, loop.time)
|
||||
)
|
||||
|
||||
|
||||
async def check_p2p(ip, port):
|
||||
writer = None
|
||||
try:
|
||||
reader, writer = await asyncio.open_connection(ip, port)
|
||||
writer.write(BlobRequest.make_request_for_blob_hash('0'*96).serialize())
|
||||
return BlobResponse.deserialize(await reader.readuntil(b'}')).get_address_response().lbrycrd_address
|
||||
except OSError:
|
||||
return None
|
||||
finally:
|
||||
if writer:
|
||||
writer.close()
|
||||
await writer.wait_closed()
|
||||
|
||||
|
||||
async def check_dht(ip, port):
|
||||
peer = make_kademlia_peer(None, ip, udp_port=int(port))
|
||||
return await NODE.protocol.get_rpc_peer(peer).ping()
|
||||
|
||||
|
||||
async def endpoint_p2p(request):
|
||||
p2p_port = request.match_info.get('p2p_port', "3333")
|
||||
try:
|
||||
address = await asyncio.wait_for(check_p2p(request.remote, p2p_port), 3)
|
||||
except asyncio.TimeoutError:
|
||||
address = None
|
||||
return {"status": address is not None, "port": p2p_port, "payment_address": address}
|
||||
|
||||
|
||||
async def endpoint_dht(request):
|
||||
dht_port = request.match_info.get('dht_port', "3333")
|
||||
try:
|
||||
response = await check_dht(request.remote, dht_port)
|
||||
except asyncio.TimeoutError:
|
||||
response = None
|
||||
return {"status": response == b'pong', "port": dht_port}
|
||||
|
||||
|
||||
async def endpoint_default(request):
|
||||
return {"dht_status": await endpoint_dht(request), "p2p_status": await endpoint_p2p(request)}
|
||||
|
||||
|
||||
def as_json_response_wrapper(endpoint):
|
||||
async def json_endpoint(*args, **kwargs):
|
||||
return web.json_response(await endpoint(*args, **kwargs))
|
||||
return json_endpoint
|
||||
|
||||
|
||||
app = web.Application()
|
||||
app.add_routes([web.get('/', as_json_response_wrapper(endpoint_default)),
|
||||
web.get('/dht/{dht_port}', as_json_response_wrapper(endpoint_dht)),
|
||||
web.get('/p2p/{p2p_port}', as_json_response_wrapper(endpoint_p2p))])
|
||||
|
||||
if __name__ == '__main__':
|
||||
loop.create_task(NODE.start_listening("0.0.0.0"))
|
||||
web.run_app(app, port=60666)
|
||||
Loaded 100 of 132 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user