Compare commits
417
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fd3c1c50f1 | ||
|
|
b153e4bb9f | ||
|
|
db9856a8db | ||
|
|
75ecea265d | ||
|
|
be8751cb73 | ||
|
|
fb25ecb4a1 | ||
|
|
f1cb7d27ac | ||
|
|
dee494e12f | ||
|
|
b13a121915 | ||
|
|
7486ee9537 | ||
|
|
4a20ccc28e | ||
|
|
f80dd2b307 | ||
|
|
b208cf6d32 | ||
|
|
39e78ff17e | ||
|
|
a8177ea7fe | ||
|
|
bedcfc154b | ||
|
|
4c38b4aa3c | ||
|
|
f6cfe266e0 | ||
|
|
4905e65f14 | ||
|
|
ccb250b410 | ||
|
|
aca57ffc62 | ||
|
|
7f375f42d8 | ||
|
|
eedcc2034d | ||
|
|
24c9a167d7 | ||
|
|
909df8ef1f | ||
|
|
3b27cb3671 | ||
|
|
3fe0db4a7d | ||
|
|
8b55814ab2 | ||
|
|
575e471553 | ||
|
|
0f5f1aebed | ||
|
|
50e17eb1ab | ||
|
|
158cc2f660 | ||
|
|
1066a31acd | ||
|
|
1f9d0f4582 | ||
|
|
a6d65233f1 | ||
|
|
eff2fe7a1b | ||
|
|
20efdc70b3 | ||
|
|
f0d8fb8f1a | ||
|
|
f7a380e9b7 | ||
|
|
e9c7cf6f63 | ||
|
|
68f1661452 | ||
|
|
36fd1b91ae | ||
|
|
a4ec430ac0 | ||
|
|
519614b2fd | ||
|
|
bf0118c8ef | ||
|
|
a4db0820bc | ||
|
|
ee7528413e | ||
|
|
7952fc8324 | ||
|
|
652773d2cf | ||
|
|
2a17787242 | ||
|
|
0a53ad5721 | ||
|
|
6da6bdc863 | ||
|
|
42ad2bb83f | ||
|
|
f309a65cb4 | ||
|
|
77e19ab1a4 | ||
|
|
1a996b6ef3 | ||
|
|
b882f1a010 | ||
|
|
82a030e6ff | ||
|
|
0758b85179 | ||
|
|
ab3d9bd080 | ||
|
|
8ff813f689 | ||
|
|
88ae72c0d3 | ||
|
|
312aa4be26 | ||
|
|
cbb06fce9d | ||
|
|
f259e497c4 | ||
|
|
dd4172ac66 | ||
|
|
66029e60d3 | ||
|
|
364f484f04 | ||
|
|
9dd5159414 | ||
|
|
13e38d6fd8 | ||
|
|
10dcb64715 | ||
|
|
7551b51e7d | ||
|
|
adb418aafc | ||
|
|
270da80d64 | ||
|
|
b2027cfd66 | ||
|
|
7f1f4eeac6 | ||
|
|
7a7446c8bd | ||
|
|
ddbae294e6 | ||
|
|
8c71b744f3 | ||
|
|
479b5d31a9 | ||
|
|
4cbf4230e8 | ||
|
|
6a610187e0 | ||
|
|
eb2a4aebba | ||
|
|
21a2e67755 | ||
|
|
3b9e312615 | ||
|
|
26dab04c9e | ||
|
|
00713c0d11 | ||
|
|
751b5f3027 | ||
|
|
e8261b000e | ||
|
|
41ecb70297 | ||
|
|
09ee104b8c | ||
|
|
e3a4964787 | ||
|
|
9bf72910a4 | ||
|
|
ee39e20e6d | ||
|
|
399d6db6f6 | ||
|
|
0821ce44b5 | ||
|
|
ea279111c6 | ||
|
|
674ce02e58 | ||
|
|
8dfa2767ec | ||
|
|
20dad7f07f | ||
|
|
751cc4c44d | ||
|
|
2318e6d8e9 | ||
|
|
61b4a492c3 | ||
|
|
9db3d01e09 | ||
|
|
8da73ad3dd | ||
|
|
b8c16d8ac5 | ||
|
|
429c0951f3 | ||
|
|
74e103c791 | ||
|
|
f941950ee2 | ||
|
|
846df2eef1 | ||
|
|
34ed058c97 | ||
|
|
eae0290978 | ||
|
|
561368570e | ||
|
|
3467d1fed0 | ||
|
|
d02ff232e5 | ||
|
|
2d1c6a5402 | ||
|
|
eab3b65629 | ||
|
|
20b435732a | ||
|
|
929617273d | ||
|
|
2717bf7d49 | ||
|
|
5cd2ebc960 | ||
|
|
9b4afe9816 | ||
|
|
23bb5598d5 | ||
|
|
af1d7813e9 | ||
|
|
16c2e5a585 | ||
|
|
c02750edbd | ||
|
|
7204ddafec | ||
|
|
faeba9a7e4 | ||
|
|
190d238a1f | ||
|
|
715451b5fb | ||
|
|
87f1895405 | ||
|
|
923d817751 | ||
|
|
0728209b66 | ||
|
|
b8b9dcc2ee | ||
|
|
f35e879852 | ||
|
|
34f4f12eb9 | ||
|
|
fa63bf758d | ||
|
|
f6b396ae64 | ||
|
|
2c7fd58e34 | ||
|
|
982f2c9634 | ||
|
|
f2fd42b47a | ||
|
|
1b4ccad938 | ||
|
|
a9de1ce8e0 | ||
|
|
ac752d5ec2 | ||
|
|
632d8d02d2 | ||
|
|
48aeb26e02 | ||
|
|
1694af8b5e | ||
|
|
83bcab9cd2 | ||
|
|
bdc7f4b3f5 | ||
|
|
39202a3d79 | ||
|
|
912065a121 | ||
|
|
c8466afac2 | ||
|
|
2619e162c1 | ||
|
|
e1112e17f8 | ||
|
|
92b2ead74c | ||
|
|
bbed9b94c1 | ||
|
|
73d07311db | ||
|
|
1cdff47477 | ||
|
|
511a5c3f82 | ||
|
|
853885e2ff | ||
|
|
d83936a66a | ||
|
|
5517d2bf56 | ||
|
|
f21ab49ac5 | ||
|
|
925a458abe | ||
|
|
76946c447f | ||
|
|
2faa29b1c4 | ||
|
|
6826cc311d | ||
|
|
5e17ce0a0b | ||
|
|
e8d299d3b6 | ||
|
|
7637aa2ab6 | ||
|
|
ab067d1d3a | ||
|
|
e6f84666c7 | ||
|
|
4c5429af15 | ||
|
|
0a0ac3b7c9 | ||
|
|
24833ce9fb | ||
|
|
ec2c18dc87 | ||
|
|
7384609e74 | ||
|
|
3047649650 | ||
|
|
d298dac3f3 | ||
|
|
1574bca8a8 | ||
|
|
ec2f6c6b80 | ||
|
|
838cc60161 | ||
|
|
310c61a5cc | ||
|
|
318cc15323 | ||
|
|
3a64ceb4d6 | ||
|
|
d0f21c0095 | ||
|
|
46dc15dd29 | ||
|
|
8dc654b513 | ||
|
|
7000ac3f3f | ||
|
|
43c2e8d8e9 | ||
|
|
0231139b01 | ||
|
|
d6ee6446dd | ||
|
|
7b666efcf8 | ||
|
|
eb5d2198fc | ||
|
|
34e44ebd1c | ||
|
|
bf2f4bc040 | ||
|
|
9dc4559aba | ||
|
|
eba8856261 | ||
|
|
a8c5aa471a | ||
|
|
52f6dcf092 | ||
|
|
dec79f3742 | ||
|
|
8bdcac0f3e | ||
|
|
8426b674a3 | ||
|
|
da391bcc8d | ||
|
|
2d7443d454 | ||
|
|
991987ed76 | ||
|
|
ec24ebf2cf | ||
|
|
6ba0976085 | ||
|
|
2b88d01a01 | ||
|
|
0c09f24cbf | ||
|
|
61d22afeba | ||
|
|
9f1ed6e8c3 | ||
|
|
bbc4113cac | ||
|
|
91194bf422 | ||
|
|
9c5f940b00 | ||
|
|
455b4043b8 | ||
|
|
bd83ee7931 | ||
|
|
f6bdf7c09a | ||
|
|
2db8afb8c2 | ||
|
|
e033129dd3 | ||
|
|
f9dc590100 | ||
|
|
8996aafe0d | ||
|
|
9dc25ef7af | ||
|
|
2b84f4d407 | ||
|
|
097c8b674c | ||
|
|
ba649d4b94 | ||
|
|
6ed1614db0 | ||
|
|
df5b6a8380 | ||
|
|
1f82a8b99e | ||
|
|
0c95d96f32 | ||
|
|
c2f5f84118 | ||
|
|
b3b5e3d8f0 | ||
|
|
506d3f3cd9 | ||
|
|
516a8c5ee5 | ||
|
|
2f81e9d374 | ||
|
|
2d8703bb8d | ||
|
|
76e60d9bc3 | ||
|
|
9d5370be5f | ||
|
|
fc1a06bc45 | ||
|
|
fce80374f4 | ||
|
|
420c9f10c2 | ||
|
|
5a39681a2e | ||
|
|
7a1b7db7c8 | ||
|
|
03a643da52 | ||
|
|
383f0ce450 | ||
|
|
c6c668676c | ||
|
|
7b01dde063 | ||
|
|
8c25f65024 | ||
|
|
1e478e3545 | ||
|
|
6c75a8978b | ||
|
|
640267fc2b | ||
|
|
33c7c3ee12 | ||
|
|
f9b41d34ae | ||
|
|
d4bec79451 | ||
|
|
ac1a8b4daf | ||
|
|
28838c1759 | ||
|
|
50ecb0dac9 | ||
|
|
e22bc01cbd | ||
|
|
6c28713a4c | ||
|
|
fc9023386c | ||
|
|
e6cae9bcc3 | ||
|
|
a9eeca1302 | ||
|
|
8c695e42ca | ||
|
|
0aa7fd47d5 | ||
|
|
70596042d6 | ||
|
|
caf616234b | ||
|
|
375187aa70 | ||
|
|
71eccdc0e3 | ||
|
|
639b1e48f5 | ||
|
|
0bb4cb4472 | ||
|
|
cc51543851 | ||
|
|
22540390e1 | ||
|
|
98565eb67c | ||
|
|
cf6a47ecb7 | ||
|
|
fa60b9f9d3 | ||
|
|
644120ca31 | ||
|
|
a50a625b3b | ||
|
|
7297c13331 | ||
|
|
f73399bfac | ||
|
|
a056cd78f7 | ||
|
|
a30f3c86c2 | ||
|
|
e70bdd86a7 | ||
|
|
bc9f33c2e0 | ||
|
|
872b89ee93 | ||
|
|
ae53062518 | ||
|
|
17f76c9cb3 | ||
|
|
5de944146a | ||
|
|
9dc6092cb0 | ||
|
|
a32a2ef04e | ||
|
|
ecfa0ae3da | ||
|
|
03595052ce | ||
|
|
1f94c53dd2 | ||
|
|
9c426373f2 | ||
|
|
c03e30a01f | ||
|
|
07f7a77ac0 | ||
|
|
3b9ea2c9a4 | ||
|
|
1beb13dd80 | ||
|
|
ddae84abb3 | ||
|
|
863b9a2c98 | ||
|
|
9d44bbdb48 | ||
|
|
8d93dd5adc | ||
|
|
48502961cf | ||
|
|
3c8bec61d3 | ||
|
|
7296c7df1a | ||
|
|
f3ee6603de | ||
|
|
ee0aabda1d | ||
|
|
08d37a4b0f | ||
|
|
f975ea99cb | ||
|
|
f030d41dc7 | ||
|
|
8d079bfcd1 | ||
|
|
d39b8654a6 | ||
|
|
ce7816a968 | ||
|
|
2ee572e68f | ||
|
|
4bbd850898 | ||
|
|
34eae6e608 | ||
|
|
6a0302fec6 | ||
|
|
c94cc293c2 | ||
|
|
cae7792a1e | ||
|
|
7f6b2fe4f1 | ||
|
|
eba430bbc0 | ||
|
|
01280c8d04 | ||
|
|
ec141ac9c9 | ||
|
|
590c892a6a | ||
|
|
ff8a50c366 | ||
|
|
46ef6c8ab7 | ||
|
|
b09eabc478 | ||
|
|
e49fcea6e3 | ||
|
|
68ed9f4ffc | ||
|
|
af94687d45 | ||
|
|
26964ecf0f | ||
|
|
77d19af359 | ||
|
|
120b82f243 | ||
|
|
a0fea30a11 | ||
|
|
8bb3e0a64d | ||
|
|
bbded12923 | ||
|
|
e8ba5d7606 | ||
|
|
af66b31a44 | ||
|
|
b000a40f28 | ||
|
|
3c85322523 | ||
|
|
a469b8bc04 | ||
|
|
78b8261a3a | ||
|
|
f20ca70c01 | ||
|
|
4e4148fc1c | ||
|
|
c22482f907 | ||
|
|
870e139fce | ||
|
|
4d58648c02 | ||
|
|
ebbb182537 | ||
|
|
1cd5377b45 | ||
|
|
1d1f0527ee | ||
|
|
37a5f77415 | ||
|
|
ced3c7efe4 | ||
|
|
c3b8f366ed | ||
|
|
de78876b1a | ||
|
|
64c25b049c | ||
|
|
8a4fe4f3ad | ||
|
|
8811b8c1fd | ||
|
|
190b01fdf9 | ||
|
|
f145d08c10 | ||
|
|
53382b7e15 | ||
|
|
6ad0242617 | ||
|
|
a7c2408c0a | ||
|
|
ce1eabaed6 | ||
|
|
f602541ede | ||
|
|
3f718e6efc | ||
|
|
ce7a985df6 | ||
|
|
6d83f7e7bd | ||
|
|
b73c00943c | ||
|
|
abaac8ef48 | ||
|
|
a2f8e7068e | ||
|
|
4d47873219 | ||
|
|
cf985486e5 | ||
|
|
b930c3fc93 | ||
|
|
dd26a96828 | ||
|
|
6865ddfc12 | ||
|
|
e888e69d4d | ||
|
|
2089059792 | ||
|
|
27739e0364 | ||
|
|
698ee271d6 | ||
|
|
543c75b293 | ||
|
|
b09c46f6f7 | ||
|
|
f2cc19e6aa | ||
|
|
814a0a123f | ||
|
|
179383540f | ||
|
|
8be1c8310d | ||
|
|
ef02d776ca | ||
|
|
750ff448ad | ||
|
|
e3abab6d4d | ||
|
|
d3ffae72fb | ||
|
|
87f751188e | ||
|
|
3469abaefd | ||
|
|
797364ee5c | ||
|
|
36c05fc4b9 | ||
|
|
79624febc0 | ||
|
|
0a9d4de126 | ||
|
|
b6f6994db4 | ||
|
|
ff7bed720a | ||
|
|
00c0f48b02 | ||
|
|
94c45cf2a0 | ||
|
|
58f77b2a1c | ||
|
|
7170e69b22 | ||
|
|
239ee2437c | ||
|
|
ced368db31 | ||
|
|
6a991e5c15 | ||
|
|
523d22262b | ||
|
|
08197a327e | ||
|
|
b7cb2a7aa5 | ||
|
|
decc5c74ef | ||
|
|
fbe0f886b6 | ||
|
|
7e23d6e2ef | ||
|
|
49458d1085 | ||
|
|
289c12bd9a | ||
|
|
9a6326b027 | ||
|
|
8269d2f83c | ||
|
|
5a4b6be974 | ||
|
|
35e8ce60a9 | ||
|
|
8b6dd9f603 | ||
|
|
8586762dde |
No files matched your search
@@ -67,6 +67,9 @@ jobs:
|
||||
run: |
|
||||
pip install pyinstaller
|
||||
pip install -e .
|
||||
# https://stackoverflow.com/a/61693590
|
||||
# https://github.com/pypa/setuptools/issues/1963
|
||||
pip install --upgrade 'setuptools<45.0.0'
|
||||
- if: startsWith(matrix.os, 'windows') == false
|
||||
name: Build & Run (Unix)
|
||||
run: |
|
||||
|
||||
+6
-1
@@ -6,12 +6,17 @@
|
||||
/.coverage*
|
||||
/lbry-venv
|
||||
/venv
|
||||
/lbry/blockchain
|
||||
|
||||
lbry.egg-info
|
||||
__pycache__
|
||||
_trial_temp/
|
||||
trending*.log
|
||||
|
||||
/tests/integration/blockchain/files
|
||||
/tests/.coverage.*
|
||||
|
||||
/lbry/wallet/bin
|
||||
/lbry/wallet/bin
|
||||
|
||||
/.vscode
|
||||
/.gitignore
|
||||
+5
-2
@@ -41,7 +41,7 @@ test:datanetwork-integration:
|
||||
stage: test
|
||||
script:
|
||||
- pip install tox-travis
|
||||
- tox -e datanetwork
|
||||
- tox -e datanetwork --recreate
|
||||
|
||||
test:blockchain-integration:
|
||||
stage: test
|
||||
@@ -94,6 +94,7 @@ build:linux:
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends python3.7-dev
|
||||
- python3.7 <(curl -q https://bootstrap.pypa.io/get-pip.py) # make sure we get pip with python3.7
|
||||
- pip install lbry-libtorrent
|
||||
|
||||
build:mac:
|
||||
extends: .build
|
||||
@@ -120,9 +121,11 @@ build:windows:
|
||||
GIT_DEPTH: 5
|
||||
before_script:
|
||||
- ./docker/install_choco.ps1
|
||||
- choco install -y --x86 python3 7zip checksum
|
||||
- choco install -y --x64 python --version=3.7.9
|
||||
- choco install -y 7zip checksum
|
||||
- python --version # | findstr /B "Python 3\.7\." # dont upgrade python on every run. just make sure we're on the right Python
|
||||
- pip --version # | findstr /E '\(python 3\.7\)'
|
||||
- python -c "import sys;print(f'{str(64 if sys.maxsize > 2**32 else 32)} bit python\n{sys.platform}')"
|
||||
- pip install virtualenv pywin32
|
||||
- virtualenv venv
|
||||
- venv/Scripts/activate.ps1
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
.PHONY: install tools lint test idea
|
||||
|
||||
install:
|
||||
pip install https://s3.amazonaws.com/files.lbry.io/python_libtorrent-1.2.4-py3-none-any.whl
|
||||
CFLAGS="-DSQLITE_MAX_VARIABLE_NUMBER=2500000" pip install -U https://github.com/rogerbinns/apsw/releases/download/3.30.1-r1/apsw-3.30.1-r1.zip \
|
||||
--global-option=fetch \
|
||||
--global-option=--version --global-option=3.30.1 --global-option=--all \
|
||||
@@ -8,7 +9,7 @@ install:
|
||||
pip install -e .
|
||||
|
||||
tools:
|
||||
pip install mypy==0.701
|
||||
pip install mypy==0.701 pylint==2.4.4
|
||||
pip install coverage astroid pylint
|
||||
|
||||
lint:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM ubuntu:19.10
|
||||
FROM ubuntu:20.04
|
||||
|
||||
ARG user=lbry
|
||||
ARG db_dir=/database
|
||||
|
||||
@@ -16,7 +16,8 @@ services:
|
||||
- "lbrycrd:/data/.lbrycrd"
|
||||
environment:
|
||||
- RUN_MODE=default
|
||||
- SNAPSHOT_URL=${LBRYCRD_SNAPSHOT_URL-https://lbry.com/snapshot/blockchain}
|
||||
# Curently not snapshot provided
|
||||
#- SNAPSHOT_URL=${LBRYCRD_SNAPSHOT_URL-https://lbry.com/snapshot/blockchain}
|
||||
- RPC_ALLOW_IP=0.0.0.0/0
|
||||
wallet_server:
|
||||
image: lbry/wallet-server:${WALLET_SERVER_TAG:-latest-release}
|
||||
@@ -30,5 +31,6 @@ services:
|
||||
volumes:
|
||||
- "wallet_server:/database"
|
||||
environment:
|
||||
- SNAPSHOT_URL=${WALLET_SERVER_SNAPSHOT_URL-https://lbry.com/snapshot/wallet}
|
||||
# Curently not snapshot provided
|
||||
# - SNAPSHOT_URL=${WALLET_SERVER_SNAPSHOT_URL-https://lbry.com/snapshot/wallet}
|
||||
- DAEMON_URL=http://lbry:lbry@lbrycrd:9245
|
||||
+549
-167
File diff suppressed because it is too large.
Load diff
+1
-1
@@ -1,2 +1,2 @@
|
||||
__version__ = "0.70.0"
|
||||
__version__ = "0.90.0"
|
||||
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
|
||||
+24
-10
@@ -110,7 +110,7 @@ class AbstractBlob:
|
||||
if reader in self.readers:
|
||||
self.readers.remove(reader)
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
def _write_blob(self, blob_bytes: bytes) -> asyncio.Task:
|
||||
raise NotImplementedError()
|
||||
|
||||
def set_length(self, length) -> None:
|
||||
@@ -198,11 +198,17 @@ class AbstractBlob:
|
||||
def save_verified_blob(self, verified_bytes: bytes):
|
||||
if self.verified.is_set():
|
||||
return
|
||||
if self.is_writeable():
|
||||
self._write_blob(verified_bytes)
|
||||
|
||||
def update_events(_):
|
||||
self.verified.set()
|
||||
self.writing.clear()
|
||||
|
||||
if self.is_writeable():
|
||||
self.writing.set()
|
||||
task = self._write_blob(verified_bytes)
|
||||
task.add_done_callback(update_events)
|
||||
if self.blob_completed_callback:
|
||||
self.blob_completed_callback(self)
|
||||
task.add_done_callback(lambda _: self.blob_completed_callback(self))
|
||||
|
||||
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
|
||||
peer_port: typing.Optional[int] = None) -> HashBlobWriter:
|
||||
@@ -261,9 +267,11 @@ class BlobBuffer(AbstractBlob):
|
||||
self.verified.clear()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
if self._verified_bytes:
|
||||
raise OSError("already have bytes for blob")
|
||||
self._verified_bytes = BytesIO(blob_bytes)
|
||||
async def write():
|
||||
if self._verified_bytes:
|
||||
raise OSError("already have bytes for blob")
|
||||
self._verified_bytes = BytesIO(blob_bytes)
|
||||
return self.loop.create_task(write())
|
||||
|
||||
def delete(self):
|
||||
if self._verified_bytes:
|
||||
@@ -319,13 +327,19 @@ class BlobFile(AbstractBlob):
|
||||
handle.close()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
with open(self.file_path, 'wb') as f:
|
||||
f.write(blob_bytes)
|
||||
def _write_blob():
|
||||
with open(self.file_path, 'wb') as f:
|
||||
f.write(blob_bytes)
|
||||
|
||||
async def write_blob():
|
||||
await self.loop.run_in_executor(None, _write_blob)
|
||||
|
||||
return self.loop.create_task(write_blob())
|
||||
|
||||
def delete(self):
|
||||
super().delete()
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
return super().delete()
|
||||
|
||||
@classmethod
|
||||
async def create_from_unencrypted(
|
||||
|
||||
@@ -2,7 +2,7 @@ import os
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
from lbry.utils import LRUCache
|
||||
from lbry.utils import LRUCacheWithMetrics
|
||||
from lbry.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
|
||||
from lbry.stream.descriptor import StreamDescriptor
|
||||
from lbry.connection_manager import ConnectionManager
|
||||
@@ -32,7 +32,7 @@ class BlobManager:
|
||||
else self._node_data_store.completed_blobs
|
||||
self.blobs: typing.Dict[str, AbstractBlob] = {}
|
||||
self.config = config
|
||||
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCache(
|
||||
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCacheWithMetrics(
|
||||
self.config.blob_lru_cache_size)
|
||||
self.connection_manager = ConnectionManager(loop)
|
||||
|
||||
|
||||
@@ -152,6 +152,8 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
log.debug(msg)
|
||||
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
|
||||
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
|
||||
# wait for the io to finish
|
||||
await self.blob.verified.wait()
|
||||
log.info("%s at %fMB/s", msg,
|
||||
round((float(self._blob_bytes_received) /
|
||||
float(time.perf_counter() - start_time)) / 1000000.0, 2))
|
||||
|
||||
+35
-17
@@ -5,9 +5,8 @@ import typing
|
||||
import logging
|
||||
from argparse import ArgumentParser
|
||||
from contextlib import contextmanager
|
||||
|
||||
import yaml
|
||||
from appdirs import user_data_dir, user_config_dir
|
||||
import yaml
|
||||
from lbry.error import InvalidCurrencyError
|
||||
from lbry.dht import constants
|
||||
from lbry.wallet.coinselection import STRATEGIES
|
||||
@@ -277,14 +276,23 @@ class Strings(ListSetting):
|
||||
class EnvironmentAccess:
|
||||
PREFIX = 'LBRY_'
|
||||
|
||||
def __init__(self, environ: dict):
|
||||
self.environ = environ
|
||||
def __init__(self, config: 'BaseConfig', environ: dict):
|
||||
self.configuration = config
|
||||
self.data = {}
|
||||
if environ:
|
||||
self.load(environ)
|
||||
|
||||
def load(self, environ):
|
||||
for setting in self.configuration.get_settings():
|
||||
value = environ.get(f'{self.PREFIX}{setting.name.upper()}', NOT_SET)
|
||||
if value != NOT_SET and not (isinstance(setting, ListSetting) and value is None):
|
||||
self.data[setting.name] = setting.deserialize(value)
|
||||
|
||||
def __contains__(self, item: str):
|
||||
return f'{self.PREFIX}{item.upper()}' in self.environ
|
||||
return item in self.data
|
||||
|
||||
def __getitem__(self, item: str):
|
||||
return self.environ[f'{self.PREFIX}{item.upper()}']
|
||||
return self.data[item]
|
||||
|
||||
|
||||
class ArgumentAccess:
|
||||
@@ -325,7 +333,7 @@ class ConfigFileAccess:
|
||||
cls = type(self.configuration)
|
||||
with open(self.path, 'r') as config_file:
|
||||
raw = config_file.read()
|
||||
serialized = yaml.load(raw) or {}
|
||||
serialized = yaml.safe_load(raw) or {}
|
||||
for key, value in serialized.items():
|
||||
attr = getattr(cls, key, None)
|
||||
if attr is None:
|
||||
@@ -443,7 +451,7 @@ class BaseConfig:
|
||||
self.arguments = ArgumentAccess(self, args)
|
||||
|
||||
def set_environment(self, environ=None):
|
||||
self.environment = EnvironmentAccess(environ or os.environ)
|
||||
self.environment = EnvironmentAccess(self, environ or os.environ)
|
||||
|
||||
def set_persisted(self, config_file_path=None):
|
||||
if config_file_path is None:
|
||||
@@ -469,16 +477,16 @@ class TranscodeConfig(BaseConfig):
|
||||
'', previous_names=['ffmpeg_folder'])
|
||||
video_encoder = String('FFmpeg codec and parameters for the video encoding. '
|
||||
'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental',
|
||||
'libx264 -crf 21 -preset faster -pix_fmt yuv420p')
|
||||
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 8400000)
|
||||
'libx264 -crf 24 -preset faster -pix_fmt yuv420p')
|
||||
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 5_000_000)
|
||||
video_scaler = String('FFmpeg scaling parameters for reducing bitrate. '
|
||||
'Example: -vf "scale=-2:720,fps=24" -maxrate 5M -bufsize 3M',
|
||||
r'-vf "scale=if(gte(iw\,ih)\,min(1920\,iw)\,-2):if(lt(iw\,ih)\,min(1920\,ih)\,-2)" '
|
||||
r'-maxrate 8400K -bufsize 5000K')
|
||||
r'-maxrate 5500K -bufsize 5000K')
|
||||
audio_encoder = String('FFmpeg codec and parameters for the audio encoding. '
|
||||
'Example: libopus -b:a 128k',
|
||||
'aac -b:a 160k')
|
||||
volume_filter = String('FFmpeg filter for audio normalization.', '-af loudnorm')
|
||||
volume_filter = String('FFmpeg filter for audio normalization. Exmple: -af loudnorm', '')
|
||||
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', 240)
|
||||
|
||||
|
||||
@@ -577,9 +585,14 @@ class Config(CLIConfig):
|
||||
)
|
||||
|
||||
# servers
|
||||
reflector_servers = Servers("Reflector re-hosting servers", [
|
||||
reflector_servers = Servers("Reflector re-hosting servers for mirroring publishes", [
|
||||
('reflector.lbry.com', 5566)
|
||||
])
|
||||
|
||||
fixed_peers = Servers("Fixed peers to fall back to if none are found on P2P for a blob", [
|
||||
('cdn.reflector.lbry.com', 5567)
|
||||
])
|
||||
|
||||
lbryum_servers = Servers("SPV wallet servers", [
|
||||
('spv11.lbry.com', 50001),
|
||||
('spv12.lbry.com', 50001),
|
||||
@@ -598,7 +611,7 @@ class Config(CLIConfig):
|
||||
('lbrynet4.lbry.com', 4444) # ASIA
|
||||
])
|
||||
|
||||
comment_server = String("Comment server API URL", "https://comments.lbry.com/api")
|
||||
comment_server = String("Comment server API URL", "https://comments.lbry.com/api/v2")
|
||||
|
||||
# blockchain
|
||||
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
|
||||
@@ -611,6 +624,9 @@ class Config(CLIConfig):
|
||||
previous_names=['upload_log', 'upload_log', 'share_debug_info']
|
||||
)
|
||||
track_bandwidth = Toggle("Track bandwidth usage", True)
|
||||
allowed_origin = String(
|
||||
"Allowed `Origin` header value for API request (sent by browser), use * to allow "
|
||||
"all hosts; default is to only allow API requests with no `Origin` value.", "")
|
||||
|
||||
# media server
|
||||
streaming_server = String('Host name and port to serve streaming media over range requests',
|
||||
@@ -622,6 +638,7 @@ class Config(CLIConfig):
|
||||
"Strategy to use when selecting UTXOs for a transaction",
|
||||
STRATEGIES, "standard")
|
||||
|
||||
transaction_cache_size = Integer("Transaction cache size", 2 ** 17)
|
||||
save_resolved_claims = Toggle(
|
||||
"Save content claims to the database when they are resolved to keep file_list up to date, "
|
||||
"only disable this if file_x commands are not needed", True
|
||||
@@ -691,9 +708,10 @@ def get_darwin_directories() -> typing.Tuple[str, str, str]:
|
||||
def get_linux_directories() -> typing.Tuple[str, str, str]:
|
||||
try:
|
||||
with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg:
|
||||
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()).group(1)
|
||||
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir)
|
||||
download_dir = re.sub('\"', '', down_dir)
|
||||
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read())
|
||||
if down_dir:
|
||||
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir.group(1))
|
||||
download_dir = re.sub('\"', '', down_dir)
|
||||
except OSError:
|
||||
download_dir = os.getenv('XDG_DOWNLOAD_DIR')
|
||||
if not download_dir:
|
||||
|
||||
+3
-20
@@ -1,14 +1,14 @@
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
import ipaddress
|
||||
from binascii import hexlify
|
||||
from dataclasses import dataclass, field
|
||||
from functools import lru_cache
|
||||
|
||||
from lbry.utils import is_valid_public_ipv4 as _is_valid_public_ipv4
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
|
||||
|
||||
ALLOW_LOCALHOST = False
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -20,26 +20,9 @@ def make_kademlia_peer(node_id: typing.Optional[bytes], address: typing.Optional
|
||||
return KademliaPeer(address, node_id, udp_port, tcp_port=tcp_port, allow_localhost=allow_localhost)
|
||||
|
||||
|
||||
# the ipaddress module does not show these subnets as reserved
|
||||
CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
|
||||
IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
|
||||
|
||||
ALLOW_LOCALHOST = False
|
||||
|
||||
|
||||
def is_valid_public_ipv4(address, allow_localhost: bool = False):
|
||||
allow_localhost = bool(allow_localhost or ALLOW_LOCALHOST)
|
||||
try:
|
||||
parsed_ip = ipaddress.ip_address(address)
|
||||
if parsed_ip.is_loopback and allow_localhost:
|
||||
return True
|
||||
return not any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local,
|
||||
parsed_ip.is_loopback, parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private,
|
||||
parsed_ip.is_reserved,
|
||||
CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
|
||||
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
|
||||
except ipaddress.AddressValueError:
|
||||
return False
|
||||
return _is_valid_public_ipv4(address, allow_localhost)
|
||||
|
||||
|
||||
class PeerManager:
|
||||
|
||||
@@ -10,6 +10,7 @@ from asyncio.protocols import DatagramProtocol
|
||||
from asyncio.transports import DatagramTransport
|
||||
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.serialization.bencoding import DecodeError
|
||||
from lbry.dht.serialization.datagram import decode_datagram, ErrorDatagram, ResponseDatagram, RequestDatagram
|
||||
from lbry.dht.serialization.datagram import RESPONSE_TYPE, ERROR_TYPE, PAGE_KEY
|
||||
from lbry.dht.error import RemoteException, TransportNotConnected
|
||||
@@ -554,7 +555,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
def datagram_received(self, datagram: bytes, address: typing.Tuple[str, int]) -> None: # pylint: disable=arguments-differ
|
||||
try:
|
||||
message = decode_datagram(datagram)
|
||||
except (ValueError, TypeError):
|
||||
except (ValueError, TypeError, DecodeError):
|
||||
self.peer_manager.report_failure(address[0], address[1])
|
||||
log.warning("Couldn't decode dht datagram from %s: %s", address, binascii.hexlify(datagram).decode())
|
||||
return
|
||||
|
||||
@@ -144,7 +144,7 @@ class ErrorDatagram(KademliaDatagramBase):
|
||||
self.response = response.decode()
|
||||
|
||||
|
||||
def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]:
|
||||
def _decode_datagram(datagram: bytes):
|
||||
msg_types = {
|
||||
REQUEST_TYPE: RequestDatagram,
|
||||
RESPONSE_TYPE: ResponseDatagram,
|
||||
@@ -152,19 +152,29 @@ def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDa
|
||||
}
|
||||
|
||||
primitive: typing.Dict = bdecode(datagram)
|
||||
if primitive[0] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
|
||||
datagram_type = primitive[0] # pylint: disable=unsubscriptable-object
|
||||
|
||||
converted = {
|
||||
str(k).encode() if not isinstance(k, bytes) else k: v for k, v in primitive.items()
|
||||
}
|
||||
|
||||
if converted[b'0'] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
|
||||
datagram_type = converted[b'0'] # pylint: disable=unsubscriptable-object
|
||||
else:
|
||||
raise ValueError("invalid datagram type")
|
||||
datagram_class = msg_types[datagram_type]
|
||||
decoded = {
|
||||
k: primitive[i] # pylint: disable=unsubscriptable-object
|
||||
k: converted[str(i).encode()] # pylint: disable=unsubscriptable-object
|
||||
for i, k in enumerate(datagram_class.required_fields)
|
||||
if i in primitive # pylint: disable=unsupported-membership-test
|
||||
if str(i).encode() in converted # pylint: disable=unsupported-membership-test
|
||||
}
|
||||
for i, _ in enumerate(OPTIONAL_FIELDS):
|
||||
if i + OPTIONAL_ARG_OFFSET in primitive:
|
||||
decoded[i + OPTIONAL_ARG_OFFSET] = primitive[i + OPTIONAL_ARG_OFFSET]
|
||||
if str(i + OPTIONAL_ARG_OFFSET).encode() in converted:
|
||||
decoded[i + OPTIONAL_ARG_OFFSET] = converted[str(i + OPTIONAL_ARG_OFFSET).encode()]
|
||||
return decoded, datagram_class
|
||||
|
||||
|
||||
def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]:
|
||||
decoded, datagram_class = _decode_datagram(datagram)
|
||||
return datagram_class(**decoded)
|
||||
|
||||
|
||||
|
||||
@@ -14,7 +14,6 @@ from aiohttp.web import GracefulExit
|
||||
from docopt import docopt
|
||||
|
||||
from lbry import __version__ as lbrynet_version
|
||||
from lbry.extras.daemon.loggly_handler import get_loggly_handler
|
||||
from lbry.extras.daemon.daemon import Daemon
|
||||
from lbry.conf import Config, CLIConfig
|
||||
|
||||
@@ -255,10 +254,6 @@ def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config
|
||||
else:
|
||||
logger.getChild('lbry').setLevel(logging.DEBUG)
|
||||
|
||||
loggly_handler = get_loggly_handler(conf)
|
||||
loggly_handler.setLevel(logging.ERROR)
|
||||
logger.getChild('lbry').addHandler(loggly_handler)
|
||||
|
||||
|
||||
def run_daemon(args: argparse.Namespace, conf: Config):
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
@@ -66,7 +66,7 @@ def _download_properties(conf: Config, external_ip: str, resolve_duration: float
|
||||
"node_rpc_timeout": conf.node_rpc_timeout,
|
||||
"peer_connect_timeout": conf.peer_connect_timeout,
|
||||
"blob_download_timeout": conf.blob_download_timeout,
|
||||
"use_fixed_peers": len(conf.reflector_servers) > 0,
|
||||
"use_fixed_peers": len(conf.fixed_peers) > 0,
|
||||
"fixed_peer_delay": fixed_peer_delay,
|
||||
"added_fixed_peers": added_fixed_peers,
|
||||
"active_peer_count": active_peer_count,
|
||||
@@ -132,7 +132,7 @@ class AnalyticsManager:
|
||||
async def run(self):
|
||||
while True:
|
||||
if self.enabled:
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
await self._send_heartbeat()
|
||||
await asyncio.sleep(1800)
|
||||
|
||||
|
||||
@@ -22,41 +22,54 @@ def cid2hash(claim_id: str) -> bytes:
|
||||
return binascii.unhexlify(claim_id.encode())[::-1]
|
||||
|
||||
|
||||
def is_comment_signed_by_channel(comment: dict, channel: Output, abandon=False):
|
||||
def is_comment_signed_by_channel(comment: dict, channel: Output, sign_comment_id=False):
|
||||
if isinstance(channel, Output):
|
||||
try:
|
||||
signing_field = comment['comment_id'] if abandon else comment['comment']
|
||||
pieces = [
|
||||
comment['signing_ts'].encode(),
|
||||
cid2hash(comment['channel_id']),
|
||||
signing_field.encode()
|
||||
]
|
||||
return Output.is_signature_valid(
|
||||
get_encoded_signature(comment['signature']),
|
||||
sha256(b''.join(pieces)),
|
||||
channel.claim.channel.public_key_bytes
|
||||
)
|
||||
signing_field = comment['comment_id'] if sign_comment_id else comment['comment']
|
||||
return verify(channel, signing_field.encode(), comment, cid2hash(comment['channel_id']))
|
||||
except KeyError:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def sign_comment(comment: dict, channel: Output, abandon=False):
|
||||
def verify(channel, data, signature, channel_hash=None):
|
||||
pieces = [
|
||||
signature['signing_ts'].encode(),
|
||||
channel_hash or channel.claim_hash,
|
||||
data
|
||||
]
|
||||
return Output.is_signature_valid(
|
||||
get_encoded_signature(signature['signature']),
|
||||
sha256(b''.join(pieces)),
|
||||
channel.claim.channel.public_key_bytes
|
||||
)
|
||||
|
||||
|
||||
def sign_comment(comment: dict, channel: Output, sign_comment_id=False):
|
||||
signing_field = comment['comment_id'] if sign_comment_id else comment['comment']
|
||||
comment.update(sign(channel, signing_field.encode()))
|
||||
|
||||
|
||||
def sign(channel, data):
|
||||
timestamp = str(int(time.time()))
|
||||
signing_field = comment['comment_id'] if abandon else comment['comment']
|
||||
pieces = [timestamp.encode(), channel.claim_hash, signing_field.encode()]
|
||||
pieces = [timestamp.encode(), channel.claim_hash, data]
|
||||
digest = sha256(b''.join(pieces))
|
||||
signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
|
||||
comment.update({
|
||||
return {
|
||||
'signature': binascii.hexlify(signature).decode(),
|
||||
'signing_ts': timestamp
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def sign_reaction(reaction: dict, channel: Output):
|
||||
signing_field = reaction['channel_name']
|
||||
reaction.update(sign(channel, signing_field.encode()))
|
||||
|
||||
|
||||
async def jsonrpc_post(url: str, method: str, params: dict = None, **kwargs) -> any:
|
||||
params = params or {}
|
||||
params.update(kwargs)
|
||||
json_body = {'jsonrpc': '2.0', 'id': None, 'method': method, 'params': params}
|
||||
json_body = {'jsonrpc': '2.0', 'id': 1, 'method': method, 'params': params}
|
||||
async with utils.aiohttp_request('POST', url, json=json_body) as response:
|
||||
try:
|
||||
result = await response.json()
|
||||
|
||||
@@ -17,11 +17,17 @@ from lbry.dht.blob_announcer import BlobAnnouncer
|
||||
from lbry.blob.blob_manager import BlobManager
|
||||
from lbry.blob_exchange.server import BlobServer
|
||||
from lbry.stream.stream_manager import StreamManager
|
||||
from lbry.file.file_manager import FileManager
|
||||
from lbry.extras.daemon.component import Component
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from lbry.torrent.torrent_manager import TorrentManager
|
||||
from lbry.wallet import WalletManager
|
||||
from lbry.wallet.usage_payment import WalletServerPayer
|
||||
try:
|
||||
from lbry.torrent.session import TorrentSession
|
||||
except ImportError:
|
||||
TorrentSession = None
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -33,10 +39,11 @@ WALLET_COMPONENT = "wallet"
|
||||
WALLET_SERVER_PAYMENTS_COMPONENT = "wallet_server_payments"
|
||||
DHT_COMPONENT = "dht"
|
||||
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
|
||||
STREAM_MANAGER_COMPONENT = "stream_manager"
|
||||
FILE_MANAGER_COMPONENT = "file_manager"
|
||||
PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
|
||||
UPNP_COMPONENT = "upnp"
|
||||
EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager"
|
||||
LIBTORRENT_COMPONENT = "libtorrent_component"
|
||||
|
||||
|
||||
class DatabaseComponent(Component):
|
||||
@@ -112,13 +119,14 @@ class WalletComponent(Component):
|
||||
async def get_status(self):
|
||||
if self.wallet_manager is None:
|
||||
return
|
||||
session_pool = self.wallet_manager.ledger.network.session_pool
|
||||
sessions = session_pool.sessions
|
||||
is_connected = self.wallet_manager.ledger.network.is_connected
|
||||
sessions = []
|
||||
connected = None
|
||||
if self.wallet_manager.ledger.network.client:
|
||||
addr_and_port = self.wallet_manager.ledger.network.client.server_address_and_port
|
||||
if addr_and_port:
|
||||
connected = f"{addr_and_port[0]}:{addr_and_port[1]}"
|
||||
if is_connected:
|
||||
addr, port = self.wallet_manager.ledger.network.client.server
|
||||
connected = f"{addr}:{port}"
|
||||
sessions.append(self.wallet_manager.ledger.network.client)
|
||||
|
||||
result = {
|
||||
'connected': connected,
|
||||
'connected_features': self.wallet_manager.ledger.network.server_features,
|
||||
@@ -130,8 +138,8 @@ class WalletComponent(Component):
|
||||
'availability': session.available,
|
||||
} for session in sessions
|
||||
],
|
||||
'known_servers': len(sessions),
|
||||
'available_servers': len(list(session_pool.available_sessions))
|
||||
'known_servers': len(self.wallet_manager.ledger.network.config['default_servers']),
|
||||
'available_servers': 1 if is_connected else 0
|
||||
}
|
||||
|
||||
if self.wallet_manager.ledger.network.remote_height:
|
||||
@@ -267,7 +275,7 @@ class DHTComponent(Component):
|
||||
external_ip = upnp_component.external_ip
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
if not external_ip:
|
||||
external_ip = await utils.get_external_ip()
|
||||
external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
if not external_ip:
|
||||
log.warning("failed to get external ip")
|
||||
|
||||
@@ -319,23 +327,23 @@ class HashAnnouncerComponent(Component):
|
||||
}
|
||||
|
||||
|
||||
class StreamManagerComponent(Component):
|
||||
component_name = STREAM_MANAGER_COMPONENT
|
||||
class FileManagerComponent(Component):
|
||||
component_name = FILE_MANAGER_COMPONENT
|
||||
depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.stream_manager: typing.Optional[StreamManager] = None
|
||||
self.file_manager: typing.Optional[FileManager] = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[StreamManager]:
|
||||
return self.stream_manager
|
||||
def component(self) -> typing.Optional[FileManager]:
|
||||
return self.file_manager
|
||||
|
||||
async def get_status(self):
|
||||
if not self.stream_manager:
|
||||
if not self.file_manager:
|
||||
return
|
||||
return {
|
||||
'managed_files': len(self.stream_manager.streams),
|
||||
'managed_files': len(self.file_manager.get_filtered()),
|
||||
}
|
||||
|
||||
async def start(self):
|
||||
@@ -344,16 +352,55 @@ class StreamManagerComponent(Component):
|
||||
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
node = self.component_manager.get_component(DHT_COMPONENT) \
|
||||
if self.component_manager.has_component(DHT_COMPONENT) else None
|
||||
try:
|
||||
torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT) if TorrentSession else None
|
||||
except NameError:
|
||||
torrent = None
|
||||
log.info('Starting the file manager')
|
||||
loop = asyncio.get_event_loop()
|
||||
self.stream_manager = StreamManager(
|
||||
loop, self.conf, blob_manager, wallet, storage, node, self.component_manager.analytics_manager
|
||||
self.file_manager = FileManager(
|
||||
loop, self.conf, wallet, storage, self.component_manager.analytics_manager
|
||||
)
|
||||
await self.stream_manager.start()
|
||||
self.file_manager.source_managers['stream'] = StreamManager(
|
||||
loop, self.conf, blob_manager, wallet, storage, node,
|
||||
)
|
||||
if TorrentSession and LIBTORRENT_COMPONENT not in self.conf.components_to_skip:
|
||||
self.file_manager.source_managers['torrent'] = TorrentManager(
|
||||
loop, self.conf, torrent, storage, self.component_manager.analytics_manager
|
||||
)
|
||||
await self.file_manager.start()
|
||||
log.info('Done setting up file manager')
|
||||
|
||||
async def stop(self):
|
||||
self.stream_manager.stop()
|
||||
self.file_manager.stop()
|
||||
|
||||
|
||||
class TorrentComponent(Component):
|
||||
component_name = LIBTORRENT_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.torrent_session = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[TorrentSession]:
|
||||
return self.torrent_session
|
||||
|
||||
async def get_status(self):
|
||||
if not self.torrent_session:
|
||||
return
|
||||
return {
|
||||
'running': True, # TODO: what to return here?
|
||||
}
|
||||
|
||||
async def start(self):
|
||||
if TorrentSession:
|
||||
self.torrent_session = TorrentSession(asyncio.get_event_loop(), None)
|
||||
await self.torrent_session.bind() # TODO: specify host/port
|
||||
|
||||
async def stop(self):
|
||||
if self.torrent_session:
|
||||
await self.torrent_session.pause()
|
||||
|
||||
|
||||
class PeerProtocolServerComponent(Component):
|
||||
@@ -429,7 +476,7 @@ class UPnPComponent(Component):
|
||||
pass
|
||||
if external_ip and not is_valid_public_ipv4(external_ip):
|
||||
log.warning("UPnP returned a private/reserved ip - %s, checking lbry.com fallback", external_ip)
|
||||
external_ip = await utils.get_external_ip()
|
||||
external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
if self.external_ip and self.external_ip != external_ip:
|
||||
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
|
||||
if external_ip:
|
||||
@@ -487,7 +534,7 @@ class UPnPComponent(Component):
|
||||
async def start(self):
|
||||
log.info("detecting external ip")
|
||||
if not self.use_upnp:
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
return
|
||||
success = False
|
||||
await self._maintain_redirects()
|
||||
@@ -502,9 +549,9 @@ class UPnPComponent(Component):
|
||||
else:
|
||||
log.error("failed to setup upnp")
|
||||
if not self.external_ip:
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
self.external_ip, probed_url = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
if self.external_ip:
|
||||
log.info("detected external ip using lbry.com fallback")
|
||||
log.info("detected external ip using %s fallback", probed_url)
|
||||
if self.component_manager.analytics_manager:
|
||||
self.component_manager.loop.create_task(
|
||||
self.component_manager.analytics_manager.send_upnp_setup_success_fail(
|
||||
|
||||
+470
-133
File diff suppressed because it is too large.
Load diff
@@ -2,6 +2,7 @@ import json
|
||||
import time
|
||||
import asyncio
|
||||
import logging
|
||||
from statistics import median
|
||||
from decimal import Decimal
|
||||
from typing import Optional, Iterable, Type
|
||||
from aiohttp.client_exceptions import ContentTypeError
|
||||
@@ -58,9 +59,12 @@ class MarketFeed:
|
||||
raise NotImplementedError()
|
||||
|
||||
async def get_response(self):
|
||||
async with aiohttp_request('get', self.url, params=self.params, timeout=self.request_timeout) as response:
|
||||
async with aiohttp_request(
|
||||
'get', self.url, params=self.params,
|
||||
timeout=self.request_timeout, headers={"User-Agent": "lbrynet"}
|
||||
) as response:
|
||||
try:
|
||||
self._last_response = await response.json()
|
||||
self._last_response = await response.json(content_type=None)
|
||||
except ContentTypeError as e:
|
||||
self._last_response = {}
|
||||
log.warning("Could not parse exchange rate response from %s: %s", self.name, e.message)
|
||||
@@ -104,70 +108,115 @@ class MarketFeed:
|
||||
self.event.clear()
|
||||
|
||||
|
||||
class BittrexFeed(MarketFeed):
|
||||
class BaseBittrexFeed(MarketFeed):
|
||||
name = "Bittrex"
|
||||
market = "BTCLBC"
|
||||
url = "https://bittrex.com/api/v1.1/public/getmarkethistory"
|
||||
params = {'market': 'BTC-LBC', 'count': 50}
|
||||
market = None
|
||||
url = None
|
||||
fee = 0.0025
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'result' not in json_response:
|
||||
if 'lastTradeRate' not in json_response:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
trades = json_response['result']
|
||||
if len(trades) == 0:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'trades not found')
|
||||
totals = sum([i['Total'] for i in trades])
|
||||
qtys = sum([i['Quantity'] for i in trades])
|
||||
if totals <= 0 or qtys <= 0:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'quantities were not positive')
|
||||
vwap = totals / qtys
|
||||
return float(1.0 / vwap)
|
||||
return 1.0 / float(json_response['lastTradeRate'])
|
||||
|
||||
|
||||
class LBRYFeed(MarketFeed):
|
||||
name = "lbry.com"
|
||||
class BittrexBTCFeed(BaseBittrexFeed):
|
||||
market = "BTCLBC"
|
||||
url = "https://api.lbry.com/lbc/exchange_rate"
|
||||
url = "https://api.bittrex.com/v3/markets/LBC-BTC/ticker"
|
||||
|
||||
|
||||
class BittrexUSDFeed(BaseBittrexFeed):
|
||||
market = "USDLBC"
|
||||
url = "https://api.bittrex.com/v3/markets/LBC-USD/ticker"
|
||||
|
||||
|
||||
class BaseCryptonatorFeed(MarketFeed):
|
||||
name = "Cryptonator"
|
||||
market = None
|
||||
url = None
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'data' not in json_response:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return 1.0 / json_response['data']['lbc_btc']
|
||||
|
||||
|
||||
class LBRYBTCFeed(LBRYFeed):
|
||||
market = "USDBTC"
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'data' not in json_response:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return 1.0 / json_response['data']['btc_usd']
|
||||
|
||||
|
||||
class CryptonatorFeed(MarketFeed):
|
||||
name = "cryptonator.com"
|
||||
market = "BTCLBC"
|
||||
url = "https://api.cryptonator.com/api/ticker/btc-lbc"
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \
|
||||
'success' not in json_response or json_response['success'] is not True:
|
||||
if 'ticker' not in json_response or 'price' not in json_response['ticker']:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return float(json_response['ticker']['price'])
|
||||
|
||||
|
||||
class CryptonatorBTCFeed(CryptonatorFeed):
|
||||
market = "USDBTC"
|
||||
url = "https://api.cryptonator.com/api/ticker/usd-btc"
|
||||
class CryptonatorBTCFeed(BaseCryptonatorFeed):
|
||||
market = "BTCLBC"
|
||||
url = "https://api.cryptonator.com/api/ticker/btc-lbc"
|
||||
|
||||
|
||||
class CryptonatorUSDFeed(BaseCryptonatorFeed):
|
||||
market = "USDLBC"
|
||||
url = "https://api.cryptonator.com/api/ticker/usd-lbc"
|
||||
|
||||
|
||||
class BaseCoinExFeed(MarketFeed):
|
||||
name = "CoinEx"
|
||||
market = None
|
||||
url = None
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'data' not in json_response or \
|
||||
'ticker' not in json_response['data'] or \
|
||||
'last' not in json_response['data']['ticker']:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return 1.0 / float(json_response['data']['ticker']['last'])
|
||||
|
||||
|
||||
class CoinExBTCFeed(BaseCoinExFeed):
|
||||
market = "BTCLBC"
|
||||
url = "https://api.coinex.com/v1/market/ticker?market=LBCBTC"
|
||||
|
||||
|
||||
class CoinExUSDFeed(BaseCoinExFeed):
|
||||
market = "USDLBC"
|
||||
url = "https://api.coinex.com/v1/market/ticker?market=LBCUSDT"
|
||||
|
||||
|
||||
class BaseHotbitFeed(MarketFeed):
|
||||
name = "hotbit"
|
||||
market = None
|
||||
url = "https://api.hotbit.io/api/v1/market.last"
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'result' not in json_response:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return 1.0 / float(json_response['result'])
|
||||
|
||||
|
||||
class HotbitBTCFeed(BaseHotbitFeed):
|
||||
market = "BTCLBC"
|
||||
params = {"market": "LBC/BTC"}
|
||||
|
||||
|
||||
class HotbitUSDFeed(BaseHotbitFeed):
|
||||
market = "USDLBC"
|
||||
params = {"market": "LBC/USDT"}
|
||||
|
||||
|
||||
class UPbitBTCFeed(MarketFeed):
|
||||
name = "UPbit"
|
||||
market = "BTCLBC"
|
||||
url = "https://api.upbit.com/v1/ticker"
|
||||
params = {"markets": "BTC-LBC"}
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if len(json_response) != 1 or 'trade_price' not in json_response[0]:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return 1.0 / float(json_response[0]['trade_price'])
|
||||
|
||||
|
||||
FEEDS: Iterable[Type[MarketFeed]] = (
|
||||
LBRYFeed,
|
||||
LBRYBTCFeed,
|
||||
BittrexFeed,
|
||||
# CryptonatorFeed,
|
||||
# CryptonatorBTCFeed,
|
||||
BittrexBTCFeed,
|
||||
BittrexUSDFeed,
|
||||
CryptonatorBTCFeed,
|
||||
CryptonatorUSDFeed,
|
||||
CoinExBTCFeed,
|
||||
CoinExUSDFeed,
|
||||
HotbitBTCFeed,
|
||||
HotbitUSDFeed,
|
||||
UPbitBTCFeed,
|
||||
)
|
||||
|
||||
|
||||
@@ -191,20 +240,23 @@ class ExchangeRateManager:
|
||||
source.stop()
|
||||
|
||||
def convert_currency(self, from_currency, to_currency, amount):
|
||||
rates = [market.rate for market in self.market_feeds]
|
||||
log.debug("Converting %f %s to %s, rates: %s", amount, from_currency, to_currency, rates)
|
||||
log.debug(
|
||||
"Converting %f %s to %s, rates: %s",
|
||||
amount, from_currency, to_currency,
|
||||
[market.rate for market in self.market_feeds]
|
||||
)
|
||||
if from_currency == to_currency:
|
||||
return round(amount, 8)
|
||||
|
||||
rates = []
|
||||
for market in self.market_feeds:
|
||||
if (market.has_rate and market.is_online and
|
||||
market.rate.currency_pair == (from_currency, to_currency)):
|
||||
return round(amount * Decimal(market.rate.spot), 8)
|
||||
for market in self.market_feeds:
|
||||
if (market.has_rate and market.is_online and
|
||||
market.rate.currency_pair[0] == from_currency):
|
||||
return round(self.convert_currency(
|
||||
market.rate.currency_pair[1], to_currency, amount * Decimal(market.rate.spot)), 8)
|
||||
rates.append(market.rate.spot)
|
||||
|
||||
if rates:
|
||||
return round(amount * Decimal(median(rates)), 8)
|
||||
|
||||
raise CurrencyConversionError(
|
||||
f'Unable to convert {amount} from {from_currency} to {to_currency}')
|
||||
|
||||
|
||||
@@ -7,6 +7,8 @@ from json import JSONEncoder
|
||||
from google.protobuf.message import DecodeError
|
||||
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.schema.support import Support
|
||||
from lbry.torrent.torrent_manager import TorrentSource
|
||||
from lbry.wallet import Wallet, Ledger, Account, Transaction, Output
|
||||
from lbry.wallet.bip32 import PubKey
|
||||
from lbry.wallet.dewies import dewies_to_lbc
|
||||
@@ -126,7 +128,7 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
return self.encode_account(obj)
|
||||
if isinstance(obj, Wallet):
|
||||
return self.encode_wallet(obj)
|
||||
if isinstance(obj, ManagedStream):
|
||||
if isinstance(obj, (ManagedStream, TorrentSource)):
|
||||
return self.encode_file(obj)
|
||||
if isinstance(obj, Transaction):
|
||||
return self.encode_transaction(obj)
|
||||
@@ -134,6 +136,8 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
return self.encode_output(obj)
|
||||
if isinstance(obj, Claim):
|
||||
return self.encode_claim(obj)
|
||||
if isinstance(obj, Support):
|
||||
return obj.to_dict()
|
||||
if isinstance(obj, PubKey):
|
||||
return obj.extended_key_string()
|
||||
if isinstance(obj, datetime):
|
||||
@@ -219,25 +223,28 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
output['claims'] = [self.encode_output(o) for o in txo.claims]
|
||||
if txo.reposted_claim is not None:
|
||||
output['reposted_claim'] = self.encode_output(txo.reposted_claim)
|
||||
if txo.script.is_claim_name or txo.script.is_update_claim:
|
||||
try:
|
||||
output['value'] = txo.claim
|
||||
if txo.script.is_claim_name or txo.script.is_update_claim or txo.script.is_support_claim_data:
|
||||
try:
|
||||
output['value'] = txo.signable
|
||||
if self.include_protobuf:
|
||||
output['protobuf'] = hexlify(txo.signable.to_bytes())
|
||||
if txo.purchase_receipt is not None:
|
||||
output['purchase_receipt'] = self.encode_output(txo.purchase_receipt)
|
||||
if txo.script.is_claim_name or txo.script.is_update_claim:
|
||||
output['value_type'] = txo.claim.claim_type
|
||||
if self.include_protobuf:
|
||||
output['protobuf'] = hexlify(txo.claim.to_bytes())
|
||||
if txo.purchase_receipt is not None:
|
||||
output['purchase_receipt'] = self.encode_output(txo.purchase_receipt)
|
||||
if txo.claim.is_channel:
|
||||
output['has_signing_key'] = txo.has_private_key
|
||||
if check_signature and txo.claim.is_signed:
|
||||
if txo.channel is not None:
|
||||
output['signing_channel'] = self.encode_output(txo.channel)
|
||||
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
|
||||
else:
|
||||
output['signing_channel'] = {'channel_id': txo.claim.signing_channel_id}
|
||||
output['is_channel_signature_valid'] = False
|
||||
except DecodeError:
|
||||
pass
|
||||
elif txo.script.is_support_claim_data:
|
||||
output['value_type'] = 'emoji'
|
||||
if check_signature and txo.signable.is_signed:
|
||||
if txo.channel is not None:
|
||||
output['signing_channel'] = self.encode_output(txo.channel)
|
||||
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
|
||||
else:
|
||||
output['signing_channel'] = {'channel_id': txo.signable.signing_channel_id}
|
||||
output['is_channel_signature_valid'] = False
|
||||
except DecodeError:
|
||||
pass
|
||||
return output
|
||||
|
||||
def encode_claim_meta(self, meta):
|
||||
@@ -273,26 +280,32 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
output_exists = managed_stream.output_file_exists
|
||||
tx_height = managed_stream.stream_claim_info.height
|
||||
best_height = self.ledger.headers.height
|
||||
return {
|
||||
'streaming_url': managed_stream.stream_url,
|
||||
is_stream = hasattr(managed_stream, 'stream_hash')
|
||||
if is_stream:
|
||||
total_bytes_lower_bound = managed_stream.descriptor.lower_bound_decrypted_length()
|
||||
total_bytes = managed_stream.descriptor.upper_bound_decrypted_length()
|
||||
else:
|
||||
total_bytes_lower_bound = total_bytes = managed_stream.torrent_length
|
||||
result = {
|
||||
'streaming_url': None,
|
||||
'completed': managed_stream.completed,
|
||||
'file_name': managed_stream.file_name if output_exists else None,
|
||||
'download_directory': managed_stream.download_directory if output_exists else None,
|
||||
'download_path': managed_stream.full_path if output_exists else None,
|
||||
'file_name': None,
|
||||
'download_directory': None,
|
||||
'download_path': None,
|
||||
'points_paid': 0.0,
|
||||
'stopped': not managed_stream.running,
|
||||
'stream_hash': managed_stream.stream_hash,
|
||||
'stream_name': managed_stream.descriptor.stream_name,
|
||||
'suggested_file_name': managed_stream.descriptor.suggested_file_name,
|
||||
'sd_hash': managed_stream.descriptor.sd_hash,
|
||||
'mime_type': managed_stream.mime_type,
|
||||
'key': managed_stream.descriptor.key,
|
||||
'total_bytes_lower_bound': managed_stream.descriptor.lower_bound_decrypted_length(),
|
||||
'total_bytes': managed_stream.descriptor.upper_bound_decrypted_length(),
|
||||
'stream_hash': None,
|
||||
'stream_name': None,
|
||||
'suggested_file_name': None,
|
||||
'sd_hash': None,
|
||||
'mime_type': None,
|
||||
'key': None,
|
||||
'total_bytes_lower_bound': total_bytes_lower_bound,
|
||||
'total_bytes': total_bytes,
|
||||
'written_bytes': managed_stream.written_bytes,
|
||||
'blobs_completed': managed_stream.blobs_completed,
|
||||
'blobs_in_stream': managed_stream.blobs_in_stream,
|
||||
'blobs_remaining': managed_stream.blobs_remaining,
|
||||
'blobs_completed': None,
|
||||
'blobs_in_stream': None,
|
||||
'blobs_remaining': None,
|
||||
'status': managed_stream.status,
|
||||
'claim_id': managed_stream.claim_id,
|
||||
'txid': managed_stream.txid,
|
||||
@@ -309,10 +322,37 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
'height': tx_height,
|
||||
'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height,
|
||||
'timestamp': self.ledger.headers.estimated_timestamp(tx_height),
|
||||
'is_fully_reflected': managed_stream.is_fully_reflected,
|
||||
'reflector_progress': managed_stream.reflector_progress,
|
||||
'uploading_to_reflector': managed_stream.uploading_to_reflector
|
||||
'is_fully_reflected': False,
|
||||
'reflector_progress': False,
|
||||
'uploading_to_reflector': False
|
||||
}
|
||||
if is_stream:
|
||||
result.update({
|
||||
'streaming_url': managed_stream.stream_url,
|
||||
'stream_hash': managed_stream.stream_hash,
|
||||
'stream_name': managed_stream.descriptor.stream_name,
|
||||
'suggested_file_name': managed_stream.descriptor.suggested_file_name,
|
||||
'sd_hash': managed_stream.descriptor.sd_hash,
|
||||
'mime_type': managed_stream.mime_type,
|
||||
'key': managed_stream.descriptor.key,
|
||||
'blobs_completed': managed_stream.blobs_completed,
|
||||
'blobs_in_stream': managed_stream.blobs_in_stream,
|
||||
'blobs_remaining': managed_stream.blobs_remaining,
|
||||
'is_fully_reflected': managed_stream.is_fully_reflected,
|
||||
'reflector_progress': managed_stream.reflector_progress,
|
||||
'uploading_to_reflector': managed_stream.uploading_to_reflector
|
||||
})
|
||||
else:
|
||||
result.update({
|
||||
'streaming_url': f'file://{managed_stream.full_path}',
|
||||
})
|
||||
if output_exists:
|
||||
result.update({
|
||||
'file_name': managed_stream.file_name,
|
||||
'download_directory': managed_stream.download_directory,
|
||||
'download_path': managed_stream.full_path,
|
||||
})
|
||||
return result
|
||||
|
||||
def encode_claim(self, claim):
|
||||
encoded = getattr(claim, claim.claim_type).to_dict()
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging.handlers
|
||||
import traceback
|
||||
|
||||
import typing
|
||||
from aiohttp.client_exceptions import ClientError
|
||||
import aiohttp
|
||||
from lbry import utils, __version__
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
|
||||
LOGGLY_TOKEN = 'BQEzZmMzLJHgAGxkBF00LGD0YGuyATVgAmqxAQEuAQZ2BQH4'
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""Format log records using json serialization"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__()
|
||||
self.attributes = kwargs
|
||||
|
||||
def format(self, record):
|
||||
data = {
|
||||
'loggerName': record.name,
|
||||
'asciTime': self.formatTime(record),
|
||||
'fileName': record.filename,
|
||||
'functionName': record.funcName,
|
||||
'levelNo': record.levelno,
|
||||
'lineNo': record.lineno,
|
||||
'levelName': record.levelname,
|
||||
'message': record.getMessage(),
|
||||
}
|
||||
data.update(self.attributes)
|
||||
if record.exc_info:
|
||||
data['exc_info'] = self.formatException(record.exc_info)
|
||||
return json.dumps(data)
|
||||
|
||||
|
||||
class HTTPSLogglyHandler(logging.Handler):
|
||||
def __init__(self, loggly_token: str, config: 'Config'):
|
||||
super().__init__()
|
||||
self.cookies = {}
|
||||
self.url = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}".format(
|
||||
token=utils.deobfuscate(loggly_token), tag='lbrynet-' + __version__
|
||||
)
|
||||
self._loop = asyncio.get_event_loop()
|
||||
self._session = aiohttp.ClientSession()
|
||||
self._config = config
|
||||
|
||||
@property
|
||||
def enabled(self):
|
||||
return self._config.share_usage_data
|
||||
|
||||
@staticmethod
|
||||
def get_full_message(record):
|
||||
if record.exc_info:
|
||||
return '\n'.join(traceback.format_exception(*record.exc_info))
|
||||
else:
|
||||
return record.getMessage()
|
||||
|
||||
async def _emit(self, record, retry=True):
|
||||
data = self.format(record).encode()
|
||||
try:
|
||||
async with self._session.post(self.url, data=data,
|
||||
cookies=self.cookies) as response:
|
||||
self.cookies.update(response.cookies)
|
||||
except ClientError:
|
||||
if self._loop.is_running() and retry and self.enabled:
|
||||
await self._session.close()
|
||||
self._session = aiohttp.ClientSession()
|
||||
return await self._emit(record, retry=False)
|
||||
|
||||
def emit(self, record):
|
||||
if not self.enabled:
|
||||
return
|
||||
try:
|
||||
asyncio.ensure_future(self._emit(record), loop=self._loop)
|
||||
except RuntimeError: # TODO: use a second loop
|
||||
print(f"\nfailed to send traceback to loggly, please file an issue with the following traceback:\n"
|
||||
f"{self.format(record)}")
|
||||
|
||||
def close(self):
|
||||
super().close()
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(self._session.close())
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
|
||||
def get_loggly_handler(config):
|
||||
handler = HTTPSLogglyHandler(LOGGLY_TOKEN, config=config)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
return handler
|
||||
@@ -0,0 +1,31 @@
|
||||
import logging
|
||||
from aiohttp import web
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def ensure_request_allowed(request, conf):
|
||||
if is_request_allowed(request, conf):
|
||||
return
|
||||
if conf.allowed_origin:
|
||||
log.warning(
|
||||
"API requests with Origin '%s' are not allowed, "
|
||||
"configuration 'allowed_origin' limits requests to: '%s'",
|
||||
request.headers.get('Origin'), conf.allowed_origin
|
||||
)
|
||||
else:
|
||||
log.warning(
|
||||
"API requests with Origin '%s' are not allowed, "
|
||||
"update configuration 'allowed_origin' to enable this origin.",
|
||||
request.headers.get('Origin')
|
||||
)
|
||||
raise web.HTTPForbidden()
|
||||
|
||||
|
||||
def is_request_allowed(request, conf) -> bool:
|
||||
origin = request.headers.get('Origin')
|
||||
return (
|
||||
origin is None or
|
||||
origin == conf.allowed_origin or
|
||||
conf.allowed_origin == '*'
|
||||
)
|
||||
@@ -9,7 +9,7 @@ from typing import Optional
|
||||
from lbry.wallet import SQLiteMixin
|
||||
from lbry.conf import Config
|
||||
from lbry.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
||||
from lbry.wallet.transaction import Transaction
|
||||
from lbry.wallet.transaction import Transaction, Output
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.dht.constants import DATA_EXPIRATION
|
||||
from lbry.blob.blob_info import BlobInfo
|
||||
@@ -727,6 +727,19 @@ class SQLiteStorage(SQLiteMixin):
|
||||
if claim_id_to_supports:
|
||||
await self.save_supports(claim_id_to_supports)
|
||||
|
||||
def save_claim_from_output(self, ledger, *outputs: Output):
|
||||
return self.save_claims([{
|
||||
"claim_id": output.claim_id,
|
||||
"name": output.claim_name,
|
||||
"amount": dewies_to_lbc(output.amount),
|
||||
"address": output.get_address(ledger),
|
||||
"txid": output.tx_ref.id,
|
||||
"nout": output.position,
|
||||
"value": output.claim,
|
||||
"height": output.tx_ref.height,
|
||||
"claim_sequence": -1,
|
||||
} for output in outputs])
|
||||
|
||||
def save_claims_for_resolve(self, claim_infos):
|
||||
to_save = {}
|
||||
for info in claim_infos:
|
||||
@@ -740,7 +753,8 @@ class SQLiteStorage(SQLiteMixin):
|
||||
return self.save_claims(to_save.values())
|
||||
|
||||
@staticmethod
|
||||
def _save_content_claim(transaction, claim_outpoint, stream_hash):
|
||||
def _save_content_claim(transaction, claim_outpoint, stream_hash=None, bt_infohash=None):
|
||||
assert stream_hash or bt_infohash
|
||||
# get the claim id and serialized metadata
|
||||
claim_info = transaction.execute(
|
||||
"select claim_id, serialized_metadata from claim where claim_outpoint=?", (claim_outpoint,)
|
||||
@@ -788,6 +802,19 @@ class SQLiteStorage(SQLiteMixin):
|
||||
if stream_hash in self.content_claim_callbacks:
|
||||
await self.content_claim_callbacks[stream_hash]()
|
||||
|
||||
async def save_torrent_content_claim(self, bt_infohash, claim_outpoint, length, name):
|
||||
def _save_torrent(transaction):
|
||||
transaction.execute(
|
||||
"insert or replace into torrent values (?, NULL, ?, ?)", (bt_infohash, length, name)
|
||||
).fetchall()
|
||||
transaction.execute(
|
||||
"insert or replace into content_claim values (NULL, ?, ?)", (bt_infohash, claim_outpoint)
|
||||
).fetchall()
|
||||
await self.db.run(_save_torrent)
|
||||
# update corresponding ManagedEncryptedFileDownloader object
|
||||
if bt_infohash in self.content_claim_callbacks:
|
||||
await self.content_claim_callbacks[bt_infohash]()
|
||||
|
||||
async def get_content_claim(self, stream_hash: str, include_supports: typing.Optional[bool] = True) -> typing.Dict:
|
||||
claims = await self.db.run(get_claims_from_stream_hashes, [stream_hash])
|
||||
claim = None
|
||||
@@ -799,6 +826,10 @@ class SQLiteStorage(SQLiteMixin):
|
||||
claim['effective_amount'] = calculate_effective_amount(claim['amount'], supports)
|
||||
return claim
|
||||
|
||||
async def get_content_claim_for_torrent(self, bt_infohash):
|
||||
claims = await self.db.run(get_claims_from_torrent_info_hashes, [bt_infohash])
|
||||
return claims[bt_infohash].as_dict() if claims else None
|
||||
|
||||
# # # # # # # # # reflector functions # # # # # # # # #
|
||||
|
||||
def update_reflected_stream(self, sd_hash, reflector_address, success=True):
|
||||
|
||||
Whitespace-only changes.
@@ -0,0 +1,300 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import typing
|
||||
from typing import Optional
|
||||
from aiohttp.web import Request
|
||||
from lbry.error import ResolveError, DownloadSDTimeoutError, InsufficientFundsError
|
||||
from lbry.error import ResolveTimeoutError, DownloadDataTimeoutError, KeyFeeAboveMaxAllowedError
|
||||
from lbry.stream.managed_stream import ManagedStream
|
||||
from lbry.torrent.torrent_manager import TorrentSource
|
||||
from lbry.utils import cache_concurrent
|
||||
from lbry.schema.url import URL
|
||||
from lbry.wallet.dewies import dewies_to_lbc
|
||||
from lbry.file.source_manager import SourceManager
|
||||
from lbry.file.source import ManagedDownloadSource
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from lbry.wallet import WalletManager, Output
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FileManager:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', wallet_manager: 'WalletManager',
|
||||
storage: 'SQLiteStorage', analytics_manager: Optional['AnalyticsManager'] = None):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
self.wallet_manager = wallet_manager
|
||||
self.storage = storage
|
||||
self.analytics_manager = analytics_manager
|
||||
self.source_managers: typing.Dict[str, SourceManager] = {}
|
||||
self.started = asyncio.Event()
|
||||
|
||||
@property
|
||||
def streams(self):
|
||||
return self.source_managers['stream']._sources
|
||||
|
||||
async def create_stream(self, file_path: str, key: Optional[bytes] = None, **kwargs) -> ManagedDownloadSource:
|
||||
if 'stream' in self.source_managers:
|
||||
return await self.source_managers['stream'].create(file_path, key, **kwargs)
|
||||
raise NotImplementedError
|
||||
|
||||
async def start(self):
|
||||
await asyncio.gather(*(source_manager.start() for source_manager in self.source_managers.values()))
|
||||
for manager in self.source_managers.values():
|
||||
await manager.started.wait()
|
||||
self.started.set()
|
||||
|
||||
def stop(self):
|
||||
for manager in self.source_managers.values():
|
||||
# fixme: pop or not?
|
||||
manager.stop()
|
||||
self.started.clear()
|
||||
|
||||
@cache_concurrent
|
||||
async def download_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
|
||||
timeout: Optional[float] = None, file_name: Optional[str] = None,
|
||||
download_directory: Optional[str] = None,
|
||||
save_file: Optional[bool] = None, resolve_timeout: float = 3.0,
|
||||
wallet: Optional['Wallet'] = None) -> ManagedDownloadSource:
|
||||
|
||||
wallet = wallet or self.wallet_manager.default_wallet
|
||||
timeout = timeout or self.config.download_timeout
|
||||
start_time = self.loop.time()
|
||||
resolved_time = None
|
||||
stream = None
|
||||
claim = None
|
||||
error = None
|
||||
outpoint = None
|
||||
if save_file is None:
|
||||
save_file = self.config.save_files
|
||||
if file_name and not save_file:
|
||||
save_file = True
|
||||
if save_file:
|
||||
download_directory = download_directory or self.config.download_dir
|
||||
else:
|
||||
download_directory = None
|
||||
|
||||
payment = None
|
||||
try:
|
||||
# resolve the claim
|
||||
if not URL.parse(uri).has_stream:
|
||||
raise ResolveError("cannot download a channel claim, specify a /path")
|
||||
try:
|
||||
resolved_result = await asyncio.wait_for(
|
||||
self.wallet_manager.ledger.resolve(
|
||||
wallet.accounts, [uri],
|
||||
include_purchase_receipt=True,
|
||||
include_is_my_output=True
|
||||
), resolve_timeout
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
raise ResolveTimeoutError(uri)
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError):
|
||||
raise
|
||||
log.exception("Unexpected error resolving stream:")
|
||||
raise ResolveError(f"Unexpected error resolving stream: {str(err)}")
|
||||
if 'error' in resolved_result:
|
||||
raise ResolveError(f"Unexpected error resolving uri for download: {resolved_result['error']}")
|
||||
if not resolved_result or uri not in resolved_result:
|
||||
raise ResolveError(f"Failed to resolve stream at '{uri}'")
|
||||
txo = resolved_result[uri]
|
||||
if isinstance(txo, dict):
|
||||
raise ResolveError(f"Failed to resolve stream at '{uri}': {txo}")
|
||||
claim = txo.claim
|
||||
outpoint = f"{txo.tx_ref.id}:{txo.position}"
|
||||
resolved_time = self.loop.time() - start_time
|
||||
await self.storage.save_claim_from_output(self.wallet_manager.ledger, txo)
|
||||
|
||||
####################
|
||||
# update or replace
|
||||
####################
|
||||
|
||||
if claim.stream.source.bt_infohash:
|
||||
source_manager = self.source_managers['torrent']
|
||||
existing = source_manager.get_filtered(bt_infohash=claim.stream.source.bt_infohash)
|
||||
else:
|
||||
source_manager = self.source_managers['stream']
|
||||
existing = source_manager.get_filtered(sd_hash=claim.stream.source.sd_hash)
|
||||
|
||||
# resume or update an existing stream, if the stream changed: download it and delete the old one after
|
||||
to_replace, updated_stream = None, None
|
||||
if existing and existing[0].claim_id != txo.claim_id:
|
||||
raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {txo.claim_id}")
|
||||
if existing:
|
||||
log.info("claim contains a metadata only update to a stream we have")
|
||||
if claim.stream.source.bt_infohash:
|
||||
await self.storage.save_torrent_content_claim(
|
||||
existing[0].identifier, outpoint, existing[0].torrent_length, existing[0].torrent_name
|
||||
)
|
||||
claim_info = await self.storage.get_content_claim_for_torrent(existing[0].identifier)
|
||||
existing[0].set_claim(claim_info, claim)
|
||||
else:
|
||||
await self.storage.save_content_claim(
|
||||
existing[0].stream_hash, outpoint
|
||||
)
|
||||
await source_manager._update_content_claim(existing[0])
|
||||
updated_stream = existing[0]
|
||||
else:
|
||||
existing_for_claim_id = self.get_filtered(claim_id=txo.claim_id)
|
||||
if existing_for_claim_id:
|
||||
log.info("claim contains an update to a stream we have, downloading it")
|
||||
if save_file and existing_for_claim_id[0].output_file_exists:
|
||||
save_file = False
|
||||
if not claim.stream.source.bt_infohash:
|
||||
existing_for_claim_id[0].downloader.node = source_manager.node
|
||||
await existing_for_claim_id[0].start(timeout=timeout, save_now=save_file)
|
||||
if not existing_for_claim_id[0].output_file_exists and (
|
||||
save_file or file_name or download_directory):
|
||||
await existing_for_claim_id[0].save_file(
|
||||
file_name=file_name, download_directory=download_directory
|
||||
)
|
||||
to_replace = existing_for_claim_id[0]
|
||||
|
||||
# resume or update an existing stream, if the stream changed: download it and delete the old one after
|
||||
if updated_stream:
|
||||
log.info("already have stream for %s", uri)
|
||||
if save_file and updated_stream.output_file_exists:
|
||||
save_file = False
|
||||
if not claim.stream.source.bt_infohash:
|
||||
updated_stream.downloader.node = source_manager.node
|
||||
await updated_stream.start(timeout=timeout, save_now=save_file)
|
||||
if not updated_stream.output_file_exists and (save_file or file_name or download_directory):
|
||||
await updated_stream.save_file(
|
||||
file_name=file_name, download_directory=download_directory
|
||||
)
|
||||
return updated_stream
|
||||
|
||||
####################
|
||||
# pay fee
|
||||
####################
|
||||
|
||||
needs_purchasing = (
|
||||
not to_replace and
|
||||
not txo.is_my_output and
|
||||
txo.has_price and
|
||||
not txo.purchase_receipt
|
||||
)
|
||||
|
||||
if needs_purchasing:
|
||||
payment = await self.wallet_manager.create_purchase_transaction(
|
||||
wallet.accounts, txo, exchange_rate_manager
|
||||
)
|
||||
|
||||
####################
|
||||
# make downloader and wait for start
|
||||
####################
|
||||
|
||||
if not claim.stream.source.bt_infohash:
|
||||
# fixme: this shouldnt be here
|
||||
stream = ManagedStream(
|
||||
self.loop, self.config, source_manager.blob_manager, claim.stream.source.sd_hash,
|
||||
download_directory, file_name, ManagedStream.STATUS_RUNNING, content_fee=payment,
|
||||
analytics_manager=self.analytics_manager
|
||||
)
|
||||
stream.downloader.node = source_manager.node
|
||||
else:
|
||||
stream = TorrentSource(
|
||||
self.loop, self.config, self.storage, identifier=claim.stream.source.bt_infohash,
|
||||
file_name=file_name, download_directory=download_directory or self.config.download_dir,
|
||||
status=ManagedStream.STATUS_RUNNING,
|
||||
analytics_manager=self.analytics_manager,
|
||||
torrent_session=source_manager.torrent_session
|
||||
)
|
||||
log.info("starting download for %s", uri)
|
||||
|
||||
before_download = self.loop.time()
|
||||
await stream.start(timeout, save_file)
|
||||
|
||||
####################
|
||||
# success case: delete to_replace if applicable, broadcast fee payment
|
||||
####################
|
||||
|
||||
if to_replace: # delete old stream now that the replacement has started downloading
|
||||
await source_manager.delete(to_replace)
|
||||
|
||||
if payment is not None:
|
||||
await self.wallet_manager.broadcast_or_release(payment)
|
||||
payment = None # to avoid releasing in `finally` later
|
||||
log.info("paid fee of %s for %s", dewies_to_lbc(stream.content_fee.outputs[0].amount), uri)
|
||||
await self.storage.save_content_fee(stream.stream_hash, stream.content_fee)
|
||||
|
||||
source_manager.add(stream)
|
||||
|
||||
if not claim.stream.source.bt_infohash:
|
||||
await self.storage.save_content_claim(stream.stream_hash, outpoint)
|
||||
else:
|
||||
await self.storage.save_torrent_content_claim(
|
||||
stream.identifier, outpoint, stream.torrent_length, stream.torrent_name
|
||||
)
|
||||
claim_info = await self.storage.get_content_claim_for_torrent(stream.identifier)
|
||||
stream.set_claim(claim_info, claim)
|
||||
if save_file:
|
||||
await asyncio.wait_for(stream.save_file(), timeout - (self.loop.time() - before_download),
|
||||
loop=self.loop)
|
||||
return stream
|
||||
except asyncio.TimeoutError:
|
||||
error = DownloadDataTimeoutError(stream.sd_hash)
|
||||
raise error
|
||||
except Exception as err: # forgive data timeout, don't delete stream
|
||||
expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError,
|
||||
KeyFeeAboveMaxAllowedError)
|
||||
if isinstance(err, expected):
|
||||
log.warning("Failed to download %s: %s", uri, str(err))
|
||||
elif isinstance(err, asyncio.CancelledError):
|
||||
pass
|
||||
else:
|
||||
log.exception("Unexpected error downloading stream:")
|
||||
error = err
|
||||
raise
|
||||
finally:
|
||||
if payment is not None:
|
||||
# payment is set to None after broadcasting, if we're here an exception probably happened
|
||||
await self.wallet_manager.ledger.release_tx(payment)
|
||||
if self.analytics_manager and claim and claim.stream.source.bt_infohash:
|
||||
# TODO: analytics for torrents
|
||||
pass
|
||||
elif self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or
|
||||
stream.downloader.time_to_first_bytes))):
|
||||
server = self.wallet_manager.ledger.network.client.server
|
||||
self.loop.create_task(
|
||||
self.analytics_manager.send_time_to_first_bytes(
|
||||
resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id,
|
||||
uri, outpoint,
|
||||
None if not stream else len(stream.downloader.blob_downloader.active_connections),
|
||||
None if not stream else len(stream.downloader.blob_downloader.scores),
|
||||
None if not stream else len(stream.downloader.blob_downloader.connection_failures),
|
||||
False if not stream else stream.downloader.added_fixed_peers,
|
||||
self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay,
|
||||
None if not stream else stream.sd_hash,
|
||||
None if not stream else stream.downloader.time_to_descriptor,
|
||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
|
||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
|
||||
None if not stream else stream.downloader.time_to_first_bytes,
|
||||
None if not error else error.__class__.__name__,
|
||||
None if not error else str(error),
|
||||
None if not server else f"{server[0]}:{server[1]}"
|
||||
)
|
||||
)
|
||||
|
||||
async def stream_partial_content(self, request: Request, sd_hash: str):
|
||||
return await self.source_managers['stream'].stream_partial_content(request, sd_hash)
|
||||
|
||||
def get_filtered(self, *args, **kwargs) -> typing.List[ManagedDownloadSource]:
|
||||
"""
|
||||
Get a list of filtered and sorted ManagedStream objects
|
||||
|
||||
:param sort_by: field to sort by
|
||||
:param reverse: reverse sorting
|
||||
:param comparison: comparison operator used for filtering
|
||||
:param search_by: fields and values to filter by
|
||||
"""
|
||||
return sum((manager.get_filtered(*args, **kwargs) for manager in self.source_managers.values()), [])
|
||||
|
||||
async def delete(self, source: ManagedDownloadSource, delete_file=False):
|
||||
for manager in self.source_managers.values():
|
||||
await manager.delete(source, delete_file)
|
||||
@@ -0,0 +1,161 @@
|
||||
import os
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
import binascii
|
||||
from typing import Optional
|
||||
from lbry.utils import generate_id
|
||||
from lbry.extras.daemon.storage import StoredContentClaim
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.wallet.transaction import Transaction
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ManagedDownloadSource:
|
||||
STATUS_RUNNING = "running"
|
||||
STATUS_STOPPED = "stopped"
|
||||
STATUS_FINISHED = "finished"
|
||||
|
||||
SAVING_ID = 1
|
||||
STREAMING_ID = 2
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage', identifier: str,
|
||||
file_name: Optional[str] = None, download_directory: Optional[str] = None,
|
||||
status: Optional[str] = STATUS_STOPPED, claim: Optional[StoredContentClaim] = None,
|
||||
download_id: Optional[str] = None, rowid: Optional[int] = None,
|
||||
content_fee: Optional['Transaction'] = None,
|
||||
analytics_manager: Optional['AnalyticsManager'] = None,
|
||||
added_on: Optional[int] = None):
|
||||
self.loop = loop
|
||||
self.storage = storage
|
||||
self.config = config
|
||||
self.identifier = identifier
|
||||
self.download_directory = download_directory
|
||||
self._file_name = file_name
|
||||
self._status = status
|
||||
self.stream_claim_info = claim
|
||||
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
|
||||
self.rowid = rowid
|
||||
self.content_fee = content_fee
|
||||
self.purchase_receipt = None
|
||||
self._added_on = added_on
|
||||
self.analytics_manager = analytics_manager
|
||||
|
||||
self.saving = asyncio.Event(loop=self.loop)
|
||||
self.finished_writing = asyncio.Event(loop=self.loop)
|
||||
self.started_writing = asyncio.Event(loop=self.loop)
|
||||
self.finished_write_attempt = asyncio.Event(loop=self.loop)
|
||||
|
||||
# @classmethod
|
||||
# async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', file_path: str,
|
||||
# key: Optional[bytes] = None,
|
||||
# iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedDownloadSource':
|
||||
# raise NotImplementedError()
|
||||
|
||||
async def start(self, timeout: Optional[float] = None, save_now: Optional[bool] = False):
|
||||
raise NotImplementedError()
|
||||
|
||||
async def stop(self, finished: bool = False):
|
||||
raise NotImplementedError()
|
||||
|
||||
async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
|
||||
raise NotImplementedError()
|
||||
|
||||
def stop_tasks(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
def set_claim(self, claim_info: typing.Dict, claim: 'Claim'):
|
||||
self.stream_claim_info = StoredContentClaim(
|
||||
f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
|
||||
claim_info['name'], claim_info['amount'], claim_info['height'],
|
||||
binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'],
|
||||
claim_info['claim_sequence'], claim_info.get('channel_name')
|
||||
)
|
||||
|
||||
# async def update_content_claim(self, claim_info: Optional[typing.Dict] = None):
|
||||
# if not claim_info:
|
||||
# claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash)
|
||||
# self.set_claim(claim_info, claim_info['value'])
|
||||
|
||||
@property
|
||||
def file_name(self) -> Optional[str]:
|
||||
return self._file_name
|
||||
|
||||
@property
|
||||
def added_on(self) -> Optional[int]:
|
||||
return self._added_on
|
||||
|
||||
@property
|
||||
def status(self) -> str:
|
||||
return self._status
|
||||
|
||||
@property
|
||||
def completed(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
# @property
|
||||
# def stream_url(self):
|
||||
# return f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash}
|
||||
|
||||
@property
|
||||
def finished(self) -> bool:
|
||||
return self.status == self.STATUS_FINISHED
|
||||
|
||||
@property
|
||||
def running(self) -> bool:
|
||||
return self.status == self.STATUS_RUNNING
|
||||
|
||||
@property
|
||||
def claim_id(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim_id
|
||||
|
||||
@property
|
||||
def txid(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.txid
|
||||
|
||||
@property
|
||||
def nout(self) -> Optional[int]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.nout
|
||||
|
||||
@property
|
||||
def outpoint(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.outpoint
|
||||
|
||||
@property
|
||||
def claim_height(self) -> Optional[int]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.height
|
||||
|
||||
@property
|
||||
def channel_claim_id(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.channel_claim_id
|
||||
|
||||
@property
|
||||
def channel_name(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.channel_name
|
||||
|
||||
@property
|
||||
def claim_name(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim_name
|
||||
|
||||
@property
|
||||
def metadata(self) -> Optional[typing.Dict]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict()
|
||||
|
||||
@property
|
||||
def metadata_protobuf(self) -> bytes:
|
||||
if self.stream_claim_info:
|
||||
return binascii.hexlify(self.stream_claim_info.claim.to_bytes())
|
||||
|
||||
@property
|
||||
def full_path(self) -> Optional[str]:
|
||||
return os.path.join(self.download_directory, os.path.basename(self.file_name)) \
|
||||
if self.file_name and self.download_directory else None
|
||||
|
||||
@property
|
||||
def output_file_exists(self):
|
||||
return os.path.isfile(self.full_path) if self.full_path else False
|
||||
@@ -0,0 +1,138 @@
|
||||
import os
|
||||
import asyncio
|
||||
import logging
|
||||
import typing
|
||||
from typing import Optional
|
||||
from lbry.file.source import ManagedDownloadSource
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
COMPARISON_OPERATORS = {
|
||||
'eq': lambda a, b: a == b,
|
||||
'ne': lambda a, b: a != b,
|
||||
'g': lambda a, b: a > b,
|
||||
'l': lambda a, b: a < b,
|
||||
'ge': lambda a, b: a >= b,
|
||||
'le': lambda a, b: a <= b,
|
||||
}
|
||||
|
||||
|
||||
class SourceManager:
|
||||
filter_fields = {
|
||||
'rowid',
|
||||
'status',
|
||||
'file_name',
|
||||
'added_on',
|
||||
'download_path',
|
||||
'claim_name',
|
||||
'claim_height',
|
||||
'claim_id',
|
||||
'outpoint',
|
||||
'txid',
|
||||
'nout',
|
||||
'channel_claim_id',
|
||||
'channel_name',
|
||||
'completed'
|
||||
}
|
||||
|
||||
set_filter_fields = {
|
||||
"claim_ids": "claim_id",
|
||||
"channel_claim_ids": "channel_claim_id",
|
||||
"outpoints": "outpoint"
|
||||
}
|
||||
|
||||
source_class = ManagedDownloadSource
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage',
|
||||
analytics_manager: Optional['AnalyticsManager'] = None):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
self.storage = storage
|
||||
self.analytics_manager = analytics_manager
|
||||
self._sources: typing.Dict[str, ManagedDownloadSource] = {}
|
||||
self.started = asyncio.Event(loop=self.loop)
|
||||
|
||||
def add(self, source: ManagedDownloadSource):
|
||||
self._sources[source.identifier] = source
|
||||
|
||||
def remove(self, source: ManagedDownloadSource):
|
||||
if source.identifier not in self._sources:
|
||||
return
|
||||
self._sources.pop(source.identifier)
|
||||
source.stop_tasks()
|
||||
|
||||
async def initialize_from_database(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
async def start(self):
|
||||
await self.initialize_from_database()
|
||||
self.started.set()
|
||||
|
||||
def stop(self):
|
||||
while self._sources:
|
||||
_, source = self._sources.popitem()
|
||||
source.stop_tasks()
|
||||
self.started.clear()
|
||||
|
||||
async def create(self, file_path: str, key: Optional[bytes] = None,
|
||||
iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedDownloadSource:
|
||||
raise NotImplementedError()
|
||||
|
||||
async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
|
||||
self.remove(source)
|
||||
if delete_file and source.output_file_exists:
|
||||
os.remove(source.full_path)
|
||||
|
||||
def get_filtered(self, sort_by: Optional[str] = None, reverse: Optional[bool] = False,
|
||||
comparison: Optional[str] = None, **search_by) -> typing.List[ManagedDownloadSource]:
|
||||
"""
|
||||
Get a list of filtered and sorted ManagedStream objects
|
||||
|
||||
:param sort_by: field to sort by
|
||||
:param reverse: reverse sorting
|
||||
:param comparison: comparison operator used for filtering
|
||||
:param search_by: fields and values to filter by
|
||||
"""
|
||||
if sort_by and sort_by not in self.filter_fields:
|
||||
raise ValueError(f"'{sort_by}' is not a valid field to sort by")
|
||||
if comparison and comparison not in COMPARISON_OPERATORS:
|
||||
raise ValueError(f"'{comparison}' is not a valid comparison")
|
||||
if 'full_status' in search_by:
|
||||
del search_by['full_status']
|
||||
|
||||
for search in search_by:
|
||||
if search not in self.filter_fields:
|
||||
raise ValueError(f"'{search}' is not a valid search operation")
|
||||
|
||||
compare_sets = {}
|
||||
if isinstance(search_by.get('claim_id'), list):
|
||||
compare_sets['claim_ids'] = search_by.pop('claim_id')
|
||||
if isinstance(search_by.get('outpoint'), list):
|
||||
compare_sets['outpoints'] = search_by.pop('outpoint')
|
||||
if isinstance(search_by.get('channel_claim_id'), list):
|
||||
compare_sets['channel_claim_ids'] = search_by.pop('channel_claim_id')
|
||||
|
||||
if search_by or compare_sets:
|
||||
comparison = comparison or 'eq'
|
||||
streams = []
|
||||
for stream in self._sources.values():
|
||||
if compare_sets and not all(
|
||||
getattr(stream, self.set_filter_fields[set_search]) in val
|
||||
for set_search, val in compare_sets.items()):
|
||||
continue
|
||||
if search_by and not all(
|
||||
COMPARISON_OPERATORS[comparison](getattr(stream, search), val)
|
||||
for search, val in search_by.items()):
|
||||
continue
|
||||
streams.append(stream)
|
||||
else:
|
||||
streams = list(self._sources.values())
|
||||
if sort_by:
|
||||
streams.sort(key=lambda s: getattr(s, sort_by))
|
||||
if reverse:
|
||||
streams.reverse()
|
||||
return streams
|
||||
@@ -199,6 +199,8 @@ class VideoFileAnalyzer:
|
||||
if not {"aac", "mp3", "flac", "vorbis", "opus"}.intersection(codec.split(",")):
|
||||
return "Audio codec is not in the approved list of AAC, FLAC, MP3, Vorbis, and Opus. " \
|
||||
f"Actual: {codec} [{stream['codec_long_name']}]"
|
||||
if int(stream['sample_rate']) > 48000:
|
||||
return "Sample rate out of range"
|
||||
|
||||
return ""
|
||||
|
||||
@@ -304,9 +306,6 @@ class VideoFileAnalyzer:
|
||||
|
||||
raise Exception(f"The audio encoder is not available. Requested: {encoder or 'aac'}")
|
||||
|
||||
async def _get_volume_filter(self):
|
||||
return self._conf.volume_filter if self._conf.volume_filter else "-af loudnorm"
|
||||
|
||||
@staticmethod
|
||||
def _get_best_container_extension(scan_data, video_encoder):
|
||||
# the container is chosen by the video format
|
||||
@@ -410,7 +409,7 @@ class VideoFileAnalyzer:
|
||||
# the plan for transcoding:
|
||||
# we have to re-encode the video if it is in a nonstandard format
|
||||
# we also re-encode if we are h264 but not yuv420p (both errors caught in video_msg)
|
||||
# we also re-encode if our bitrate is too high
|
||||
# we also re-encode if our bitrate or sample rate is too high
|
||||
|
||||
try:
|
||||
transcode_command = [f'-i "{file_path}" -y -c:s copy -c:d copy -c:v']
|
||||
@@ -430,9 +429,10 @@ class VideoFileAnalyzer:
|
||||
if audio_msg or volume_msg:
|
||||
audio_encoder = await self._get_audio_encoder(extension)
|
||||
transcode_command.append(audio_encoder)
|
||||
if volume_msg:
|
||||
volume_filter = await self._get_volume_filter()
|
||||
transcode_command.append(volume_filter)
|
||||
if volume_msg and self._conf.volume_filter:
|
||||
transcode_command.append(self._conf.volume_filter)
|
||||
if audio_msg == "Sample rate out of range":
|
||||
transcode_command.append(" -ar 48000 ")
|
||||
else:
|
||||
transcode_command.append("copy")
|
||||
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
import time
|
||||
import logging
|
||||
import asyncio
|
||||
import asyncio.tasks
|
||||
from aiohttp import web
|
||||
from prometheus_client import generate_latest as prom_generate_latest
|
||||
from prometheus_client import Counter, Histogram, Gauge
|
||||
|
||||
|
||||
PROBES_IN_FLIGHT = Counter("probes_in_flight", "Number of loop probes in flight", namespace='asyncio')
|
||||
PROBES_FINISHED = Counter("probes_finished", "Number of finished loop probes", namespace='asyncio')
|
||||
PROBE_TIMES = Histogram("probe_times", "Loop probe times", namespace='asyncio')
|
||||
TASK_COUNT = Gauge("running_tasks", "Number of running tasks", namespace='asyncio')
|
||||
|
||||
|
||||
def get_loop_metrics(delay=1):
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def callback(started):
|
||||
PROBE_TIMES.observe(time.perf_counter() - started - delay)
|
||||
PROBES_FINISHED.inc()
|
||||
|
||||
async def monitor_loop_responsiveness():
|
||||
while True:
|
||||
now = time.perf_counter()
|
||||
loop.call_later(delay, callback, now)
|
||||
PROBES_IN_FLIGHT.inc()
|
||||
TASK_COUNT.set(len(asyncio.tasks._all_tasks))
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
return loop.create_task(monitor_loop_responsiveness())
|
||||
|
||||
|
||||
class PrometheusServer:
|
||||
def __init__(self, logger=None):
|
||||
self.runner = None
|
||||
self.logger = logger or logging.getLogger(__name__)
|
||||
self._monitor_loop_task = None
|
||||
|
||||
async def start(self, interface: str, port: int):
|
||||
self.logger.info("start prometheus metrics")
|
||||
prom_app = web.Application()
|
||||
prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
|
||||
self.runner = web.AppRunner(prom_app)
|
||||
await self.runner.setup()
|
||||
|
||||
metrics_site = web.TCPSite(self.runner, interface, port, shutdown_timeout=.5)
|
||||
await metrics_site.start()
|
||||
self.logger.info(
|
||||
'prometheus metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2]
|
||||
)
|
||||
self._monitor_loop_task = get_loop_metrics()
|
||||
|
||||
async def handle_metrics_get_request(self, request: web.Request):
|
||||
try:
|
||||
return web.Response(
|
||||
text=prom_generate_latest().decode(),
|
||||
content_type='text/plain; version=0.0.4'
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception('could not generate prometheus data')
|
||||
raise
|
||||
|
||||
async def stop(self):
|
||||
if self._monitor_loop_task and not self._monitor_loop_task.done():
|
||||
self._monitor_loop_task.cancel()
|
||||
self._monitor_loop_task = None
|
||||
await self.runner.cleanup()
|
||||
@@ -30,14 +30,10 @@ class Claim(Signable):
|
||||
COLLECTION = 'collection'
|
||||
REPOST = 'repost'
|
||||
|
||||
__slots__ = 'version',
|
||||
__slots__ = ()
|
||||
|
||||
message_class = ClaimMessage
|
||||
|
||||
def __init__(self, message=None):
|
||||
super().__init__(message)
|
||||
self.version = 2
|
||||
|
||||
@property
|
||||
def claim_type(self) -> str:
|
||||
return self.message.WhichOneof('type')
|
||||
|
||||
@@ -46,8 +46,8 @@ types_map = {
|
||||
'.ksh': ('text/plain', 'document'),
|
||||
'.latex': ('application/x-latex', 'binary'),
|
||||
'.m1v': ('video/mpeg', 'video'),
|
||||
'.m3u': ('application/vnd.apple.mpegurl', 'audio'),
|
||||
'.m3u8': ('application/vnd.apple.mpegurl', 'audio'),
|
||||
'.m3u': ('application/x-mpegurl', 'audio'),
|
||||
'.m3u8': ('application/x-mpegurl', 'video'),
|
||||
'.man': ('application/x-troff-man', 'document'),
|
||||
'.markdown': ('text/markdown', 'document'),
|
||||
'.md': ('text/markdown', 'document'),
|
||||
@@ -61,10 +61,12 @@ types_map = {
|
||||
'.mp3': ('audio/mpeg', 'audio'),
|
||||
'.mp4': ('video/mp4', 'video'),
|
||||
'.mpa': ('video/mpeg', 'video'),
|
||||
'.mpd': ('application/dash+xml', 'video'),
|
||||
'.mpe': ('video/mpeg', 'video'),
|
||||
'.mpeg': ('video/mpeg', 'video'),
|
||||
'.mpg': ('video/mpeg', 'video'),
|
||||
'.ms': ('application/x-troff-ms', 'binary'),
|
||||
'.m4s': ('video/iso.segment', 'binary'),
|
||||
'.nc': ('application/x-netcdf', 'binary'),
|
||||
'.nws': ('message/rfc822', 'document'),
|
||||
'.o': ('application/octet-stream', 'binary'),
|
||||
@@ -118,10 +120,12 @@ types_map = {
|
||||
'.tif': ('image/tiff', 'image'),
|
||||
'.tiff': ('image/tiff', 'image'),
|
||||
'.tr': ('application/x-troff', 'binary'),
|
||||
'.ts': ('video/mp2t', 'video'),
|
||||
'.tsv': ('text/tab-separated-values', 'document'),
|
||||
'.txt': ('text/plain', 'document'),
|
||||
'.ustar': ('application/x-ustar', 'binary'),
|
||||
'.vcf': ('text/x-vcard', 'document'),
|
||||
'.vtt': ('text/vtt', 'document'),
|
||||
'.wav': ('audio/x-wav', 'audio'),
|
||||
'.webm': ('video/webm', 'video'),
|
||||
'.wiz': ('application/msword', 'document'),
|
||||
|
||||
@@ -25,12 +25,14 @@ def set_reference(reference, claim_hash, rows):
|
||||
|
||||
class Censor:
|
||||
|
||||
__slots__ = 'streams', 'channels', 'censored', 'total'
|
||||
__slots__ = 'streams', 'channels', 'limit_claims_per_channel', 'censored', 'claims_in_channel', 'total'
|
||||
|
||||
def __init__(self, streams: dict = None, channels: dict = None):
|
||||
def __init__(self, streams: dict = None, channels: dict = None, limit_claims_per_channel: int = None):
|
||||
self.streams = streams or {}
|
||||
self.channels = channels or {}
|
||||
self.limit_claims_per_channel = limit_claims_per_channel # doesn't count as censored
|
||||
self.censored = {}
|
||||
self.claims_in_channel = {}
|
||||
self.total = 0
|
||||
|
||||
def censor(self, row) -> bool:
|
||||
@@ -49,6 +51,11 @@ class Censor:
|
||||
break
|
||||
if was_censored:
|
||||
self.total += 1
|
||||
if not was_censored and self.limit_claims_per_channel is not None and row['channel_hash']:
|
||||
self.claims_in_channel.setdefault(row['channel_hash'], 0)
|
||||
self.claims_in_channel[row['channel_hash']] += 1
|
||||
if self.claims_in_channel[row['channel_hash']] > self.limit_claims_per_channel:
|
||||
return True
|
||||
return was_censored
|
||||
|
||||
def to_message(self, outputs: OutputsMessage, extra_txo_rows):
|
||||
|
||||
+10
-1
@@ -1,6 +1,15 @@
|
||||
from lbry.schema.base import Signable
|
||||
from lbry.schema.types.v2.support_pb2 import Support as SupportMessage
|
||||
|
||||
|
||||
class Support(Signable):
|
||||
__slots__ = ()
|
||||
message_class = None # TODO: add support protobufs
|
||||
message_class = SupportMessage
|
||||
|
||||
@property
|
||||
def emoji(self) -> str:
|
||||
return self.message.emoji
|
||||
|
||||
@emoji.setter
|
||||
def emoji(self, emoji: str):
|
||||
self.message.emoji = emoji
|
||||
@@ -0,0 +1,69 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: support.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import message as _message
|
||||
from google.protobuf import reflection as _reflection
|
||||
from google.protobuf import symbol_database as _symbol_database
|
||||
from google.protobuf import descriptor_pb2
|
||||
# @@protoc_insertion_point(imports)
|
||||
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='support.proto',
|
||||
package='pb',
|
||||
syntax='proto3',
|
||||
serialized_pb=_b('\n\rsupport.proto\x12\x02pb\"\x18\n\x07Support\x12\r\n\x05\x65moji\x18\x01 \x01(\tb\x06proto3')
|
||||
)
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
|
||||
|
||||
|
||||
_SUPPORT = _descriptor.Descriptor(
|
||||
name='Support',
|
||||
full_name='pb.Support',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='emoji', full_name='pb.Support.emoji', index=0,
|
||||
number=1, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=21,
|
||||
serialized_end=45,
|
||||
)
|
||||
|
||||
DESCRIPTOR.message_types_by_name['Support'] = _SUPPORT
|
||||
|
||||
Support = _reflection.GeneratedProtocolMessageType('Support', (_message.Message,), dict(
|
||||
DESCRIPTOR = _SUPPORT,
|
||||
__module__ = 'support_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.Support)
|
||||
))
|
||||
_sym_db.RegisterMessage(Support)
|
||||
|
||||
|
||||
# @@protoc_insertion_point(module_scope)
|
||||
+2
-9
@@ -22,8 +22,7 @@ def _create_url_regex():
|
||||
return _group(
|
||||
_named(name+"_name", prefix + invalid_names_regex) +
|
||||
_oneof(
|
||||
_group('#' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
|
||||
_group(':' + _named(name+"_sequence", '[1-9][0-9]*')),
|
||||
_group('[:#]' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
|
||||
_group(r'\$' + _named(name+"_amount_order", '[1-9][0-9]*'))
|
||||
) + '?'
|
||||
)
|
||||
@@ -50,7 +49,6 @@ def normalize_name(name):
|
||||
class PathSegment(NamedTuple):
|
||||
name: str
|
||||
claim_id: str = None
|
||||
sequence: int = None
|
||||
amount_order: int = None
|
||||
|
||||
@property
|
||||
@@ -61,17 +59,13 @@ class PathSegment(NamedTuple):
|
||||
q = {'name': self.name}
|
||||
if self.claim_id is not None:
|
||||
q['claim_id'] = self.claim_id
|
||||
if self.sequence is not None:
|
||||
q['sequence'] = self.sequence
|
||||
if self.amount_order is not None:
|
||||
q['amount_order'] = self.amount_order
|
||||
return q
|
||||
|
||||
def __str__(self):
|
||||
if self.claim_id is not None:
|
||||
return f"{self.name}#{self.claim_id}"
|
||||
elif self.sequence is not None:
|
||||
return f"{self.name}:{self.sequence}"
|
||||
return f"{self.name}:{self.claim_id}"
|
||||
elif self.amount_order is not None:
|
||||
return f"{self.name}${self.amount_order}"
|
||||
return self.name
|
||||
@@ -118,7 +112,6 @@ class URL(NamedTuple):
|
||||
segments[segment] = PathSegment(
|
||||
parts[f'{segment}_name'],
|
||||
parts[f'{segment}_claim_id'],
|
||||
parts[f'{segment}_sequence'],
|
||||
parts[f'{segment}_amount_order']
|
||||
)
|
||||
|
||||
|
||||
+17
-10
@@ -44,18 +44,25 @@ def random_iv_generator() -> typing.Generator[bytes, None, None]:
|
||||
yield os.urandom(AES.block_size // 8)
|
||||
|
||||
|
||||
def file_reader(file_path: str):
|
||||
def read_bytes(file_path: str, offset: int, to_read: int):
|
||||
with open(file_path, 'rb') as f:
|
||||
f.seek(offset)
|
||||
return f.read(to_read)
|
||||
|
||||
|
||||
async def file_reader(file_path: str):
|
||||
length = int(os.stat(file_path).st_size)
|
||||
offset = 0
|
||||
|
||||
with open(file_path, 'rb') as stream_file:
|
||||
while offset < length:
|
||||
bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1)
|
||||
if not bytes_to_read:
|
||||
break
|
||||
blob_bytes = stream_file.read(bytes_to_read)
|
||||
yield blob_bytes
|
||||
offset += bytes_to_read
|
||||
while offset < length:
|
||||
bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1)
|
||||
if not bytes_to_read:
|
||||
break
|
||||
blob_bytes = await asyncio.get_event_loop().run_in_executor(
|
||||
None, read_bytes, file_path, offset, bytes_to_read
|
||||
)
|
||||
yield blob_bytes
|
||||
offset += bytes_to_read
|
||||
|
||||
|
||||
def sanitize_file_name(dirty_name: str, default_file_name: str = 'lbry_download'):
|
||||
@@ -245,7 +252,7 @@ class StreamDescriptor:
|
||||
iv_generator = iv_generator or random_iv_generator()
|
||||
key = key or os.urandom(AES.block_size // 8)
|
||||
blob_num = -1
|
||||
for blob_bytes in file_reader(file_path):
|
||||
async for blob_bytes in file_reader(file_path):
|
||||
blob_num += 1
|
||||
blob_info = await BlobFile.create_from_unencrypted(
|
||||
loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num, blob_completed_callback
|
||||
|
||||
@@ -40,7 +40,7 @@ class StreamDownloader:
|
||||
async def cached_read_blob(blob_info: 'BlobInfo') -> bytes:
|
||||
return await self.read_blob(blob_info, 2)
|
||||
|
||||
if self.blob_manager.decrypted_blob_lru_cache:
|
||||
if self.blob_manager.decrypted_blob_lru_cache is not None:
|
||||
cached_read_blob = lru_cache_concurrent(override_lru_cache=self.blob_manager.decrypted_blob_lru_cache)(
|
||||
cached_read_blob
|
||||
)
|
||||
@@ -51,15 +51,15 @@ class StreamDownloader:
|
||||
def _delayed_add_fixed_peers():
|
||||
self.added_fixed_peers = True
|
||||
self.peer_queue.put_nowait([
|
||||
make_kademlia_peer(None, address, None, tcp_port=port + 1, allow_localhost=True)
|
||||
make_kademlia_peer(None, address, None, tcp_port=port, allow_localhost=True)
|
||||
for address, port in addresses
|
||||
])
|
||||
|
||||
if not self.config.reflector_servers:
|
||||
if not self.config.fixed_peers:
|
||||
return
|
||||
addresses = [
|
||||
(await resolve_host(url, port + 1, proto='tcp'), port)
|
||||
for url, port in self.config.reflector_servers
|
||||
(await resolve_host(url, port, proto='tcp'), port)
|
||||
for url, port in self.config.fixed_peers
|
||||
]
|
||||
if 'dht' in self.config.components_to_skip or not self.node or not \
|
||||
len(self.node.protocol.routing_table.get_peers()) > 0:
|
||||
@@ -92,8 +92,8 @@ class StreamDownloader:
|
||||
|
||||
async def start(self, node: typing.Optional['Node'] = None, connection_id: int = 0):
|
||||
# set up peer accumulation
|
||||
if node:
|
||||
self.node = node
|
||||
self.node = node or self.node # fixme: this shouldnt be set here!
|
||||
if self.node:
|
||||
if self.accumulate_task and not self.accumulate_task.done():
|
||||
self.accumulate_task.cancel()
|
||||
_, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue)
|
||||
|
||||
+77
-175
@@ -3,9 +3,8 @@ import asyncio
|
||||
import time
|
||||
import typing
|
||||
import logging
|
||||
import binascii
|
||||
from typing import Optional
|
||||
from aiohttp.web import Request, StreamResponse, HTTPRequestRangeNotSatisfiable
|
||||
from lbry.utils import generate_id
|
||||
from lbry.error import DownloadSDTimeoutError
|
||||
from lbry.schema.mime_types import guess_media_type
|
||||
from lbry.stream.downloader import StreamDownloader
|
||||
@@ -13,6 +12,7 @@ from lbry.stream.descriptor import StreamDescriptor, sanitize_file_name
|
||||
from lbry.stream.reflector.client import StreamReflectorClient
|
||||
from lbry.extras.daemon.storage import StoredContentClaim
|
||||
from lbry.blob import MAX_BLOB_SIZE
|
||||
from lbry.file.source import ManagedDownloadSource
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
@@ -40,80 +40,35 @@ async def get_next_available_file_name(loop: asyncio.AbstractEventLoop, download
|
||||
return await loop.run_in_executor(None, _get_next_available_file_name, download_directory, file_name)
|
||||
|
||||
|
||||
class ManagedStream:
|
||||
STATUS_RUNNING = "running"
|
||||
STATUS_STOPPED = "stopped"
|
||||
STATUS_FINISHED = "finished"
|
||||
|
||||
SAVING_ID = 1
|
||||
STREAMING_ID = 2
|
||||
|
||||
__slots__ = [
|
||||
'loop',
|
||||
'config',
|
||||
'blob_manager',
|
||||
'sd_hash',
|
||||
'download_directory',
|
||||
'_file_name',
|
||||
'_added_on',
|
||||
'_status',
|
||||
'stream_claim_info',
|
||||
'download_id',
|
||||
'rowid',
|
||||
'content_fee',
|
||||
'purchase_receipt',
|
||||
'downloader',
|
||||
'analytics_manager',
|
||||
'fully_reflected',
|
||||
'reflector_progress',
|
||||
'file_output_task',
|
||||
'delayed_stop_task',
|
||||
'streaming_responses',
|
||||
'streaming',
|
||||
'_running',
|
||||
'saving',
|
||||
'finished_writing',
|
||||
'started_writing',
|
||||
'finished_write_attempt',
|
||||
'uploading_to_reflector'
|
||||
]
|
||||
|
||||
class ManagedStream(ManagedDownloadSource):
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
sd_hash: str, download_directory: typing.Optional[str] = None, file_name: typing.Optional[str] = None,
|
||||
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional[StoredContentClaim] = None,
|
||||
download_id: typing.Optional[str] = None, rowid: typing.Optional[int] = None,
|
||||
descriptor: typing.Optional[StreamDescriptor] = None,
|
||||
content_fee: typing.Optional['Transaction'] = None,
|
||||
analytics_manager: typing.Optional['AnalyticsManager'] = None,
|
||||
added_on: typing.Optional[int] = None):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
sd_hash: str, download_directory: Optional[str] = None, file_name: Optional[str] = None,
|
||||
status: Optional[str] = ManagedDownloadSource.STATUS_STOPPED,
|
||||
claim: Optional[StoredContentClaim] = None,
|
||||
download_id: Optional[str] = None, rowid: Optional[int] = None,
|
||||
descriptor: Optional[StreamDescriptor] = None,
|
||||
content_fee: Optional['Transaction'] = None,
|
||||
analytics_manager: Optional['AnalyticsManager'] = None,
|
||||
added_on: Optional[int] = None):
|
||||
super().__init__(loop, config, blob_manager.storage, sd_hash, file_name, download_directory, status, claim,
|
||||
download_id, rowid, content_fee, analytics_manager, added_on)
|
||||
self.blob_manager = blob_manager
|
||||
self.sd_hash = sd_hash
|
||||
self.download_directory = download_directory
|
||||
self._file_name = file_name
|
||||
self._status = status
|
||||
self.stream_claim_info = claim
|
||||
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
|
||||
self.rowid = rowid
|
||||
self.content_fee = content_fee
|
||||
self.purchase_receipt = None
|
||||
self._added_on = added_on
|
||||
self.downloader = StreamDownloader(self.loop, self.config, self.blob_manager, sd_hash, descriptor)
|
||||
self.analytics_manager = analytics_manager
|
||||
|
||||
self.fully_reflected = asyncio.Event(loop=self.loop)
|
||||
self.reflector_progress = 0
|
||||
self.uploading_to_reflector = False
|
||||
self.file_output_task: typing.Optional[asyncio.Task] = None
|
||||
self.delayed_stop_task: typing.Optional[asyncio.Task] = None
|
||||
self.streaming_responses: typing.List[typing.Tuple[Request, StreamResponse]] = []
|
||||
self.fully_reflected = asyncio.Event(loop=self.loop)
|
||||
self.streaming = asyncio.Event(loop=self.loop)
|
||||
self._running = asyncio.Event(loop=self.loop)
|
||||
self.saving = asyncio.Event(loop=self.loop)
|
||||
self.finished_writing = asyncio.Event(loop=self.loop)
|
||||
self.started_writing = asyncio.Event(loop=self.loop)
|
||||
self.finished_write_attempt = asyncio.Event(loop=self.loop)
|
||||
|
||||
@property
|
||||
def sd_hash(self) -> str:
|
||||
return self.identifier
|
||||
|
||||
@property
|
||||
def is_fully_reflected(self) -> bool:
|
||||
@@ -128,17 +83,9 @@ class ManagedStream:
|
||||
return self.descriptor.stream_hash
|
||||
|
||||
@property
|
||||
def file_name(self) -> typing.Optional[str]:
|
||||
def file_name(self) -> Optional[str]:
|
||||
return self._file_name or (self.descriptor.suggested_file_name if self.descriptor else None)
|
||||
|
||||
@property
|
||||
def added_on(self) -> typing.Optional[int]:
|
||||
return self._added_on
|
||||
|
||||
@property
|
||||
def status(self) -> str:
|
||||
return self._status
|
||||
|
||||
@property
|
||||
def written_bytes(self) -> int:
|
||||
return 0 if not self.output_file_exists else os.stat(self.full_path).st_size
|
||||
@@ -156,55 +103,6 @@ class ManagedStream:
|
||||
self._status = status
|
||||
await self.blob_manager.storage.change_file_status(self.stream_hash, status)
|
||||
|
||||
@property
|
||||
def finished(self) -> bool:
|
||||
return self.status == self.STATUS_FINISHED
|
||||
|
||||
@property
|
||||
def running(self) -> bool:
|
||||
return self.status == self.STATUS_RUNNING
|
||||
|
||||
@property
|
||||
def claim_id(self) -> typing.Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim_id
|
||||
|
||||
@property
|
||||
def txid(self) -> typing.Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.txid
|
||||
|
||||
@property
|
||||
def nout(self) -> typing.Optional[int]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.nout
|
||||
|
||||
@property
|
||||
def outpoint(self) -> typing.Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.outpoint
|
||||
|
||||
@property
|
||||
def claim_height(self) -> typing.Optional[int]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.height
|
||||
|
||||
@property
|
||||
def channel_claim_id(self) -> typing.Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.channel_claim_id
|
||||
|
||||
@property
|
||||
def channel_name(self) -> typing.Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.channel_name
|
||||
|
||||
@property
|
||||
def claim_name(self) -> typing.Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim_name
|
||||
|
||||
@property
|
||||
def metadata(self) -> typing.Optional[typing.Dict]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict()
|
||||
|
||||
@property
|
||||
def metadata_protobuf(self) -> bytes:
|
||||
if self.stream_claim_info:
|
||||
return binascii.hexlify(self.stream_claim_info.claim.to_bytes())
|
||||
|
||||
@property
|
||||
def blobs_completed(self) -> int:
|
||||
return sum([1 if b.blob_hash in self.blob_manager.completed_blob_hashes else 0
|
||||
@@ -218,39 +116,34 @@ class ManagedStream:
|
||||
def blobs_remaining(self) -> int:
|
||||
return self.blobs_in_stream - self.blobs_completed
|
||||
|
||||
@property
|
||||
def full_path(self) -> typing.Optional[str]:
|
||||
return os.path.join(self.download_directory, os.path.basename(self.file_name)) \
|
||||
if self.file_name and self.download_directory else None
|
||||
|
||||
@property
|
||||
def output_file_exists(self):
|
||||
return os.path.isfile(self.full_path) if self.full_path else False
|
||||
|
||||
@property
|
||||
def mime_type(self):
|
||||
return guess_media_type(os.path.basename(self.descriptor.suggested_file_name))[0]
|
||||
|
||||
@classmethod
|
||||
async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
file_path: str, key: typing.Optional[bytes] = None,
|
||||
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedStream':
|
||||
"""
|
||||
Generate a stream from a file and save it to the db
|
||||
"""
|
||||
descriptor = await StreamDescriptor.create_stream(
|
||||
loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
|
||||
blob_completed_callback=blob_manager.blob_completed
|
||||
)
|
||||
await blob_manager.storage.store_stream(
|
||||
blob_manager.get_blob(descriptor.sd_hash), descriptor
|
||||
)
|
||||
row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path),
|
||||
os.path.dirname(file_path), 0)
|
||||
return cls(loop, config, blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
|
||||
os.path.basename(file_path), status=cls.STATUS_FINISHED, rowid=row_id, descriptor=descriptor)
|
||||
@property
|
||||
def download_path(self):
|
||||
return f"{self.download_directory}/{self._file_name}" if self.download_directory and self._file_name else None
|
||||
|
||||
async def start(self, node: typing.Optional['Node'] = None, timeout: typing.Optional[float] = None,
|
||||
# @classmethod
|
||||
# async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config',
|
||||
# file_path: str, key: Optional[bytes] = None,
|
||||
# iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedDownloadSource':
|
||||
# """
|
||||
# Generate a stream from a file and save it to the db
|
||||
# """
|
||||
# descriptor = await StreamDescriptor.create_stream(
|
||||
# loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
|
||||
# blob_completed_callback=blob_manager.blob_completed
|
||||
# )
|
||||
# await blob_manager.storage.store_stream(
|
||||
# blob_manager.get_blob(descriptor.sd_hash), descriptor
|
||||
# )
|
||||
# row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path),
|
||||
# os.path.dirname(file_path), 0)
|
||||
# return cls(loop, config, blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
|
||||
# os.path.basename(file_path), status=cls.STATUS_FINISHED, rowid=row_id, descriptor=descriptor)
|
||||
|
||||
async def start(self, timeout: Optional[float] = None,
|
||||
save_now: bool = False):
|
||||
timeout = timeout or self.config.download_timeout
|
||||
if self._running.is_set():
|
||||
@@ -258,7 +151,7 @@ class ManagedStream:
|
||||
log.info("start downloader for stream (sd hash: %s)", self.sd_hash)
|
||||
self._running.set()
|
||||
try:
|
||||
await asyncio.wait_for(self.downloader.start(node), timeout, loop=self.loop)
|
||||
await asyncio.wait_for(self.downloader.start(), timeout, loop=self.loop)
|
||||
except asyncio.TimeoutError:
|
||||
self._running.clear()
|
||||
raise DownloadSDTimeoutError(self.sd_hash)
|
||||
@@ -268,6 +161,11 @@ class ManagedStream:
|
||||
self.delayed_stop_task = self.loop.create_task(self._delayed_stop())
|
||||
if not await self.blob_manager.storage.file_exists(self.sd_hash):
|
||||
if save_now:
|
||||
if not self._file_name:
|
||||
self._file_name = await get_next_available_file_name(
|
||||
self.loop, self.download_directory,
|
||||
self._file_name or sanitize_file_name(self.descriptor.suggested_file_name)
|
||||
)
|
||||
file_name, download_dir = self._file_name, self.download_directory
|
||||
else:
|
||||
file_name, download_dir = None, None
|
||||
@@ -287,7 +185,7 @@ class ManagedStream:
|
||||
if (finished and self.status != self.STATUS_FINISHED) or self.status == self.STATUS_RUNNING:
|
||||
await self.update_status(self.STATUS_FINISHED if finished else self.STATUS_STOPPED)
|
||||
|
||||
async def _aiter_read_stream(self, start_blob_num: typing.Optional[int] = 0, connection_id: int = 0)\
|
||||
async def _aiter_read_stream(self, start_blob_num: Optional[int] = 0, connection_id: int = 0)\
|
||||
-> typing.AsyncIterator[typing.Tuple['BlobInfo', bytes]]:
|
||||
if start_blob_num >= len(self.descriptor.blobs[:-1]):
|
||||
raise IndexError(start_blob_num)
|
||||
@@ -299,13 +197,13 @@ class ManagedStream:
|
||||
decrypted = await self.downloader.read_blob(blob_info, connection_id)
|
||||
yield (blob_info, decrypted)
|
||||
|
||||
async def stream_file(self, request: Request, node: typing.Optional['Node'] = None) -> StreamResponse:
|
||||
async def stream_file(self, request: Request) -> StreamResponse:
|
||||
log.info("stream file to browser for lbry://%s#%s (sd hash %s...)", self.claim_name, self.claim_id,
|
||||
self.sd_hash[:6])
|
||||
headers, size, skip_blobs, first_blob_start_offset = self._prepare_range_response_headers(
|
||||
request.headers.get('range', 'bytes=0-')
|
||||
)
|
||||
await self.start(node)
|
||||
await self.start()
|
||||
response = StreamResponse(
|
||||
status=206,
|
||||
headers=headers
|
||||
@@ -343,9 +241,10 @@ class ManagedStream:
|
||||
self.streaming.clear()
|
||||
|
||||
@staticmethod
|
||||
def _write_decrypted_blob(handle: typing.IO, data: bytes):
|
||||
handle.write(data)
|
||||
handle.flush()
|
||||
def _write_decrypted_blob(output_path: str, data: bytes):
|
||||
with open(output_path, 'ab') as handle:
|
||||
handle.write(data)
|
||||
handle.flush()
|
||||
|
||||
async def _save_file(self, output_path: str):
|
||||
log.info("save file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id, self.sd_hash[:6],
|
||||
@@ -355,12 +254,12 @@ class ManagedStream:
|
||||
self.finished_writing.clear()
|
||||
self.started_writing.clear()
|
||||
try:
|
||||
with open(output_path, 'wb') as file_write_handle:
|
||||
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
|
||||
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
|
||||
await self.loop.run_in_executor(None, self._write_decrypted_blob, file_write_handle, decrypted)
|
||||
if not self.started_writing.is_set():
|
||||
self.started_writing.set()
|
||||
open(output_path, 'wb').close()
|
||||
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
|
||||
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
|
||||
await self.loop.run_in_executor(None, self._write_decrypted_blob, output_path, decrypted)
|
||||
if not self.started_writing.is_set():
|
||||
self.started_writing.set()
|
||||
await self.update_status(ManagedStream.STATUS_FINISHED)
|
||||
if self.analytics_manager:
|
||||
self.loop.create_task(self.analytics_manager.send_download_finished(
|
||||
@@ -390,9 +289,8 @@ class ManagedStream:
|
||||
self.saving.clear()
|
||||
self.finished_write_attempt.set()
|
||||
|
||||
async def save_file(self, file_name: typing.Optional[str] = None, download_directory: typing.Optional[str] = None,
|
||||
node: typing.Optional['Node'] = None):
|
||||
await self.start(node)
|
||||
async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
|
||||
await self.start()
|
||||
if self.file_output_task and not self.file_output_task.done(): # cancel an already running save task
|
||||
self.file_output_task.cancel()
|
||||
self.download_directory = download_directory or self.download_directory or self.config.download_dir
|
||||
@@ -456,26 +354,30 @@ class ManagedStream:
|
||||
self.reflector_progress = int((i + 1) / len(we_have) * 100)
|
||||
except (asyncio.TimeoutError, ValueError):
|
||||
return sent
|
||||
except ConnectionRefusedError:
|
||||
except ConnectionError:
|
||||
return sent
|
||||
except (OSError, Exception) as err:
|
||||
if isinstance(err, asyncio.CancelledError):
|
||||
log.warning("stopped uploading %s#%s to reflector", self.claim_name, self.claim_id)
|
||||
elif isinstance(err, OSError):
|
||||
log.warning(
|
||||
"stopped uploading %s#%s to reflector because blobs were deleted or moved", self.claim_name,
|
||||
self.claim_id
|
||||
)
|
||||
else:
|
||||
log.exception("unexpected error reflecting %s#%s", self.claim_name, self.claim_id)
|
||||
return sent
|
||||
finally:
|
||||
if protocol.transport:
|
||||
protocol.transport.close()
|
||||
self.uploading_to_reflector = False
|
||||
|
||||
if not self.fully_reflected.is_set():
|
||||
self.fully_reflected.set()
|
||||
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
|
||||
return sent
|
||||
|
||||
def set_claim(self, claim_info: typing.Dict, claim: 'Claim'):
|
||||
self.stream_claim_info = StoredContentClaim(
|
||||
f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
|
||||
claim_info['name'], claim_info['amount'], claim_info['height'],
|
||||
binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'],
|
||||
claim_info['claim_sequence'], claim_info.get('channel_name')
|
||||
)
|
||||
|
||||
async def update_content_claim(self, claim_info: typing.Optional[typing.Dict] = None):
|
||||
async def update_content_claim(self, claim_info: Optional[typing.Dict] = None):
|
||||
if not claim_info:
|
||||
claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash)
|
||||
self.set_claim(claim_info, claim_info['value'])
|
||||
|
||||
@@ -35,6 +35,8 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
def connection_lost(self, exc: typing.Optional[Exception]):
|
||||
self.transport = None
|
||||
self.connected.clear()
|
||||
if self.pending_request:
|
||||
self.pending_request.cancel()
|
||||
if self.reflected_blobs:
|
||||
log.info("Finished sending reflector %i blobs", len(self.reflected_blobs))
|
||||
|
||||
@@ -56,12 +58,18 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
self.response_buff = b''
|
||||
return
|
||||
|
||||
async def send_request(self, request_dict: typing.Dict):
|
||||
async def send_request(self, request_dict: typing.Dict, timeout: int = 180):
|
||||
msg = json.dumps(request_dict)
|
||||
self.transport.write(msg.encode())
|
||||
try:
|
||||
self.pending_request = self.loop.create_task(self.response_queue.get())
|
||||
self.transport.write(msg.encode())
|
||||
self.pending_request = self.loop.create_task(asyncio.wait_for(self.response_queue.get(), timeout))
|
||||
return await self.pending_request
|
||||
except (AttributeError, asyncio.CancelledError) as err:
|
||||
# attribute error happens when we transport.write after disconnect
|
||||
# cancelled error happens when the pending_request task is cancelled by a disconnect
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
raise err if isinstance(err, asyncio.CancelledError) else asyncio.CancelledError()
|
||||
finally:
|
||||
self.pending_request = None
|
||||
|
||||
@@ -86,8 +94,16 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
needed = response.get('needed_blobs', [])
|
||||
sent_sd = False
|
||||
if response['send_sd_blob']:
|
||||
await sd_blob.sendfile(self)
|
||||
received = await self.response_queue.get()
|
||||
try:
|
||||
sent = await sd_blob.sendfile(self)
|
||||
if sent == -1:
|
||||
log.warning("failed to send sd blob")
|
||||
raise asyncio.CancelledError()
|
||||
received = await asyncio.wait_for(self.response_queue.get(), 30)
|
||||
except asyncio.CancelledError as err:
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
raise err
|
||||
if received.get('received_sd_blob'):
|
||||
sent_sd = True
|
||||
if not needed:
|
||||
@@ -110,8 +126,16 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
if 'send_blob' not in response:
|
||||
raise ValueError("I don't know whether to send the blob or not!")
|
||||
if response['send_blob']:
|
||||
await blob.sendfile(self)
|
||||
received = await self.response_queue.get()
|
||||
try:
|
||||
sent = await blob.sendfile(self)
|
||||
if sent == -1:
|
||||
log.warning("failed to send blob")
|
||||
raise asyncio.CancelledError()
|
||||
received = await asyncio.wait_for(self.response_queue.get(), 30)
|
||||
except asyncio.CancelledError as err:
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
raise err
|
||||
if received.get('received_blob'):
|
||||
self.reflected_blobs.append(blob.blob_hash)
|
||||
log.info("Sent reflector blob %s", blob.blob_hash[:8])
|
||||
|
||||
@@ -15,7 +15,9 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ReflectorServerProtocol(asyncio.Protocol):
|
||||
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000):
|
||||
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000,
|
||||
stop_event: asyncio.Event = None, incoming_event: asyncio.Event = None,
|
||||
not_incoming_event: asyncio.Event = None):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
@@ -27,11 +29,25 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
self.descriptor: typing.Optional['StreamDescriptor'] = None
|
||||
self.sd_blob: typing.Optional['BlobFile'] = None
|
||||
self.received = []
|
||||
self.incoming = asyncio.Event(loop=self.loop)
|
||||
self.incoming = incoming_event or asyncio.Event(loop=self.loop)
|
||||
self.not_incoming = not_incoming_event or asyncio.Event(loop=self.loop)
|
||||
self.stop_event = stop_event or asyncio.Event(loop=self.loop)
|
||||
self.chunk_size = response_chunk_size
|
||||
self.wait_for_stop_task: typing.Optional[asyncio.Task] = None
|
||||
|
||||
async def wait_for_stop(self):
|
||||
await self.stop_event.wait()
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
|
||||
def connection_made(self, transport):
|
||||
self.transport = transport
|
||||
self.wait_for_stop_task = self.loop.create_task(self.wait_for_stop())
|
||||
|
||||
def connection_lost(self, exc):
|
||||
if self.wait_for_stop_task:
|
||||
self.wait_for_stop_task.cancel()
|
||||
self.wait_for_stop_task = None
|
||||
|
||||
def data_received(self, data: bytes):
|
||||
if self.incoming.is_set():
|
||||
@@ -73,6 +89,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
self.sd_blob = self.blob_manager.get_blob(request['sd_blob_hash'], request['sd_blob_size'])
|
||||
if not self.sd_blob.get_is_verified():
|
||||
self.writer = self.sd_blob.get_blob_writer(self.transport.get_extra_info('peername'))
|
||||
self.not_incoming.clear()
|
||||
self.incoming.set()
|
||||
self.send_response({"send_sd_blob": True})
|
||||
try:
|
||||
@@ -86,6 +103,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
self.transport.close()
|
||||
finally:
|
||||
self.incoming.clear()
|
||||
self.not_incoming.set()
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
else:
|
||||
@@ -93,6 +111,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
self.loop, self.blob_manager.blob_dir, self.sd_blob
|
||||
)
|
||||
self.incoming.clear()
|
||||
self.not_incoming.set()
|
||||
if self.writer:
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
@@ -112,6 +131,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
blob = self.blob_manager.get_blob(request['blob_hash'], request['blob_size'])
|
||||
if not blob.get_is_verified():
|
||||
self.writer = blob.get_blob_writer(self.transport.get_extra_info('peername'))
|
||||
self.not_incoming.clear()
|
||||
self.incoming.set()
|
||||
self.send_response({"send_blob": True})
|
||||
try:
|
||||
@@ -120,6 +140,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
except asyncio.TimeoutError:
|
||||
self.send_response({"received_blob": False})
|
||||
self.incoming.clear()
|
||||
self.not_incoming.set()
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
else:
|
||||
@@ -130,12 +151,18 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
|
||||
|
||||
class ReflectorServer:
|
||||
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000):
|
||||
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000,
|
||||
stop_event: asyncio.Event = None, incoming_event: asyncio.Event = None,
|
||||
not_incoming_event: asyncio.Event = None):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: typing.Optional[asyncio.Task] = None
|
||||
self.started_listening = asyncio.Event(loop=self.loop)
|
||||
self.stopped_listening = asyncio.Event(loop=self.loop)
|
||||
self.incoming_event = incoming_event or asyncio.Event(loop=self.loop)
|
||||
self.not_incoming_event = not_incoming_event or asyncio.Event(loop=self.loop)
|
||||
self.response_chunk_size = response_chunk_size
|
||||
self.stop_event = stop_event
|
||||
|
||||
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
|
||||
if self.server_task is not None:
|
||||
@@ -143,13 +170,20 @@ class ReflectorServer:
|
||||
|
||||
async def _start_server():
|
||||
server = await self.loop.create_server(
|
||||
lambda: ReflectorServerProtocol(self.blob_manager, self.response_chunk_size),
|
||||
lambda: ReflectorServerProtocol(
|
||||
self.blob_manager, self.response_chunk_size, self.stop_event, self.incoming_event,
|
||||
self.not_incoming_event
|
||||
),
|
||||
interface, port
|
||||
)
|
||||
self.started_listening.set()
|
||||
self.stopped_listening.clear()
|
||||
log.info("Reflector server listening on TCP %s:%i", interface, port)
|
||||
async with server:
|
||||
await server.serve_forever()
|
||||
try:
|
||||
async with server:
|
||||
await server.serve_forever()
|
||||
finally:
|
||||
self.stopped_listening.set()
|
||||
|
||||
self.server_task = self.loop.create_task(_start_server())
|
||||
|
||||
|
||||
+96
-351
@@ -6,93 +6,67 @@ import random
|
||||
import typing
|
||||
from typing import Optional
|
||||
from aiohttp.web import Request
|
||||
from lbry.error import ResolveError, InvalidStreamDescriptorError, DownloadSDTimeoutError, InsufficientFundsError
|
||||
from lbry.error import ResolveTimeoutError, DownloadDataTimeoutError, KeyFeeAboveMaxAllowedError
|
||||
from lbry.utils import cache_concurrent
|
||||
from lbry.error import InvalidStreamDescriptorError
|
||||
from lbry.file.source_manager import SourceManager
|
||||
from lbry.stream.descriptor import StreamDescriptor
|
||||
from lbry.stream.managed_stream import ManagedStream
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.schema.url import URL
|
||||
from lbry.wallet.dewies import dewies_to_lbc
|
||||
from lbry.wallet import Output
|
||||
|
||||
from lbry.file.source import ManagedDownloadSource
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.blob.blob_manager import BlobManager
|
||||
from lbry.dht.node import Node
|
||||
from lbry.wallet.wallet import WalletManager
|
||||
from lbry.wallet.transaction import Transaction
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
from lbry.wallet.transaction import Transaction
|
||||
from lbry.wallet.manager import WalletManager
|
||||
from lbry.wallet.wallet import Wallet
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
FILTER_FIELDS = [
|
||||
'rowid',
|
||||
'status',
|
||||
'file_name',
|
||||
'added_on',
|
||||
'sd_hash',
|
||||
'stream_hash',
|
||||
'claim_name',
|
||||
'claim_height',
|
||||
'claim_id',
|
||||
'outpoint',
|
||||
'txid',
|
||||
'nout',
|
||||
'channel_claim_id',
|
||||
'channel_name',
|
||||
'full_status', # TODO: remove
|
||||
'blobs_remaining',
|
||||
'blobs_in_stream'
|
||||
]
|
||||
|
||||
SET_FILTER_FIELDS = {
|
||||
"claim_ids": "claim_id",
|
||||
"channel_claim_ids": "channel_claim_id",
|
||||
"outpoints": "outpoint"
|
||||
}
|
||||
|
||||
COMPARISON_OPERATORS = {
|
||||
'eq': lambda a, b: a == b,
|
||||
'ne': lambda a, b: a != b,
|
||||
'g': lambda a, b: a > b,
|
||||
'l': lambda a, b: a < b,
|
||||
'ge': lambda a, b: a >= b,
|
||||
'le': lambda a, b: a <= b,
|
||||
'in': lambda a, b: a in b
|
||||
}
|
||||
|
||||
|
||||
def path_or_none(path) -> Optional[str]:
|
||||
if not path:
|
||||
def path_or_none(encoded_path) -> Optional[str]:
|
||||
if not encoded_path:
|
||||
return
|
||||
return binascii.unhexlify(path).decode()
|
||||
return binascii.unhexlify(encoded_path).decode()
|
||||
|
||||
|
||||
class StreamManager:
|
||||
class StreamManager(SourceManager):
|
||||
_sources: typing.Dict[str, ManagedStream]
|
||||
|
||||
filter_fields = SourceManager.filter_fields
|
||||
filter_fields.update({
|
||||
'sd_hash',
|
||||
'stream_hash',
|
||||
'full_status', # TODO: remove
|
||||
'blobs_remaining',
|
||||
'blobs_in_stream',
|
||||
'uploading_to_reflector',
|
||||
'is_fully_reflected'
|
||||
})
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
wallet_manager: 'WalletManager', storage: 'SQLiteStorage', node: Optional['Node'],
|
||||
analytics_manager: Optional['AnalyticsManager'] = None):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
super().__init__(loop, config, storage, analytics_manager)
|
||||
self.blob_manager = blob_manager
|
||||
self.wallet_manager = wallet_manager
|
||||
self.storage = storage
|
||||
self.node = node
|
||||
self.analytics_manager = analytics_manager
|
||||
self.streams: typing.Dict[str, ManagedStream] = {}
|
||||
self.resume_saving_task: Optional[asyncio.Task] = None
|
||||
self.re_reflect_task: Optional[asyncio.Task] = None
|
||||
self.update_stream_finished_futs: typing.List[asyncio.Future] = []
|
||||
self.running_reflector_uploads: typing.Dict[str, asyncio.Task] = {}
|
||||
self.started = asyncio.Event(loop=self.loop)
|
||||
|
||||
@property
|
||||
def streams(self):
|
||||
return self._sources
|
||||
|
||||
def add(self, source: ManagedStream):
|
||||
super().add(source)
|
||||
self.storage.content_claim_callbacks[source.stream_hash] = lambda: self._update_content_claim(source)
|
||||
|
||||
async def _update_content_claim(self, stream: ManagedStream):
|
||||
claim_info = await self.storage.get_content_claim(stream.stream_hash)
|
||||
self.streams.setdefault(stream.sd_hash, stream).set_claim(claim_info, claim_info['value'])
|
||||
self._sources.setdefault(stream.sd_hash, stream).set_claim(claim_info, claim_info['value'])
|
||||
|
||||
async def recover_streams(self, file_infos: typing.List[typing.Dict]):
|
||||
to_restore = []
|
||||
@@ -123,10 +97,10 @@ class StreamManager:
|
||||
# if self.blob_manager._save_blobs:
|
||||
# log.info("Recovered %i/%i attempted streams", len(to_restore), len(file_infos))
|
||||
|
||||
async def add_stream(self, rowid: int, sd_hash: str, file_name: Optional[str],
|
||||
download_directory: Optional[str], status: str,
|
||||
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
|
||||
added_on: Optional[int], fully_reflected: bool):
|
||||
async def _load_stream(self, rowid: int, sd_hash: str, file_name: Optional[str],
|
||||
download_directory: Optional[str], status: str,
|
||||
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
|
||||
added_on: Optional[int], fully_reflected: Optional[bool]):
|
||||
try:
|
||||
descriptor = await self.blob_manager.get_stream_descriptor(sd_hash)
|
||||
except InvalidStreamDescriptorError as err:
|
||||
@@ -139,10 +113,9 @@ class StreamManager:
|
||||
)
|
||||
if fully_reflected:
|
||||
stream.fully_reflected.set()
|
||||
self.streams[sd_hash] = stream
|
||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||
self.add(stream)
|
||||
|
||||
async def load_and_resume_streams_from_database(self):
|
||||
async def initialize_from_database(self):
|
||||
to_recover = []
|
||||
to_start = []
|
||||
|
||||
@@ -156,7 +129,6 @@ class StreamManager:
|
||||
to_recover.append(file_info)
|
||||
to_start.append(file_info)
|
||||
if to_recover:
|
||||
log.info("Recover %i files", len(to_recover))
|
||||
await self.recover_streams(to_recover)
|
||||
|
||||
log.info("Initializing %i files", len(to_start))
|
||||
@@ -167,7 +139,7 @@ class StreamManager:
|
||||
download_directory = path_or_none(file_info['download_directory'])
|
||||
if file_name and download_directory and not file_info['saved_file'] and file_info['status'] == 'running':
|
||||
to_resume_saving.append((file_name, download_directory, file_info['sd_hash']))
|
||||
add_stream_tasks.append(self.loop.create_task(self.add_stream(
|
||||
add_stream_tasks.append(self.loop.create_task(self._load_stream(
|
||||
file_info['rowid'], file_info['sd_hash'], file_name,
|
||||
download_directory, file_info['status'],
|
||||
file_info['claim'], file_info['content_fee'],
|
||||
@@ -175,25 +147,32 @@ class StreamManager:
|
||||
)))
|
||||
if add_stream_tasks:
|
||||
await asyncio.gather(*add_stream_tasks, loop=self.loop)
|
||||
log.info("Started stream manager with %i files", len(self.streams))
|
||||
log.info("Started stream manager with %i files", len(self._sources))
|
||||
if not self.node:
|
||||
log.info("no DHT node given, resuming downloads trusting that we can contact reflector")
|
||||
if to_resume_saving:
|
||||
self.resume_saving_task = self.loop.create_task(self.resume(to_resume_saving))
|
||||
|
||||
async def resume(self, to_resume_saving):
|
||||
log.info("Resuming saving %i files", len(to_resume_saving))
|
||||
await asyncio.gather(
|
||||
*(self.streams[sd_hash].save_file(file_name, download_directory, node=self.node)
|
||||
for (file_name, download_directory, sd_hash) in to_resume_saving),
|
||||
loop=self.loop
|
||||
)
|
||||
log.info("Resuming saving %i files", len(to_resume_saving))
|
||||
self.resume_saving_task = asyncio.ensure_future(asyncio.gather(
|
||||
*(self._sources[sd_hash].save_file(file_name, download_directory)
|
||||
for (file_name, download_directory, sd_hash) in to_resume_saving),
|
||||
loop=self.loop
|
||||
))
|
||||
|
||||
async def reflect_streams(self):
|
||||
try:
|
||||
return await self._reflect_streams()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
log.exception("reflector task encountered an unexpected error!")
|
||||
|
||||
async def _reflect_streams(self):
|
||||
# todo: those debug statements are temporary for #2987 - remove them if its closed
|
||||
while True:
|
||||
if self.config.reflect_streams and self.config.reflector_servers:
|
||||
log.debug("collecting streams to reflect")
|
||||
sd_hashes = await self.storage.get_streams_to_re_reflect()
|
||||
sd_hashes = [sd for sd in sd_hashes if sd in self.streams]
|
||||
sd_hashes = [sd for sd in sd_hashes if sd in self._sources]
|
||||
batch = []
|
||||
while sd_hashes:
|
||||
stream = self.streams[sd_hashes.pop()]
|
||||
@@ -202,25 +181,26 @@ class StreamManager:
|
||||
stream.fully_reflected.is_set():
|
||||
batch.append(self.reflect_stream(stream))
|
||||
if len(batch) >= self.config.concurrent_reflector_uploads:
|
||||
log.debug("waiting for batch of %s reflecting streams", len(batch))
|
||||
await asyncio.gather(*batch, loop=self.loop)
|
||||
log.debug("done processing %s streams", len(batch))
|
||||
batch = []
|
||||
if batch:
|
||||
log.debug("waiting for batch of %s reflecting streams", len(batch))
|
||||
await asyncio.gather(*batch, loop=self.loop)
|
||||
log.debug("done processing %s streams", len(batch))
|
||||
await asyncio.sleep(300, loop=self.loop)
|
||||
|
||||
async def start(self):
|
||||
await self.load_and_resume_streams_from_database()
|
||||
await super().start()
|
||||
self.re_reflect_task = self.loop.create_task(self.reflect_streams())
|
||||
self.started.set()
|
||||
|
||||
def stop(self):
|
||||
super().stop()
|
||||
if self.resume_saving_task and not self.resume_saving_task.done():
|
||||
self.resume_saving_task.cancel()
|
||||
if self.re_reflect_task and not self.re_reflect_task.done():
|
||||
self.re_reflect_task.cancel()
|
||||
while self.streams:
|
||||
_, stream = self.streams.popitem()
|
||||
stream.stop_tasks()
|
||||
while self.update_stream_finished_futs:
|
||||
self.update_stream_finished_futs.pop().cancel()
|
||||
while self.running_reflector_uploads:
|
||||
@@ -243,280 +223,45 @@ class StreamManager:
|
||||
)
|
||||
return task
|
||||
|
||||
async def create_stream(self, file_path: str, key: Optional[bytes] = None,
|
||||
iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
|
||||
stream = await ManagedStream.create(self.loop, self.config, self.blob_manager, file_path, key, iv_generator)
|
||||
async def create(self, file_path: str, key: Optional[bytes] = None,
|
||||
iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
|
||||
descriptor = await StreamDescriptor.create_stream(
|
||||
self.loop, self.blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
|
||||
blob_completed_callback=self.blob_manager.blob_completed
|
||||
)
|
||||
await self.storage.store_stream(
|
||||
self.blob_manager.get_blob(descriptor.sd_hash), descriptor
|
||||
)
|
||||
row_id = await self.storage.save_published_file(
|
||||
descriptor.stream_hash, os.path.basename(file_path), os.path.dirname(file_path), 0
|
||||
)
|
||||
stream = ManagedStream(
|
||||
self.loop, self.config, self.blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
|
||||
os.path.basename(file_path), status=ManagedDownloadSource.STATUS_FINISHED,
|
||||
rowid=row_id, descriptor=descriptor
|
||||
)
|
||||
self.streams[stream.sd_hash] = stream
|
||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||
if self.config.reflect_streams and self.config.reflector_servers:
|
||||
self.reflect_stream(stream)
|
||||
return stream
|
||||
|
||||
async def delete_stream(self, stream: ManagedStream, delete_file: Optional[bool] = False):
|
||||
if stream.sd_hash in self.running_reflector_uploads:
|
||||
self.running_reflector_uploads[stream.sd_hash].cancel()
|
||||
stream.stop_tasks()
|
||||
if stream.sd_hash in self.streams:
|
||||
del self.streams[stream.sd_hash]
|
||||
blob_hashes = [stream.sd_hash] + [b.blob_hash for b in stream.descriptor.blobs[:-1]]
|
||||
async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
|
||||
if not isinstance(source, ManagedStream):
|
||||
return
|
||||
if source.identifier in self.running_reflector_uploads:
|
||||
self.running_reflector_uploads[source.identifier].cancel()
|
||||
source.stop_tasks()
|
||||
if source.identifier in self.streams:
|
||||
del self.streams[source.identifier]
|
||||
blob_hashes = [source.identifier] + [b.blob_hash for b in source.descriptor.blobs[:-1]]
|
||||
await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False)
|
||||
await self.storage.delete_stream(stream.descriptor)
|
||||
if delete_file and stream.output_file_exists:
|
||||
os.remove(stream.full_path)
|
||||
|
||||
def get_stream_by_stream_hash(self, stream_hash: str) -> Optional[ManagedStream]:
|
||||
streams = tuple(filter(lambda stream: stream.stream_hash == stream_hash, self.streams.values()))
|
||||
if streams:
|
||||
return streams[0]
|
||||
|
||||
def get_filtered_streams(self, sort_by: Optional[str] = None, reverse: Optional[bool] = False,
|
||||
comparison: Optional[str] = None,
|
||||
**search_by) -> typing.List[ManagedStream]:
|
||||
"""
|
||||
Get a list of filtered and sorted ManagedStream objects
|
||||
|
||||
:param sort_by: field to sort by
|
||||
:param reverse: reverse sorting
|
||||
:param comparison: comparison operator used for filtering
|
||||
:param search_by: fields and values to filter by
|
||||
"""
|
||||
if sort_by and sort_by not in FILTER_FIELDS:
|
||||
raise ValueError(f"'{sort_by}' is not a valid field to sort by")
|
||||
if comparison and comparison not in COMPARISON_OPERATORS:
|
||||
raise ValueError(f"'{comparison}' is not a valid comparison")
|
||||
if 'full_status' in search_by:
|
||||
del search_by['full_status']
|
||||
|
||||
for search in search_by:
|
||||
if search not in FILTER_FIELDS:
|
||||
raise ValueError(f"'{search}' is not a valid search operation")
|
||||
|
||||
compare_sets = {}
|
||||
if isinstance(search_by.get('claim_id'), list):
|
||||
compare_sets['claim_ids'] = search_by.pop('claim_id')
|
||||
if isinstance(search_by.get('outpoint'), list):
|
||||
compare_sets['outpoints'] = search_by.pop('outpoint')
|
||||
if isinstance(search_by.get('channel_claim_id'), list):
|
||||
compare_sets['channel_claim_ids'] = search_by.pop('channel_claim_id')
|
||||
|
||||
if search_by:
|
||||
comparison = comparison or 'eq'
|
||||
streams = []
|
||||
for stream in self.streams.values():
|
||||
matched = False
|
||||
for set_search, val in compare_sets.items():
|
||||
if COMPARISON_OPERATORS[comparison](getattr(stream, SET_FILTER_FIELDS[set_search]), val):
|
||||
streams.append(stream)
|
||||
matched = True
|
||||
break
|
||||
if matched:
|
||||
continue
|
||||
for search, val in search_by.items():
|
||||
this_stream = getattr(stream, search)
|
||||
if COMPARISON_OPERATORS[comparison](this_stream, val):
|
||||
streams.append(stream)
|
||||
break
|
||||
else:
|
||||
streams = list(self.streams.values())
|
||||
if sort_by:
|
||||
streams.sort(key=lambda s: getattr(s, sort_by))
|
||||
if reverse:
|
||||
streams.reverse()
|
||||
return streams
|
||||
|
||||
async def _check_update_or_replace(self, outpoint: str, claim_id: str, claim: Claim
|
||||
) -> typing.Tuple[Optional[ManagedStream], Optional[ManagedStream]]:
|
||||
existing = self.get_filtered_streams(outpoint=outpoint)
|
||||
if existing:
|
||||
return existing[0], None
|
||||
existing = self.get_filtered_streams(sd_hash=claim.stream.source.sd_hash)
|
||||
if existing and existing[0].claim_id != claim_id:
|
||||
raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {claim_id}")
|
||||
if existing:
|
||||
log.info("claim contains a metadata only update to a stream we have")
|
||||
await self.storage.save_content_claim(
|
||||
existing[0].stream_hash, outpoint
|
||||
)
|
||||
await self._update_content_claim(existing[0])
|
||||
return existing[0], None
|
||||
else:
|
||||
existing_for_claim_id = self.get_filtered_streams(claim_id=claim_id)
|
||||
if existing_for_claim_id:
|
||||
log.info("claim contains an update to a stream we have, downloading it")
|
||||
return None, existing_for_claim_id[0]
|
||||
return None, None
|
||||
|
||||
@staticmethod
|
||||
def _convert_to_old_resolve_output(wallet_manager, resolves):
|
||||
result = {}
|
||||
for url, txo in resolves.items():
|
||||
if isinstance(txo, Output):
|
||||
tx_height = txo.tx_ref.height
|
||||
best_height = wallet_manager.ledger.headers.height
|
||||
result[url] = {
|
||||
'name': txo.claim_name,
|
||||
'value': txo.claim,
|
||||
'protobuf': binascii.hexlify(txo.claim.to_bytes()),
|
||||
'claim_id': txo.claim_id,
|
||||
'txid': txo.tx_ref.id,
|
||||
'nout': txo.position,
|
||||
'amount': dewies_to_lbc(txo.amount),
|
||||
'effective_amount': txo.meta.get('effective_amount', 0),
|
||||
'height': tx_height,
|
||||
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
|
||||
'claim_sequence': -1,
|
||||
'address': txo.get_address(wallet_manager.ledger),
|
||||
'valid_at_height': txo.meta.get('activation_height', None),
|
||||
'timestamp': wallet_manager.ledger.headers.estimated_timestamp(tx_height),
|
||||
'supports': []
|
||||
}
|
||||
else:
|
||||
result[url] = txo
|
||||
return result
|
||||
|
||||
@cache_concurrent
|
||||
async def download_stream_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
|
||||
timeout: Optional[float] = None,
|
||||
file_name: Optional[str] = None,
|
||||
download_directory: Optional[str] = None,
|
||||
save_file: Optional[bool] = None,
|
||||
resolve_timeout: float = 3.0,
|
||||
wallet: Optional['Wallet'] = None) -> ManagedStream:
|
||||
manager = self.wallet_manager
|
||||
wallet = wallet or manager.default_wallet
|
||||
timeout = timeout or self.config.download_timeout
|
||||
start_time = self.loop.time()
|
||||
resolved_time = None
|
||||
stream = None
|
||||
txo: Optional[Output] = None
|
||||
error = None
|
||||
outpoint = None
|
||||
if save_file is None:
|
||||
save_file = self.config.save_files
|
||||
if file_name and not save_file:
|
||||
save_file = True
|
||||
if save_file:
|
||||
download_directory = download_directory or self.config.download_dir
|
||||
else:
|
||||
download_directory = None
|
||||
|
||||
payment = None
|
||||
try:
|
||||
# resolve the claim
|
||||
if not URL.parse(uri).has_stream:
|
||||
raise ResolveError("cannot download a channel claim, specify a /path")
|
||||
try:
|
||||
response = await asyncio.wait_for(
|
||||
manager.ledger.resolve(wallet.accounts, [uri], include_purchase_receipt=True),
|
||||
resolve_timeout
|
||||
)
|
||||
resolved_result = self._convert_to_old_resolve_output(manager, response)
|
||||
except asyncio.TimeoutError:
|
||||
raise ResolveTimeoutError(uri)
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
|
||||
raise
|
||||
log.exception("Unexpected error resolving stream:")
|
||||
raise ResolveError(f"Unexpected error resolving stream: {str(err)}")
|
||||
await self.storage.save_claims_for_resolve([
|
||||
value for value in resolved_result.values() if 'error' not in value
|
||||
])
|
||||
resolved = resolved_result.get(uri, {})
|
||||
resolved = resolved if 'value' in resolved else resolved.get('claim')
|
||||
if not resolved:
|
||||
raise ResolveError(f"Failed to resolve stream at '{uri}'")
|
||||
if 'error' in resolved:
|
||||
raise ResolveError(f"error resolving stream: {resolved['error']}")
|
||||
txo = response[uri]
|
||||
|
||||
claim = Claim.from_bytes(binascii.unhexlify(resolved['protobuf']))
|
||||
outpoint = f"{resolved['txid']}:{resolved['nout']}"
|
||||
resolved_time = self.loop.time() - start_time
|
||||
|
||||
# resume or update an existing stream, if the stream changed: download it and delete the old one after
|
||||
updated_stream, to_replace = await self._check_update_or_replace(outpoint, resolved['claim_id'], claim)
|
||||
if updated_stream:
|
||||
log.info("already have stream for %s", uri)
|
||||
if save_file and updated_stream.output_file_exists:
|
||||
save_file = False
|
||||
await updated_stream.start(node=self.node, timeout=timeout, save_now=save_file)
|
||||
if not updated_stream.output_file_exists and (save_file or file_name or download_directory):
|
||||
await updated_stream.save_file(
|
||||
file_name=file_name, download_directory=download_directory, node=self.node
|
||||
)
|
||||
return updated_stream
|
||||
|
||||
if not to_replace and txo.has_price and not txo.purchase_receipt:
|
||||
payment = await manager.create_purchase_transaction(
|
||||
wallet.accounts, txo, exchange_rate_manager
|
||||
)
|
||||
|
||||
stream = ManagedStream(
|
||||
self.loop, self.config, self.blob_manager, claim.stream.source.sd_hash, download_directory,
|
||||
file_name, ManagedStream.STATUS_RUNNING, content_fee=payment,
|
||||
analytics_manager=self.analytics_manager
|
||||
)
|
||||
log.info("starting download for %s", uri)
|
||||
|
||||
before_download = self.loop.time()
|
||||
await stream.start(self.node, timeout)
|
||||
stream.set_claim(resolved, claim)
|
||||
if to_replace: # delete old stream now that the replacement has started downloading
|
||||
await self.delete_stream(to_replace)
|
||||
|
||||
if payment is not None:
|
||||
await manager.broadcast_or_release(payment)
|
||||
payment = None # to avoid releasing in `finally` later
|
||||
log.info("paid fee of %s for %s", dewies_to_lbc(stream.content_fee.outputs[0].amount), uri)
|
||||
await self.storage.save_content_fee(stream.stream_hash, stream.content_fee)
|
||||
|
||||
self.streams[stream.sd_hash] = stream
|
||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||
await self.storage.save_content_claim(stream.stream_hash, outpoint)
|
||||
if save_file:
|
||||
await asyncio.wait_for(stream.save_file(node=self.node), timeout - (self.loop.time() - before_download),
|
||||
loop=self.loop)
|
||||
return stream
|
||||
except asyncio.TimeoutError:
|
||||
error = DownloadDataTimeoutError(stream.sd_hash)
|
||||
raise error
|
||||
except Exception as err: # forgive data timeout, don't delete stream
|
||||
expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError,
|
||||
KeyFeeAboveMaxAllowedError)
|
||||
if isinstance(err, expected):
|
||||
log.warning("Failed to download %s: %s", uri, str(err))
|
||||
elif isinstance(err, asyncio.CancelledError):
|
||||
pass
|
||||
else:
|
||||
log.exception("Unexpected error downloading stream:")
|
||||
error = err
|
||||
raise
|
||||
finally:
|
||||
if payment is not None:
|
||||
# payment is set to None after broadcasting, if we're here an exception probably happened
|
||||
await manager.ledger.release_tx(payment)
|
||||
if self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or
|
||||
stream.downloader.time_to_first_bytes))):
|
||||
server = self.wallet_manager.ledger.network.client.server
|
||||
self.loop.create_task(
|
||||
self.analytics_manager.send_time_to_first_bytes(
|
||||
resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id,
|
||||
uri, outpoint,
|
||||
None if not stream else len(stream.downloader.blob_downloader.active_connections),
|
||||
None if not stream else len(stream.downloader.blob_downloader.scores),
|
||||
None if not stream else len(stream.downloader.blob_downloader.connection_failures),
|
||||
False if not stream else stream.downloader.added_fixed_peers,
|
||||
self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay,
|
||||
None if not stream else stream.sd_hash,
|
||||
None if not stream else stream.downloader.time_to_descriptor,
|
||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
|
||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
|
||||
None if not stream else stream.downloader.time_to_first_bytes,
|
||||
None if not error else error.__class__.__name__,
|
||||
None if not error else str(error),
|
||||
None if not server else f"{server[0]}:{server[1]}"
|
||||
)
|
||||
)
|
||||
await self.storage.delete_stream(source.descriptor)
|
||||
if delete_file and source.output_file_exists:
|
||||
os.remove(source.full_path)
|
||||
|
||||
async def stream_partial_content(self, request: Request, sd_hash: str):
|
||||
return await self.streams[sd_hash].stream_file(request, self.node)
|
||||
stream = self._sources[sd_hash]
|
||||
if not stream.downloader.node:
|
||||
stream.downloader.node = self.node
|
||||
return await stream.stream_file(request)
|
||||
+13
-6
@@ -23,12 +23,13 @@ from lbry.wallet.orchstr8.node import BlockchainNode, WalletNode
|
||||
from lbry.extras.daemon.daemon import Daemon, jsonrpc_dumps_pretty
|
||||
from lbry.extras.daemon.components import Component, WalletComponent
|
||||
from lbry.extras.daemon.components import (
|
||||
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
DHT_COMPONENT,
|
||||
HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, LIBTORRENT_COMPONENT
|
||||
)
|
||||
from lbry.extras.daemon.componentmanager import ComponentManager
|
||||
from lbry.extras.daemon.exchange_rate_manager import (
|
||||
ExchangeRateManager, ExchangeRate, LBRYFeed, LBRYBTCFeed
|
||||
ExchangeRateManager, ExchangeRate, BittrexBTCFeed, BittrexUSDFeed
|
||||
)
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from lbry.blob.blob_manager import BlobManager
|
||||
@@ -281,8 +282,8 @@ class FakeExchangeRateManager(ExchangeRateManager):
|
||||
|
||||
def get_fake_exchange_rate_manager(rates=None):
|
||||
return FakeExchangeRateManager(
|
||||
[LBRYFeed(), LBRYBTCFeed()],
|
||||
rates or {'BTCLBC': 3.0, 'USDBTC': 2.0}
|
||||
[BittrexBTCFeed(), BittrexUSDFeed()],
|
||||
rates or {'BTCLBC': 3.0, 'USDLBC': 2.0}
|
||||
)
|
||||
|
||||
|
||||
@@ -320,6 +321,7 @@ class CommandTestCase(IntegrationTestCase):
|
||||
self.server_blob_manager = None
|
||||
self.server = None
|
||||
self.reflector = None
|
||||
self.skip_libtorrent = True
|
||||
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
@@ -340,6 +342,7 @@ class CommandTestCase(IntegrationTestCase):
|
||||
server_tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, server_tmp_dir)
|
||||
self.server_config = Config()
|
||||
self.server_config.transaction_cache_size = 10000
|
||||
self.server_storage = SQLiteStorage(self.server_config, ':memory:')
|
||||
await self.server_storage.open()
|
||||
|
||||
@@ -386,12 +389,16 @@ class CommandTestCase(IntegrationTestCase):
|
||||
conf.blockchain_name = 'lbrycrd_regtest'
|
||||
conf.lbryum_servers = [('127.0.0.1', 50001)]
|
||||
conf.reflector_servers = [('127.0.0.1', 5566)]
|
||||
conf.fixed_peers = [('127.0.0.1', 5567)]
|
||||
conf.known_dht_nodes = []
|
||||
conf.blob_lru_cache_size = self.blob_lru_cache_size
|
||||
conf.transaction_cache_size = 10000
|
||||
conf.components_to_skip = [
|
||||
DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
|
||||
PEER_PROTOCOL_SERVER_COMPONENT
|
||||
]
|
||||
if self.skip_libtorrent:
|
||||
conf.components_to_skip.append(LIBTORRENT_COMPONENT)
|
||||
wallet_node.manager.config = conf
|
||||
|
||||
def wallet_maker(component_manager):
|
||||
@@ -403,7 +410,7 @@ class CommandTestCase(IntegrationTestCase):
|
||||
daemon = Daemon(conf, ComponentManager(
|
||||
conf, skip_components=conf.components_to_skip, wallet=wallet_maker,
|
||||
exchange_rate_manager=partial(ExchangeRateManagerComponent, rates={
|
||||
'BTCLBC': 1.0, 'USDBTC': 2.0
|
||||
'BTCLBC': 1.0, 'USDLBC': 2.0
|
||||
})
|
||||
))
|
||||
await daemon.initialize()
|
||||
|
||||
Whitespace-only changes.
@@ -0,0 +1,290 @@
|
||||
import asyncio
|
||||
import binascii
|
||||
import os
|
||||
import logging
|
||||
import random
|
||||
from hashlib import sha1
|
||||
from tempfile import mkdtemp
|
||||
from typing import Optional
|
||||
|
||||
import libtorrent
|
||||
|
||||
|
||||
NOTIFICATION_MASKS = [
|
||||
"error",
|
||||
"peer",
|
||||
"port_mapping",
|
||||
"storage",
|
||||
"tracker",
|
||||
"debug",
|
||||
"status",
|
||||
"progress",
|
||||
"ip_block",
|
||||
"dht",
|
||||
"stats",
|
||||
"session_log",
|
||||
"torrent_log",
|
||||
"peer_log",
|
||||
"incoming_request",
|
||||
"dht_log",
|
||||
"dht_operation",
|
||||
"port_mapping_log",
|
||||
"picker_log",
|
||||
"file_progress",
|
||||
"piece_progress",
|
||||
"upload",
|
||||
"block_progress"
|
||||
]
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
DEFAULT_FLAGS = ( # fixme: somehow the logic here is inverted?
|
||||
libtorrent.add_torrent_params_flags_t.flag_auto_managed
|
||||
| libtorrent.add_torrent_params_flags_t.flag_update_subscribe
|
||||
)
|
||||
|
||||
|
||||
def get_notification_type(notification) -> str:
|
||||
for i, notification_type in enumerate(NOTIFICATION_MASKS):
|
||||
if (1 << i) & notification:
|
||||
return notification_type
|
||||
raise ValueError("unrecognized notification type")
|
||||
|
||||
|
||||
class TorrentHandle:
|
||||
def __init__(self, loop, executor, handle):
|
||||
self._loop = loop
|
||||
self._executor = executor
|
||||
self._handle: libtorrent.torrent_handle = handle
|
||||
self.started = asyncio.Event(loop=loop)
|
||||
self.finished = asyncio.Event(loop=loop)
|
||||
self.metadata_completed = asyncio.Event(loop=loop)
|
||||
self.size = 0
|
||||
self.total_wanted_done = 0
|
||||
self.name = ''
|
||||
self.tasks = []
|
||||
self.torrent_file: Optional[libtorrent.file_storage] = None
|
||||
self._base_path = None
|
||||
self._handle.set_sequential_download(1)
|
||||
|
||||
@property
|
||||
def largest_file(self) -> Optional[str]:
|
||||
if not self.torrent_file:
|
||||
return None
|
||||
index = self.largest_file_index
|
||||
return os.path.join(self._base_path, self.torrent_file.at(index).path)
|
||||
|
||||
@property
|
||||
def largest_file_index(self):
|
||||
largest_size, index = 0, 0
|
||||
for file_num in range(self.torrent_file.num_files()):
|
||||
if self.torrent_file.file_size(file_num) > largest_size:
|
||||
largest_size = self.torrent_file.file_size(file_num)
|
||||
index = file_num
|
||||
return index
|
||||
|
||||
def stop_tasks(self):
|
||||
while self.tasks:
|
||||
self.tasks.pop().cancel()
|
||||
|
||||
def _show_status(self):
|
||||
# fixme: cleanup
|
||||
if not self._handle.is_valid():
|
||||
return
|
||||
status = self._handle.status()
|
||||
if status.has_metadata:
|
||||
self.size = status.total_wanted
|
||||
self.total_wanted_done = status.total_wanted_done
|
||||
self.name = status.name
|
||||
if not self.metadata_completed.is_set():
|
||||
self.metadata_completed.set()
|
||||
log.info("Metadata completed for btih:%s - %s", status.info_hash, self.name)
|
||||
self.torrent_file = self._handle.get_torrent_info().files()
|
||||
self._base_path = status.save_path
|
||||
first_piece = self.torrent_file.at(self.largest_file_index).offset
|
||||
if not self.started.is_set():
|
||||
if self._handle.have_piece(first_piece):
|
||||
self.started.set()
|
||||
else:
|
||||
# prioritize it
|
||||
self._handle.set_piece_deadline(first_piece, 100)
|
||||
if not status.is_seeding:
|
||||
log.debug('%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d seeds: %d) %s - %s',
|
||||
status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000,
|
||||
status.num_peers, status.num_seeds, status.state, status.save_path)
|
||||
elif not self.finished.is_set():
|
||||
self.finished.set()
|
||||
log.info("Torrent finished: %s", self.name)
|
||||
|
||||
async def status_loop(self):
|
||||
while True:
|
||||
self._show_status()
|
||||
if self.finished.is_set():
|
||||
break
|
||||
await asyncio.sleep(0.1, loop=self._loop)
|
||||
|
||||
async def pause(self):
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, self._handle.pause
|
||||
)
|
||||
|
||||
async def resume(self):
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, lambda: self._handle.resume() # pylint: disable=unnecessary-lambda
|
||||
)
|
||||
|
||||
|
||||
class TorrentSession:
|
||||
def __init__(self, loop, executor):
|
||||
self._loop = loop
|
||||
self._executor = executor
|
||||
self._session: Optional[libtorrent.session] = None
|
||||
self._handles = {}
|
||||
self.tasks = []
|
||||
self.wait_start = True
|
||||
|
||||
async def add_fake_torrent(self):
|
||||
tmpdir = mkdtemp()
|
||||
info, btih = _create_fake_torrent(tmpdir)
|
||||
flags = libtorrent.add_torrent_params_flags_t.flag_seed_mode
|
||||
handle = self._session.add_torrent({
|
||||
'ti': info, 'save_path': tmpdir, 'flags': flags
|
||||
})
|
||||
self._handles[btih] = TorrentHandle(self._loop, self._executor, handle)
|
||||
return btih
|
||||
|
||||
async def bind(self, interface: str = '0.0.0.0', port: int = 10889):
|
||||
settings = {
|
||||
'listen_interfaces': f"{interface}:{port}",
|
||||
'enable_outgoing_utp': True,
|
||||
'enable_incoming_utp': True,
|
||||
'enable_outgoing_tcp': False,
|
||||
'enable_incoming_tcp': False
|
||||
}
|
||||
self._session = await self._loop.run_in_executor(
|
||||
self._executor, libtorrent.session, settings # pylint: disable=c-extension-no-member
|
||||
)
|
||||
self.tasks.append(self._loop.create_task(self.process_alerts()))
|
||||
|
||||
def stop(self):
|
||||
while self.tasks:
|
||||
self.tasks.pop().cancel()
|
||||
self._session.save_state()
|
||||
self._session.pause()
|
||||
self._session.stop_dht()
|
||||
self._session.stop_lsd()
|
||||
self._session.stop_natpmp()
|
||||
self._session.stop_upnp()
|
||||
self._session = None
|
||||
|
||||
def _pop_alerts(self):
|
||||
for alert in self._session.pop_alerts():
|
||||
log.info("torrent alert: %s", alert)
|
||||
|
||||
async def process_alerts(self):
|
||||
while True:
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, self._pop_alerts
|
||||
)
|
||||
await asyncio.sleep(1, loop=self._loop)
|
||||
|
||||
async def pause(self):
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, lambda: self._session.save_state() # pylint: disable=unnecessary-lambda
|
||||
)
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, lambda: self._session.pause() # pylint: disable=unnecessary-lambda
|
||||
)
|
||||
|
||||
async def resume(self):
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, self._session.resume
|
||||
)
|
||||
|
||||
def _add_torrent(self, btih: str, download_directory: Optional[str]):
|
||||
params = {'info_hash': binascii.unhexlify(btih.encode()), 'flags': DEFAULT_FLAGS}
|
||||
if download_directory:
|
||||
params['save_path'] = download_directory
|
||||
handle = self._session.add_torrent(params)
|
||||
handle.force_dht_announce()
|
||||
self._handles[btih] = TorrentHandle(self._loop, self._executor, handle)
|
||||
|
||||
def full_path(self, btih):
|
||||
return self._handles[btih].largest_file
|
||||
|
||||
async def add_torrent(self, btih, download_path):
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, self._add_torrent, btih, download_path
|
||||
)
|
||||
self._handles[btih].tasks.append(self._loop.create_task(self._handles[btih].status_loop()))
|
||||
await self._handles[btih].metadata_completed.wait()
|
||||
if self.wait_start:
|
||||
# fixme: temporary until we add streaming support, otherwise playback fails!
|
||||
await self._handles[btih].started.wait()
|
||||
|
||||
def remove_torrent(self, btih, remove_files=False):
|
||||
if btih in self._handles:
|
||||
handle = self._handles[btih]
|
||||
handle.stop_tasks()
|
||||
self._session.remove_torrent(handle._handle, 1 if remove_files else 0)
|
||||
self._handles.pop(btih)
|
||||
|
||||
async def save_file(self, btih, download_directory):
|
||||
handle = self._handles[btih]
|
||||
await handle.resume()
|
||||
|
||||
def get_size(self, btih):
|
||||
return self._handles[btih].size
|
||||
|
||||
def get_name(self, btih):
|
||||
return self._handles[btih].name
|
||||
|
||||
def get_downloaded(self, btih):
|
||||
return self._handles[btih].total_wanted_done
|
||||
|
||||
def is_completed(self, btih):
|
||||
return self._handles[btih].finished.is_set()
|
||||
|
||||
|
||||
def get_magnet_uri(btih):
|
||||
return f"magnet:?xt=urn:btih:{btih}"
|
||||
|
||||
|
||||
def _create_fake_torrent(tmpdir):
|
||||
# beware, that's just for testing
|
||||
path = os.path.join(tmpdir, 'tmp')
|
||||
with open(path, 'wb') as myfile:
|
||||
size = myfile.write(bytes([random.randint(0, 255) for _ in range(40)]) * 1024)
|
||||
file_storage = libtorrent.file_storage()
|
||||
file_storage.add_file('tmp', size)
|
||||
t = libtorrent.create_torrent(file_storage, 0, 4 * 1024 * 1024)
|
||||
libtorrent.set_piece_hashes(t, tmpdir)
|
||||
info = libtorrent.torrent_info(t.generate())
|
||||
btih = sha1(info.metadata()).hexdigest()
|
||||
return info, btih
|
||||
|
||||
|
||||
async def main():
|
||||
if os.path.exists("~/Downloads/ubuntu-18.04.3-live-server-amd64.torrent"):
|
||||
os.remove("~/Downloads/ubuntu-18.04.3-live-server-amd64.torrent")
|
||||
if os.path.exists("~/Downloads/ubuntu-18.04.3-live-server-amd64.iso"):
|
||||
os.remove("~/Downloads/ubuntu-18.04.3-live-server-amd64.iso")
|
||||
|
||||
btih = "dd8255ecdc7ca55fb0bbf81323d87062db1f6d1c"
|
||||
|
||||
executor = None
|
||||
session = TorrentSession(asyncio.get_event_loop(), executor)
|
||||
session2 = TorrentSession(asyncio.get_event_loop(), executor)
|
||||
await session.bind('localhost', port=4040)
|
||||
await session2.bind('localhost', port=4041)
|
||||
btih = await session.add_fake_torrent()
|
||||
session2._session.add_dht_node(('localhost', 4040))
|
||||
await session2.add_torrent(btih, "/tmp/down")
|
||||
while True:
|
||||
await asyncio.sleep(100)
|
||||
await session.pause()
|
||||
executor.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,72 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import typing
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TorrentInfo:
|
||||
__slots__ = ('dht_seeds', 'http_seeds', 'trackers', 'total_size')
|
||||
|
||||
def __init__(self, dht_seeds: typing.Tuple[typing.Tuple[str, int]],
|
||||
http_seeds: typing.Tuple[typing.Dict[str, typing.Any]],
|
||||
trackers: typing.Tuple[typing.Tuple[str, int]], total_size: int):
|
||||
self.dht_seeds = dht_seeds
|
||||
self.http_seeds = http_seeds
|
||||
self.trackers = trackers
|
||||
self.total_size = total_size
|
||||
|
||||
@classmethod
|
||||
def from_libtorrent_info(cls, torrent_info):
|
||||
return cls(
|
||||
torrent_info.nodes(), tuple(
|
||||
{
|
||||
'url': web_seed['url'],
|
||||
'type': web_seed['type'],
|
||||
'auth': web_seed['auth']
|
||||
} for web_seed in torrent_info.web_seeds()
|
||||
), tuple(
|
||||
(tracker.url, tracker.tier) for tracker in torrent_info.trackers()
|
||||
), torrent_info.total_size()
|
||||
)
|
||||
|
||||
|
||||
class Torrent:
|
||||
def __init__(self, loop, handle):
|
||||
self._loop = loop
|
||||
self._handle = handle
|
||||
self.finished = asyncio.Event(loop=loop)
|
||||
|
||||
def _threaded_update_status(self):
|
||||
status = self._handle.status()
|
||||
if not status.is_seeding:
|
||||
log.info(
|
||||
'%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d) %s',
|
||||
status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000,
|
||||
status.num_peers, status.state
|
||||
)
|
||||
elif not self.finished.is_set():
|
||||
self.finished.set()
|
||||
|
||||
async def wait_for_finished(self):
|
||||
while True:
|
||||
await self._loop.run_in_executor(
|
||||
None, self._threaded_update_status
|
||||
)
|
||||
if self.finished.is_set():
|
||||
log.info("finished downloading torrent!")
|
||||
await self.pause()
|
||||
break
|
||||
await asyncio.sleep(1, loop=self._loop)
|
||||
|
||||
async def pause(self):
|
||||
log.info("pause torrent")
|
||||
await self._loop.run_in_executor(
|
||||
None, self._handle.pause
|
||||
)
|
||||
|
||||
async def resume(self):
|
||||
await self._loop.run_in_executor(
|
||||
None, self._handle.resume
|
||||
)
|
||||
@@ -0,0 +1,140 @@
|
||||
import asyncio
|
||||
import binascii
|
||||
import logging
|
||||
import os
|
||||
import typing
|
||||
from typing import Optional
|
||||
from aiohttp.web import Request
|
||||
from lbry.file.source_manager import SourceManager
|
||||
from lbry.file.source import ManagedDownloadSource
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.torrent.session import TorrentSession
|
||||
from lbry.conf import Config
|
||||
from lbry.wallet.transaction import Transaction
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim
|
||||
from lbry.extras.daemon.storage import StoredContentClaim
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def path_or_none(encoded_path) -> Optional[str]:
|
||||
if not encoded_path:
|
||||
return
|
||||
return binascii.unhexlify(encoded_path).decode()
|
||||
|
||||
|
||||
class TorrentSource(ManagedDownloadSource):
|
||||
STATUS_STOPPED = "stopped"
|
||||
filter_fields = SourceManager.filter_fields
|
||||
filter_fields.update({
|
||||
'bt_infohash'
|
||||
})
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage', identifier: str,
|
||||
file_name: Optional[str] = None, download_directory: Optional[str] = None,
|
||||
status: Optional[str] = STATUS_STOPPED, claim: Optional['StoredContentClaim'] = None,
|
||||
download_id: Optional[str] = None, rowid: Optional[int] = None,
|
||||
content_fee: Optional['Transaction'] = None,
|
||||
analytics_manager: Optional['AnalyticsManager'] = None,
|
||||
added_on: Optional[int] = None, torrent_session: Optional['TorrentSession'] = None):
|
||||
super().__init__(loop, config, storage, identifier, file_name, download_directory, status, claim, download_id,
|
||||
rowid, content_fee, analytics_manager, added_on)
|
||||
self.torrent_session = torrent_session
|
||||
|
||||
@property
|
||||
def full_path(self) -> Optional[str]:
|
||||
full_path = self.torrent_session.full_path(self.identifier)
|
||||
self.download_directory = os.path.dirname(full_path)
|
||||
return full_path
|
||||
|
||||
async def start(self, timeout: Optional[float] = None, save_now: Optional[bool] = False):
|
||||
await self.torrent_session.add_torrent(self.identifier, self.download_directory)
|
||||
|
||||
async def stop(self, finished: bool = False):
|
||||
await self.torrent_session.remove_torrent(self.identifier)
|
||||
|
||||
async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
|
||||
await self.torrent_session.save_file(self.identifier, download_directory)
|
||||
|
||||
@property
|
||||
def torrent_length(self):
|
||||
return self.torrent_session.get_size(self.identifier)
|
||||
|
||||
@property
|
||||
def written_bytes(self):
|
||||
return self.torrent_session.get_downloaded(self.identifier)
|
||||
|
||||
@property
|
||||
def torrent_name(self):
|
||||
return self.torrent_session.get_name(self.identifier)
|
||||
|
||||
@property
|
||||
def bt_infohash(self):
|
||||
return self.identifier
|
||||
|
||||
def stop_tasks(self):
|
||||
pass
|
||||
|
||||
@property
|
||||
def completed(self):
|
||||
return self.torrent_session.is_completed(self.identifier)
|
||||
|
||||
|
||||
class TorrentManager(SourceManager):
|
||||
_sources: typing.Dict[str, ManagedDownloadSource]
|
||||
|
||||
filter_fields = set(SourceManager.filter_fields)
|
||||
filter_fields.update({
|
||||
'bt_infohash',
|
||||
'blobs_remaining', # TODO: here they call them "parts", but its pretty much the same concept
|
||||
'blobs_in_stream'
|
||||
})
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', torrent_session: 'TorrentSession',
|
||||
storage: 'SQLiteStorage', analytics_manager: Optional['AnalyticsManager'] = None):
|
||||
super().__init__(loop, config, storage, analytics_manager)
|
||||
self.torrent_session: 'TorrentSession' = torrent_session
|
||||
|
||||
async def recover_streams(self, file_infos: typing.List[typing.Dict]):
|
||||
raise NotImplementedError
|
||||
|
||||
async def _load_stream(self, rowid: int, bt_infohash: str, file_name: Optional[str],
|
||||
download_directory: Optional[str], status: str,
|
||||
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
|
||||
added_on: Optional[int]):
|
||||
stream = TorrentSource(
|
||||
self.loop, self.config, self.storage, identifier=bt_infohash, file_name=file_name,
|
||||
download_directory=download_directory, status=status, claim=claim, rowid=rowid,
|
||||
content_fee=content_fee, analytics_manager=self.analytics_manager, added_on=added_on,
|
||||
torrent_session=self.torrent_session
|
||||
)
|
||||
self.add(stream)
|
||||
|
||||
async def initialize_from_database(self):
|
||||
pass
|
||||
|
||||
async def start(self):
|
||||
await super().start()
|
||||
|
||||
def stop(self):
|
||||
super().stop()
|
||||
log.info("finished stopping the torrent manager")
|
||||
|
||||
async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
|
||||
await super().delete(source, delete_file)
|
||||
self.torrent_session.remove_torrent(source.identifier, delete_file)
|
||||
|
||||
async def create(self, file_path: str, key: Optional[bytes] = None,
|
||||
iv_generator: Optional[typing.Generator[bytes, None, None]] = None):
|
||||
raise NotImplementedError
|
||||
|
||||
async def _delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
|
||||
raise NotImplementedError
|
||||
# blob_hashes = [source.sd_hash] + [b.blob_hash for b in source.descriptor.blobs[:-1]]
|
||||
# await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False)
|
||||
# await self.storage.delete_stream(source.descriptor)
|
||||
|
||||
async def stream_partial_content(self, request: Request, sd_hash: str):
|
||||
raise NotImplementedError
|
||||
+209
-11
@@ -3,6 +3,7 @@ import codecs
|
||||
import datetime
|
||||
import random
|
||||
import socket
|
||||
import time
|
||||
import string
|
||||
import sys
|
||||
import json
|
||||
@@ -19,8 +20,11 @@ import pkg_resources
|
||||
|
||||
import certifi
|
||||
import aiohttp
|
||||
from prometheus_client import Counter
|
||||
from prometheus_client.registry import REGISTRY
|
||||
from lbry.schema.claim import Claim
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -188,6 +192,8 @@ def cache_concurrent(async_fn):
|
||||
async def resolve_host(url: str, port: int, proto: str) -> str:
|
||||
if proto not in ['udp', 'tcp']:
|
||||
raise Exception("invalid protocol")
|
||||
if url.lower() == 'localhost':
|
||||
return '127.0.0.1'
|
||||
try:
|
||||
if ipaddress.ip_address(url):
|
||||
return url
|
||||
@@ -202,18 +208,44 @@ async def resolve_host(url: str, port: int, proto: str) -> str:
|
||||
))[0][4][0]
|
||||
|
||||
|
||||
class LRUCache:
|
||||
class LRUCacheWithMetrics:
|
||||
__slots__ = [
|
||||
'capacity',
|
||||
'cache'
|
||||
'cache',
|
||||
'_track_metrics',
|
||||
'hits',
|
||||
'misses'
|
||||
]
|
||||
|
||||
def __init__(self, capacity):
|
||||
def __init__(self, capacity: int, metric_name: typing.Optional[str] = None, namespace: str = "daemon_cache"):
|
||||
self.capacity = capacity
|
||||
self.cache = collections.OrderedDict()
|
||||
if metric_name is None:
|
||||
self._track_metrics = False
|
||||
self.hits = self.misses = None
|
||||
else:
|
||||
self._track_metrics = True
|
||||
try:
|
||||
self.hits = Counter(
|
||||
f"{metric_name}_cache_hit_count", "Number of cache hits", namespace=namespace
|
||||
)
|
||||
self.misses = Counter(
|
||||
f"{metric_name}_cache_miss_count", "Number of cache misses", namespace=namespace
|
||||
)
|
||||
except ValueError as err:
|
||||
log.debug("failed to set up prometheus %s_cache_miss_count metric: %s", metric_name, err)
|
||||
self._track_metrics = False
|
||||
self.hits = self.misses = None
|
||||
|
||||
def get(self, key):
|
||||
value = self.cache.pop(key)
|
||||
def get(self, key, default=None):
|
||||
try:
|
||||
value = self.cache.pop(key)
|
||||
if self._track_metrics:
|
||||
self.hits.inc()
|
||||
except KeyError:
|
||||
if self._track_metrics:
|
||||
self.misses.inc()
|
||||
return default
|
||||
self.cache[key] = value
|
||||
return value
|
||||
|
||||
@@ -225,16 +257,94 @@ class LRUCache:
|
||||
self.cache.popitem(last=False)
|
||||
self.cache[key] = value
|
||||
|
||||
def clear(self):
|
||||
self.cache.clear()
|
||||
|
||||
def pop(self, key):
|
||||
return self.cache.pop(key)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
return self.set(key, value)
|
||||
|
||||
def __getitem__(self, item):
|
||||
return self.get(item)
|
||||
|
||||
def __contains__(self, item) -> bool:
|
||||
return item in self.cache
|
||||
|
||||
def __len__(self):
|
||||
return len(self.cache)
|
||||
|
||||
def __delitem__(self, key):
|
||||
self.cache.pop(key)
|
||||
|
||||
def __del__(self):
|
||||
self.clear()
|
||||
if self._track_metrics: # needed for tests
|
||||
try:
|
||||
REGISTRY.unregister(self.hits)
|
||||
REGISTRY.unregister(self.misses)
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
|
||||
class LRUCache:
|
||||
__slots__ = [
|
||||
'capacity',
|
||||
'cache'
|
||||
]
|
||||
|
||||
def __init__(self, capacity: int):
|
||||
self.capacity = capacity
|
||||
self.cache = collections.OrderedDict()
|
||||
|
||||
def get(self, key, default=None):
|
||||
try:
|
||||
value = self.cache.pop(key)
|
||||
except KeyError:
|
||||
return default
|
||||
self.cache[key] = value
|
||||
return value
|
||||
|
||||
def set(self, key, value):
|
||||
try:
|
||||
self.cache.pop(key)
|
||||
except KeyError:
|
||||
if len(self.cache) >= self.capacity:
|
||||
self.cache.popitem(last=False)
|
||||
self.cache[key] = value
|
||||
|
||||
def clear(self):
|
||||
self.cache.clear()
|
||||
|
||||
def pop(self, key):
|
||||
return self.cache.pop(key)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
return self.set(key, value)
|
||||
|
||||
def __getitem__(self, item):
|
||||
return self.get(item)
|
||||
|
||||
def __contains__(self, item) -> bool:
|
||||
return item in self.cache
|
||||
|
||||
def __len__(self):
|
||||
return len(self.cache)
|
||||
|
||||
def __delitem__(self, key):
|
||||
self.cache.pop(key)
|
||||
|
||||
def __del__(self):
|
||||
self.clear()
|
||||
|
||||
|
||||
def lru_cache_concurrent(cache_size: typing.Optional[int] = None,
|
||||
override_lru_cache: typing.Optional[LRUCache] = None):
|
||||
override_lru_cache: typing.Optional[LRUCacheWithMetrics] = None):
|
||||
if not cache_size and override_lru_cache is None:
|
||||
raise ValueError("invalid cache size")
|
||||
concurrent_cache = {}
|
||||
lru_cache = override_lru_cache or LRUCache(cache_size)
|
||||
lru_cache = override_lru_cache if override_lru_cache is not None else LRUCacheWithMetrics(cache_size)
|
||||
|
||||
def wrapper(async_fn):
|
||||
|
||||
@@ -265,20 +375,108 @@ def get_ssl_context() -> ssl.SSLContext:
|
||||
@contextlib.asynccontextmanager
|
||||
async def aiohttp_request(method, url, **kwargs) -> typing.AsyncContextManager[aiohttp.ClientResponse]:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.request(method, url, ssl=get_ssl_context(), **kwargs) as response:
|
||||
async with session.request(method, url, **kwargs) as response:
|
||||
yield response
|
||||
|
||||
|
||||
async def get_external_ip() -> typing.Optional[str]: # used if upnp is disabled or non-functioning
|
||||
# the ipaddress module does not show these subnets as reserved
|
||||
CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
|
||||
IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
|
||||
|
||||
|
||||
def is_valid_public_ipv4(address, allow_localhost: bool = False):
|
||||
try:
|
||||
parsed_ip = ipaddress.ip_address(address)
|
||||
if parsed_ip.is_loopback and allow_localhost:
|
||||
return True
|
||||
if any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local, parsed_ip.is_loopback,
|
||||
parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private, parsed_ip.is_reserved)):
|
||||
return False
|
||||
else:
|
||||
return not any((CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
|
||||
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
|
||||
except (ipaddress.AddressValueError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
async def fallback_get_external_ip(): # used if spv servers can't be used for ip detection
|
||||
try:
|
||||
async with aiohttp_request("get", "https://api.lbry.com/ip") as resp:
|
||||
response = await resp.json()
|
||||
if response['success']:
|
||||
return response['data']['ip']
|
||||
return response['data']['ip'], None
|
||||
except Exception:
|
||||
return
|
||||
return None, None
|
||||
|
||||
|
||||
async def _get_external_ip(default_servers) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]:
|
||||
# used if upnp is disabled or non-functioning
|
||||
from lbry.wallet.server.udp import SPVStatusClientProtocol # pylint: disable=C0415
|
||||
|
||||
hostname_to_ip = {}
|
||||
ip_to_hostnames = collections.defaultdict(list)
|
||||
|
||||
async def resolve_spv(server, port):
|
||||
try:
|
||||
server_addr = await resolve_host(server, port, 'udp')
|
||||
hostname_to_ip[server] = (server_addr, port)
|
||||
ip_to_hostnames[(server_addr, port)].append(server)
|
||||
except Exception:
|
||||
log.exception("error looking up dns for spv servers")
|
||||
|
||||
# accumulate the dns results
|
||||
await asyncio.gather(*(resolve_spv(server, port) for (server, port) in default_servers))
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
pong_responses = asyncio.Queue()
|
||||
connection = SPVStatusClientProtocol(pong_responses)
|
||||
try:
|
||||
await loop.create_datagram_endpoint(lambda: connection, ('0.0.0.0', 0))
|
||||
# could raise OSError if it cant bind
|
||||
randomized_servers = list(ip_to_hostnames.keys())
|
||||
random.shuffle(randomized_servers)
|
||||
for server in randomized_servers:
|
||||
connection.ping(server)
|
||||
try:
|
||||
_, pong = await asyncio.wait_for(pong_responses.get(), 1)
|
||||
if is_valid_public_ipv4(pong.ip_address):
|
||||
return pong.ip_address, ip_to_hostnames[server][0]
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
return None, None
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
async def get_external_ip(default_servers) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]:
|
||||
ip_from_spv_servers = await _get_external_ip(default_servers)
|
||||
if not ip_from_spv_servers[1]:
|
||||
return await fallback_get_external_ip()
|
||||
return ip_from_spv_servers
|
||||
|
||||
|
||||
def is_running_from_bundle():
|
||||
# see https://pyinstaller.readthedocs.io/en/stable/runtime-information.html
|
||||
return getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS')
|
||||
|
||||
|
||||
class LockWithMetrics(asyncio.Lock):
|
||||
def __init__(self, acquire_metric, held_time_metric, loop=None):
|
||||
super().__init__(loop=loop)
|
||||
self._acquire_metric = acquire_metric
|
||||
self._lock_held_time_metric = held_time_metric
|
||||
self._lock_acquired_time = None
|
||||
|
||||
async def acquire(self):
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
return await super().acquire()
|
||||
finally:
|
||||
self._lock_acquired_time = time.perf_counter()
|
||||
self._acquire_metric.observe(self._lock_acquired_time - start)
|
||||
|
||||
def release(self):
|
||||
try:
|
||||
return super().release()
|
||||
finally:
|
||||
self._lock_held_time_metric.observe(time.perf_counter() - self._lock_acquired_time)
|
||||
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
|
||||
__node_cli__ = 'lbrycrd-cli'
|
||||
__node_bin__ = ''
|
||||
__node_url__ = (
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.5/lbrycrd-linux-1745.zip'
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.6/lbrycrd-linux-1746.zip'
|
||||
)
|
||||
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
|
||||
|
||||
|
||||
@@ -311,7 +311,7 @@ class Account:
|
||||
private_key=private_key,
|
||||
public_key=public_key,
|
||||
address_generator=d.get('address_generator', {}),
|
||||
modified_on=d.get('modified_on', time.time()),
|
||||
modified_on=int(d.get('modified_on', time.time())),
|
||||
channel_keys=d.get('certificates', {})
|
||||
)
|
||||
|
||||
@@ -343,7 +343,7 @@ class Account:
|
||||
def merge(self, d: dict):
|
||||
if d.get('modified_on', 0) > self.modified_on:
|
||||
self.name = d['name']
|
||||
self.modified_on = d.get('modified_on', time.time())
|
||||
self.modified_on = int(d.get('modified_on', time.time()))
|
||||
assert self.address_generator.name == d['address_generator']['name']
|
||||
for chain_name in ('change', 'receiving'):
|
||||
if chain_name in d['address_generator']:
|
||||
@@ -525,11 +525,13 @@ class Account:
|
||||
channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes)
|
||||
self.channel_keys[channel_pubkey_hash] = private_key.to_pem().decode()
|
||||
|
||||
def get_channel_private_key(self, public_key_bytes):
|
||||
async def get_channel_private_key(self, public_key_bytes):
|
||||
channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes)
|
||||
private_key_pem = self.channel_keys.get(channel_pubkey_hash)
|
||||
if private_key_pem:
|
||||
return ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=sha256)
|
||||
return await asyncio.get_event_loop().run_in_executor(
|
||||
None, ecdsa.SigningKey.from_pem, private_key_pem, sha256
|
||||
)
|
||||
|
||||
async def maybe_migrate_certificates(self):
|
||||
def to_der(private_key_pem):
|
||||
|
||||
@@ -740,4 +740,145 @@ HASHES = {
|
||||
738000: 'aebdf15b23eb7a37600f67d45bf6586b1d5bff3d5f3459adc2f6211ab3dd0bcb',
|
||||
739000: '3f5a894ac42f95f7d54ce25c42ea0baf1a05b2da0e9406978de0dc53484d8b04',
|
||||
740000: '55debc22f995d844eafa0a90296c9f4f433e2b7f38456fff45dd3c66cef04e37',
|
||||
741000: '927b47fc909b4b55c067bbd75d8638af1400fac076cb642e9500a747d849e458',
|
||||
742000: '97fa3d83eb94114496e418c118f549ebfb8f6d123d0b40a12ecb093239557646',
|
||||
743000: '482b66d8d5084703079c28e3ae69e5dee735f762d6fcf9743e75f04e139fd181',
|
||||
744000: 'f406890d5c70808a58fb14429bad812a3185bdb9dace1aa57de76663f92b5013',
|
||||
745000: '2bd0802cbb8aa4441a159104d39515a4ff6fc8dfe616bc83e88197847c78bcff',
|
||||
746000: '24d090a7b6359db3d5d714a69ddc9a6f2e8ff8f044b723220a8ba32df785fd54',
|
||||
747000: '07c4ce9ce5310ee472cf753ddb03c39c5fee6c910d491daffd38615205411633',
|
||||
748000: 'ea913798c0f09d0a27eae7c852954c2c88b8c3b7f23f8fba26b68a3952d0ffde',
|
||||
749000: '23f256adebfe35d49ba84ad49f3f71fc67f7745091c91f22e65f1cc2e23b8f2c',
|
||||
750000: '96db12ee3a295f3d5c56d244e6e7493f58c08d3427e379940e5d4f891a41ec26',
|
||||
751000: 'cedaf12415dac1314942e58ced80830b92fbfabc41f42a0b0f054f0672ef9822',
|
||||
752000: '293606bcd9fbbee5584724301b2cf86bb69204820023e1fb46c238ddfbc660ab',
|
||||
753000: 'f4d43cbb38b7d97919dedc0f5a6dc8007896c4f443b76f3e5693e25bc46760cf',
|
||||
754000: 'fcaad22fd815311280fe451086516375d1d9d92b2990c7c351407df5aa19011e',
|
||||
755000: 'b9276f10d1844cb5b0308766c8db960490ac34a73c4653d0a91202789a6ccb9b',
|
||||
756000: '2fe5581f1110c1c8dcea46cad647551bd6bd640cb37738d863e189bd8f368347',
|
||||
757000: 'b9d915f366f0b010429a52245b0fb02774157eb9fd8f66bce32dcd3acc71c2a1',
|
||||
758000: '62d1854fc15db56b5d0e05ceeb54c1297966bf9dc7f7a0a14b42c059fc485d1b',
|
||||
759000: 'f4ca9f69d16d092f4a0ea5102e6343b21204c4ea9cd9b22cddd77dbb5d68ade3',
|
||||
760000: 'df3bb86641330d8cc7f55a2fd0da28251219e95babe960a308b18e08a7d88fc8',
|
||||
761000: 'a93029475de4bc7569b6ae802d658cd91c84cc253772712a279f140a6c3b91b1',
|
||||
762000: '307e289dc6ec8bcd62ca8831e4159d5edd780f2fae55ba55dd446225450f46f8',
|
||||
763000: '293f73514abca24f374473bd0394179812952a04ea13dc60ef5ada5331fa274f',
|
||||
764000: 'dd8b082db9281e3d9bacf15d6b352fda186d2d2923c7731844d0d4764dd71db8',
|
||||
765000: '201239e562d2571bf47347b3522fff89632aecea3b2d8cef05151f88b2b0bcdb',
|
||||
766000: '4a55a538b51b5650979e64521998cd5c5ad055ba9f3ac0e3e2a28febc6cc2798',
|
||||
767000: '3916666f2adbb05ea98ec1961f9546b9afa0f6910ec95e42ce37267f2ae4f79c',
|
||||
768000: 'dc0ad881eedcb5fd4954238f462080d6e7636b058d481698ed1c077e0ce2207e',
|
||||
769000: 'eaf10a1e1ec6e129289b8479a05df03e0808f1f0946f1995de6524e9ebe7a461',
|
||||
770000: '7200c64f22e32de7f999583361c933680fc9a2ffcb9a5ab73d3076fd49ec7537',
|
||||
771000: 'd883111a2eeacff80ce31df35ab6c943805b9e48877b413fccf371e5dbfa7fb2',
|
||||
772000: '3977d3c60edb9c80c97bb2b759b1659cbb650ad2d3a6f61d2caec83f1b2ae84c',
|
||||
773000: '9c7175fb8646a1a82383b4c534fd01bcf92d65c43d87ae854d51a784b04dc77e',
|
||||
774000: 'e0e92485f86e5fffa87b3497424e43b02a37710517d9d3f272392e8cdc56e5e9',
|
||||
775000: '6395229113d3aa2105afbaeb8b59621a536fc61fe272314b2fc3bdda98dd66cc',
|
||||
776000: 'b4b00207328b5f032bd4f0b634f91323ff520ada8c8bfec241b23c8e4bfd5a4e',
|
||||
777000: '14cdc6f5f7b4bd5bad745dfe6fcd114e9194026412a2e1b3f345be2eef433d16',
|
||||
778000: 'd3cd7b68be504c32117b670d38d59d44b02dcf3d65811efc2ca5531d902623cc',
|
||||
779000: 'afcd220e4040cb5f92d4b38fc204e59822df2218f767f2c4b33597b238a35f77',
|
||||
780000: '78252a9cfc289a70192ed8dd3dddeb1b9a4f9b8eff9a5d0ac259b3254472cf68',
|
||||
781000: '02ebc3f17d947481a311b4771c254f1e002b6a9198d4a5258ce6c13165aadddc',
|
||||
782000: '8dd9f1f372ee6d688a0bcdc3b342c77804ba5a646a218be4bc2aa02d846206c0',
|
||||
783000: 'e46b0d02ec2ef488fae455665e107520e1bd2b4f35ca52af7ad8addd2f72fa73',
|
||||
784000: '9ee8a8de94231e3ae3a610b82fdbca48dc14d9b80791d20af6c365a31822df6f',
|
||||
785000: '21e1cc12def8173a50158b2833bd91a62140c61646f5e08aecaee3e6da20735e',
|
||||
786000: 'b3e659f84d73de42888cc0f2b69bae71dd5fa6756a437a4b21958b182faa316e',
|
||||
787000: 'a9be7ba00ea6a9ea6bd03d8412ec014ca7e8cda6bdc33382f165e702811b8836',
|
||||
788000: 'a4c14729f8a68c03f5a0ccd890ac6a92b39c143f1f752fe81ad051eb52d8dce0',
|
||||
789000: '5cf66d224e5645097efc9c3c0392b51c8ca8ea1295151921a7912a2f04ee1274',
|
||||
790000: '676769ade71c33bc102bce416e66eb2c6794b03d7b8f5a590c87c380da463775',
|
||||
791000: '0228e074451797bf6bfbc941bcafcbadc972d32e4e1e0c5da015513f65714217',
|
||||
792000: '0fa3d00a1f19c5ac060e10a410cf7cea18eac5f89018d79ce51ac3fc66bbb365',
|
||||
793000: '5f68d0868b424e32f5ce3d8e7d9f18979da7b831b8ef4e3974d62fb20ff53a97',
|
||||
794000: '34508c56423739c00a837801b654b07decb274d02b383eff396d23c4d64bc0e9',
|
||||
795000: '7f70910c855d1fd88cd7f9be8a3b94314ee408a31a2da6301404bf8deb07c12c',
|
||||
796000: 'b74ab8813b1d2a0967fea0e66597572e5f0b5a285e21f5150fcc9d5f757de130',
|
||||
797000: 'bba27b1491d907ab1baa456cb651dc5b071231b1b6ad27b62d351ca12c25dbfd',
|
||||
798000: 'e75dcb15b2fc91f02e75e600dde9f6f46c09672533bc82a5d6916c4a2cd8613a',
|
||||
799000: 'adf62c826a3e0b33af439a7881918ae4ce19c5fb2ca37d21243415f7d716aa65',
|
||||
800000: 'd8f0ca13a8c8a19c254a3a6ba15150a34711dca96f2d877162cc44aa2acfb268',
|
||||
801000: '2a8c7104c4040a2bc31913ae25e9361df5bac9477368c708f86c1ca640480887',
|
||||
802000: '1f3b09d3561c4a8a056b263289bd492dc6c0d604c3fa195935e735d1c0ddc40e',
|
||||
803000: '037769628c40a701fdb4b16d79084b8fbb319fde79770a7ac842f3cdc813099e',
|
||||
804000: 'a0c6a089e5fa1e3589ca282085fe7201a5705776d81b257ffd252b2947fa6428',
|
||||
805000: 'b2ac99bfc4a488e7b7624b31ee061991a6dd0881bb005cd13f3dd2e66a08fe19',
|
||||
806000: 'ffe63cb999a278280b80a667d2dcb60c40e43a53f733914d8bec808b694ebf83',
|
||||
807000: 'eddb09fc6c4869a59b520d0befb1fb6ac952333f3cc5de086539c85ea8558778',
|
||||
808000: '0f4fb3f9172e52897ea992d9f3a2024126c4d2e63e9888739f11fb1f5e4c1f46',
|
||||
809000: '9641dd720d23ced2f1cb6e5cf46ac4e547afb9f56263c4cf58e3b19d407cf401',
|
||||
810000: 'de6dc953acd7e5ef213b3aaf1c4a9ee1d5b756bfce5525ee105214647e243a85',
|
||||
811000: 'c52c83712ca12b24b2db1b4a575e7f352b1d560cbf702e121a03bdca9e8be23d',
|
||||
812000: '83143734bb965318a53a38a7e403dcdb3e3fadedb01ab12c370417fc2a0655c0',
|
||||
813000: 'e480deff10c5a84fc957e3aed936690e24b74dd08fa8858a8a953c2f7383b914',
|
||||
814000: '810d33afcee07b9abe16c6cdc3a041038daa131c476b0daf48a080007f08b490',
|
||||
815000: 'b4aeb9e16fddd27844b2d56bc2b221134039bb5642c9e9ba88372afbdeac3972',
|
||||
816000: '86e73b67aae3d248011b8f66ed414cb8a9ba4b2a3cf7e32773cfbff055d719b7',
|
||||
817000: '3ebb8b83752b48242016cb682f0f6bd14e15371bf1163a5933193eaa0edeb351',
|
||||
818000: '4d925e17f642f220bbf317d3d5355d2f41fbce325f190f8c3b32dc0b337d24d6',
|
||||
819000: 'b9cc126d620f6b99d90a00d35957b0e428aaaa7c986bc9e816a60e4334572961',
|
||||
820000: '9c2f8c142bed1f94dca29276f7c83958be8cfe11773bb9b56c808fbcf7d3b1f8',
|
||||
821000: 'e5509eb98895cfa12a8da5d54c1df3f52472ffcbdf707adbf84a4a9c5d356203',
|
||||
822000: '764aada4802ebfe4ef935ab50af06a4f83aa556c49fdde3d9e12e1abd230c16b',
|
||||
823000: '1dbd745c2e96a365d865f990d109137d32d42977f503af55d8c00b109d31d3c3',
|
||||
824000: '954304a0b0c8f549c3bffd5ff46b5b8f05b0f0fde2a36f24fd5af9d774fb3079',
|
||||
825000: '17808b14f2056c1a5d46cb7617e9de9be6a1a6084edbc1bdb778586467a72297',
|
||||
826000: '3ca1167d4cac8b187829b23001b438617c43704b42462c4eb001b0d434cb9651',
|
||||
827000: '246d1607245e4a202f420393ac2e30e9cbf5eb5570dc997073b897f6d8643023',
|
||||
828000: '1764730a8dc3e89d02d168ff6bb54e8c903820b74711af6ff27bd0c8545577e7',
|
||||
829000: 'd9f3ab0cd823c6305bd8b95a96188bb4f2ca90b4d66c5d12293e8b6192bac0f2',
|
||||
830000: 'd4ff51f0092b04aedf8d39937680d8e8309b1be21d36e7833ed36f8e30aad6ea',
|
||||
831000: '3e92e76721b962396dce52993fa7606552f0907b38f7b2bd7b21ada98c145f47',
|
||||
832000: 'df12fcdb4cbe53ba627ace6de898298de175f8671d3d90170732d110fcdc34b8',
|
||||
833000: '25167ff38ae4a5964b618cabe0a12d4de62ac7a4c47448cdb4499e09e108d5b9',
|
||||
834000: 'd31f5309ea179a1e386e835fc372e47dcda6871a3a239abfba50c4f368994f13',
|
||||
835000: 'aff7e8dd3e55ea807fcbe284014075f420b3a23f1b0eb47bacdc1c91d2899813',
|
||||
836000: '3b5ac6d64c470739bb17d1544a285affb40f2d33e92687e5ba7c5ac602e0d72a',
|
||||
837000: 'd5619cbfe4f27c55f2bf9351b4891636cf64fef88212a5eeeae7bd3de47fe0bd',
|
||||
838000: '1f9102a49c6ac470cb5d0050e5300b1443840d6d65719b835e3bea484aafb2ec',
|
||||
839000: '3f63e391f0fbc5787fbe4ace3bada3816261294ea1c6ee435001801023682f90',
|
||||
840000: '777894fd12bd0d6dee7bcde2995c68e55e7094e3122da38571e4b6c4304b75e0',
|
||||
841000: 'ceb0c598c788e25e43e25aa4beff5c7377035824844cf1675eaea537074df028',
|
||||
842000: '8661cf2065dc713d2ba043f0b81f0effcc940eeb3e91906a21ff22c210561dcd',
|
||||
843000: '0dc2766f90415009d0c86bedffee6ebcf58042eb08262c0c67c4e9ed86b2aec8',
|
||||
844000: '26d072da864cab268a12794977b04ec44fb69ef3978e2342e82225974dac54dd',
|
||||
845000: '95e93bb60be8d5f07a1f4d26290c914957a82fc9d26ae8a3f20082eda27406ff',
|
||||
846000: 'f1bdc39af7705e58ab8b6c31dc70dce1e115db1cfd8cc9b037949dfbec82a59a',
|
||||
847000: 'f5f10f06396ecf2765d8a081141d489737c1d8d57c281f28f57c4cb2f90db883',
|
||||
848000: '331b8ef08605bae8d749893af9ed54f0df4f07a5a002108a2a0aea82d0360979',
|
||||
849000: '75b5f6233ab9a1bbc3c8b2893e5b22a0aa98e7ea635261255dc3c281f67d2260',
|
||||
850000: '5d7e6fe83e0ea1910a54a00090704737671d6f44df4228e21440ad1fc15e595f',
|
||||
851000: '7822db25d3ff0f6695ee38bad91edf317b5c6611673d28f1d22053110bb558be',
|
||||
852000: '2f0effad83a3561fc1a2806a562786a641d9ddb18d16bb9308006e7d324a21e9',
|
||||
853000: 'f603b2eaff11d5296377d990651317d40a1b2599ad2c5250eab131090f4b9458',
|
||||
854000: '34d59b26a50f18a9f250736d0f2e69d28b7e196fbef9b8a26c6b0b75c16aa194',
|
||||
855000: '76dd1ffff3946c0878969886fcf177ce5ab5560df19ddf006f9bcb02ae3e4e4f',
|
||||
856000: '74ff0b6f64e9dd5802fec2aac1d3ae194d28b9264114adaf0a882b46c8c918fe',
|
||||
857000: '7b5badfa2e4f40aa597a504d7ebe83c3705a2c6169a8c168ce293db223bc2d32',
|
||||
858000: '2bb0767a0f72b20d45ecfc3e34517dbda16d85758e040cf0e147f4cbd0cc57ac',
|
||||
859000: '3d741b9c365a91ed76f85824b94d19ec19b608d232660840ba59c7aa4b2cb67f',
|
||||
860000: 'd481a5a117878c0e3acd1f5844e150fb30e617577947d9846b1d214d703b71b0',
|
||||
861000: '54033424e488a3f1ad6946d4a6d9acb48465d6b1dbe8e1c2504a54cc84d7cad4',
|
||||
862000: '464bc3820a8cc8844dc9e26c388009e9982c656d46ef4b4fd0a2cb0e4eea0aaa',
|
||||
863000: 'd1aa94be2174f66780c4f226b9da3f6712b0f37af8dec33360bea83ca261b342',
|
||||
864000: '8c16008f11de5bc395d88cd802514ff647450f1bc136724b9aaf2ccce10a494f',
|
||||
865000: '3dae86012e97a201e2e1a47c899001ac00f78dc108026ed7c4194858c6c6dd5a',
|
||||
866000: 'afe5b0ccab995e1a1fa25fbc24c1d4b1a92c43042d03395f8743dcd806e72fd8',
|
||||
867000: 'c83716ac171aa9ab0d414833db340fa30e82bfda6cc616d3038529caab9b5600',
|
||||
868000: '8c409fe03cd35ef2d8e366818788b40eaeb4c8f6ae91450d75f4a66ca5f69cad',
|
||||
869000: '1d47909ceba790b8e1ce2e9902ee2775ea99e58efdb95668f9803a8ccf95f286',
|
||||
870000: '9adf5da1476388f053aa42de636da169d1cf1c9652cdf7cd9ad4fb18a0eb3388',
|
||||
871000: '8ad57fb1e74bcba0b5614fbac003be2bb32275dd85b38f2d28a0585005a99cfc',
|
||||
872000: '84a32e92012a356106e9657da8dab1a5491ea588fc29d411c69b20680c666420',
|
||||
873000: 'adf5921bbbfaa43929f67e6a070975313b77b456e262c700a27be611fceb17ae',
|
||||
874000: '09eaa7c4b18c79a46a2895190333f72336826223d5c986849a06f5153f49f2a5',
|
||||
875000: '235d7e4f31966507312149ea4c5e294aa84c695cf840117f0ef5963be7a0bda1',
|
||||
876000: '9aa9cb806ccbec0475ac330b496c5b2edeba38ba3f1e13ddd54a01457634a288',
|
||||
877000: 'c1e7f9b2b20bb1c4c0deadbc786d31fdf36f262325342aa23d1a66e2846b22bc',
|
||||
878000: 'ee0d2b20ac28ce23ab38698a57c6beff14f12b7af9d027c05cc92f652695f46b',
|
||||
879000: '0eb0810f4b81d1845b0a88f05449408df2e45715c9210a656f45278c5fdf7956',
|
||||
880000: 'e7d613027e3b4ca38d09bbef07998b57db237c6d67f1e8ea50024d2e0d9a1a72',
|
||||
881000: '21af4d355d8756b8bf0369b2d79b5c824148ae069026ba5c14f9dd6b7555e1db',
|
||||
}
|
||||
@@ -5,7 +5,7 @@ from lbry.wallet.transaction import OutputEffectiveAmountEstimator
|
||||
|
||||
MAXIMUM_TRIES = 100000
|
||||
|
||||
STRATEGIES = []
|
||||
STRATEGIES = ['sqlite'] # sqlite coin chooser is in database.py
|
||||
|
||||
|
||||
def strategy(method):
|
||||
|
||||
+287
-65
@@ -4,22 +4,33 @@ import asyncio
|
||||
import sqlite3
|
||||
import platform
|
||||
from binascii import hexlify
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from contextvars import ContextVar
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
from concurrent.futures.process import ProcessPoolExecutor
|
||||
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable, Dict, Optional
|
||||
from datetime import date
|
||||
from prometheus_client import Gauge, Counter, Histogram
|
||||
from lbry.utils import LockWithMetrics
|
||||
|
||||
from .bip32 import PubKey
|
||||
from .transaction import Transaction, Output, OutputScript, TXRefImmutable
|
||||
from .transaction import Transaction, Output, OutputScript, TXRefImmutable, Input
|
||||
from .constants import TXO_TYPES, CLAIM_TYPES
|
||||
from .util import date_to_julian_day
|
||||
|
||||
from concurrent.futures.thread import ThreadPoolExecutor # pylint: disable=wrong-import-order
|
||||
if platform.system() == 'Windows' or 'ANDROID_ARGUMENT' or 'KIVY_BUILD' in os.environ:
|
||||
from concurrent.futures.thread import ThreadPoolExecutor as ReaderExecutorClass # pylint: disable=reimported
|
||||
else:
|
||||
from concurrent.futures.process import ProcessPoolExecutor as ReaderExecutorClass
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
sqlite3.enable_callback_tracebacks(True)
|
||||
|
||||
HISTOGRAM_BUCKETS = (
|
||||
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReaderProcessState:
|
||||
@@ -55,24 +66,39 @@ def run_read_only_fetchone(sql, params):
|
||||
raise
|
||||
|
||||
|
||||
if platform.system() == 'Windows' or 'ANDROID_ARGUMENT' in os.environ:
|
||||
ReaderExecutorClass = ThreadPoolExecutor
|
||||
else:
|
||||
ReaderExecutorClass = ProcessPoolExecutor
|
||||
|
||||
|
||||
class AIOSQLite:
|
||||
reader_executor: ReaderExecutorClass
|
||||
|
||||
waiting_writes_metric = Gauge(
|
||||
"waiting_writes_count", "Number of waiting db writes", namespace="daemon_database"
|
||||
)
|
||||
waiting_reads_metric = Gauge(
|
||||
"waiting_reads_count", "Number of waiting db writes", namespace="daemon_database"
|
||||
)
|
||||
write_count_metric = Counter(
|
||||
"write_count", "Number of database writes", namespace="daemon_database"
|
||||
)
|
||||
read_count_metric = Counter(
|
||||
"read_count", "Number of database reads", namespace="daemon_database"
|
||||
)
|
||||
acquire_write_lock_metric = Histogram(
|
||||
f'write_lock_acquired', 'Time to acquire the write lock', namespace="daemon_database", buckets=HISTOGRAM_BUCKETS
|
||||
)
|
||||
held_write_lock_metric = Histogram(
|
||||
f'write_lock_held', 'Length of time the write lock is held for', namespace="daemon_database",
|
||||
buckets=HISTOGRAM_BUCKETS
|
||||
)
|
||||
|
||||
def __init__(self):
|
||||
# has to be single threaded as there is no mapping of thread:connection
|
||||
self.writer_executor = ThreadPoolExecutor(max_workers=1)
|
||||
self.writer_connection: Optional[sqlite3.Connection] = None
|
||||
self._closing = False
|
||||
self.query_count = 0
|
||||
self.write_lock = asyncio.Lock()
|
||||
self.write_lock = LockWithMetrics(self.acquire_write_lock_metric, self.held_write_lock_metric)
|
||||
self.writers = 0
|
||||
self.read_ready = asyncio.Event()
|
||||
self.urgent_read_done = asyncio.Event()
|
||||
|
||||
@classmethod
|
||||
async def connect(cls, path: Union[bytes, str], *args, **kwargs):
|
||||
@@ -88,13 +114,20 @@ class AIOSQLite:
|
||||
)
|
||||
await asyncio.get_event_loop().run_in_executor(db.writer_executor, _connect_writer)
|
||||
db.read_ready.set()
|
||||
db.urgent_read_done.set()
|
||||
return db
|
||||
|
||||
async def close(self):
|
||||
if self._closing:
|
||||
return
|
||||
self._closing = True
|
||||
await asyncio.get_event_loop().run_in_executor(self.writer_executor, self.writer_connection.close)
|
||||
|
||||
def __checkpoint_and_close(conn: sqlite3.Connection):
|
||||
conn.execute("PRAGMA WAL_CHECKPOINT(FULL);")
|
||||
log.info("DB checkpoint finished.")
|
||||
conn.close()
|
||||
await asyncio.get_event_loop().run_in_executor(
|
||||
self.writer_executor, __checkpoint_and_close, self.writer_connection)
|
||||
self.writer_executor.shutdown(wait=True)
|
||||
self.reader_executor.shutdown(wait=True)
|
||||
self.read_ready.clear()
|
||||
@@ -112,12 +145,30 @@ class AIOSQLite:
|
||||
read_only=False, fetch_all: bool = False) -> List[dict]:
|
||||
read_only_fn = run_read_only_fetchall if fetch_all else run_read_only_fetchone
|
||||
parameters = parameters if parameters is not None else []
|
||||
still_waiting = False
|
||||
urgent_read = False
|
||||
if read_only:
|
||||
while self.writers:
|
||||
await self.read_ready.wait()
|
||||
return await asyncio.get_event_loop().run_in_executor(
|
||||
self.reader_executor, read_only_fn, sql, parameters
|
||||
)
|
||||
self.waiting_reads_metric.inc()
|
||||
self.read_count_metric.inc()
|
||||
try:
|
||||
while self.writers and not self._closing: # more writes can come in while we are waiting for the first
|
||||
if not urgent_read and still_waiting and self.urgent_read_done.is_set():
|
||||
# throttle the writes if they pile up
|
||||
self.urgent_read_done.clear()
|
||||
urgent_read = True
|
||||
# wait until the running writes have finished
|
||||
await self.read_ready.wait()
|
||||
still_waiting = True
|
||||
if self._closing:
|
||||
raise asyncio.CancelledError()
|
||||
return await asyncio.get_event_loop().run_in_executor(
|
||||
self.reader_executor, read_only_fn, sql, parameters
|
||||
)
|
||||
finally:
|
||||
if urgent_read:
|
||||
# unthrottle the writers if they had to be throttled
|
||||
self.urgent_read_done.set()
|
||||
self.waiting_reads_metric.dec()
|
||||
if fetch_all:
|
||||
return await self.run(lambda conn: conn.execute(sql, parameters).fetchall())
|
||||
return await self.run(lambda conn: conn.execute(sql, parameters).fetchone())
|
||||
@@ -135,17 +186,34 @@ class AIOSQLite:
|
||||
return self.run(lambda conn: conn.execute(sql, parameters))
|
||||
|
||||
async def run(self, fun, *args, **kwargs):
|
||||
self.write_count_metric.inc()
|
||||
self.waiting_writes_metric.inc()
|
||||
# it's possible many writes are coming in one after the other, these can
|
||||
# block reader calls for a long time
|
||||
# if the reader waits for the writers to finish and then has to wait for
|
||||
# yet more, it will clear the urgent_read_done event to block more writers
|
||||
# piling on
|
||||
try:
|
||||
await self.urgent_read_done.wait()
|
||||
except Exception as e:
|
||||
self.waiting_writes_metric.dec()
|
||||
raise e
|
||||
self.writers += 1
|
||||
# block readers
|
||||
self.read_ready.clear()
|
||||
async with self.write_lock:
|
||||
try:
|
||||
try:
|
||||
async with self.write_lock:
|
||||
if self._closing:
|
||||
raise asyncio.CancelledError()
|
||||
return await asyncio.get_event_loop().run_in_executor(
|
||||
self.writer_executor, lambda: self.__run_transaction(fun, *args, **kwargs)
|
||||
)
|
||||
finally:
|
||||
self.writers -= 1
|
||||
if not self.writers:
|
||||
self.read_ready.set()
|
||||
finally:
|
||||
self.writers -= 1
|
||||
self.waiting_writes_metric.dec()
|
||||
if not self.writers:
|
||||
# unblock the readers once the last enqueued writer finishes
|
||||
self.read_ready.set()
|
||||
|
||||
def __run_transaction(self, fun: Callable[[sqlite3.Connection, Any, Any], Any], *args, **kwargs):
|
||||
self.writer_connection.execute('begin')
|
||||
@@ -160,10 +228,28 @@ class AIOSQLite:
|
||||
log.warning("rolled back")
|
||||
raise
|
||||
|
||||
def run_with_foreign_keys_disabled(self, fun, *args, **kwargs) -> Awaitable:
|
||||
return asyncio.get_event_loop().run_in_executor(
|
||||
self.writer_executor, self.__run_transaction_with_foreign_keys_disabled, fun, args, kwargs
|
||||
)
|
||||
async def run_with_foreign_keys_disabled(self, fun, *args, **kwargs):
|
||||
self.write_count_metric.inc()
|
||||
self.waiting_writes_metric.inc()
|
||||
try:
|
||||
await self.urgent_read_done.wait()
|
||||
except Exception as e:
|
||||
self.waiting_writes_metric.dec()
|
||||
raise e
|
||||
self.writers += 1
|
||||
self.read_ready.clear()
|
||||
try:
|
||||
async with self.write_lock:
|
||||
if self._closing:
|
||||
raise asyncio.CancelledError()
|
||||
return await asyncio.get_event_loop().run_in_executor(
|
||||
self.writer_executor, self.__run_transaction_with_foreign_keys_disabled, fun, args, kwargs
|
||||
)
|
||||
finally:
|
||||
self.writers -= 1
|
||||
self.waiting_writes_metric.dec()
|
||||
if not self.writers:
|
||||
self.read_ready.set()
|
||||
|
||||
def __run_transaction_with_foreign_keys_disabled(self,
|
||||
fun: Callable[[sqlite3.Connection, Any, Any], Any],
|
||||
@@ -301,14 +387,31 @@ def interpolate(sql, values):
|
||||
return sql
|
||||
|
||||
|
||||
def constrain_single_or_list(constraints, column, value, convert=lambda x: x):
|
||||
def constrain_single_or_list(constraints, column, value, convert=lambda x: x, negate=False):
|
||||
if value is not None:
|
||||
if isinstance(value, list):
|
||||
value = [convert(v) for v in value]
|
||||
if len(value) == 1:
|
||||
constraints[column] = value[0]
|
||||
if negate:
|
||||
constraints[f"{column}__or"] = {
|
||||
f"{column}__is_null": True,
|
||||
f"{column}__not": value[0]
|
||||
}
|
||||
else:
|
||||
constraints[column] = value[0]
|
||||
elif len(value) > 1:
|
||||
constraints[f"{column}__in"] = value
|
||||
if negate:
|
||||
constraints[f"{column}__or"] = {
|
||||
f"{column}__is_null": True,
|
||||
f"{column}__not_in": value
|
||||
}
|
||||
else:
|
||||
constraints[f"{column}__in"] = value
|
||||
elif negate:
|
||||
constraints[f"{column}__or"] = {
|
||||
f"{column}__is_null": True,
|
||||
f"{column}__not": convert(value)
|
||||
}
|
||||
else:
|
||||
constraints[column] = convert(value)
|
||||
return constraints
|
||||
@@ -345,7 +448,7 @@ class SQLiteMixin:
|
||||
return
|
||||
await self.db.executescript('\n'.join(
|
||||
f"DROP TABLE {table};" for table in tables
|
||||
))
|
||||
) + '\n' + 'PRAGMA WAL_CHECKPOINT(FULL);' + '\n' + 'VACUUM;')
|
||||
await self.db.execute(self.CREATE_VERSION_TABLE)
|
||||
await self.db.execute("INSERT INTO version VALUES (?)", (self.SCHEMA_VERSION,))
|
||||
await self.db.executescript(self.CREATE_TABLES_QUERY)
|
||||
@@ -391,9 +494,105 @@ def dict_row_factory(cursor, row):
|
||||
return d
|
||||
|
||||
|
||||
SQLITE_MAX_INTEGER = 9223372036854775807
|
||||
|
||||
|
||||
def _get_spendable_utxos(transaction: sqlite3.Connection, accounts: List, decoded_transactions: Dict[str, Transaction],
|
||||
result: Dict[Tuple[bytes, int, bool], List[int]], reserved: List[Transaction],
|
||||
amount_to_reserve: int, reserved_amount: int, floor: int, ceiling: int,
|
||||
fee_per_byte: int) -> int:
|
||||
accounts_fmt = ",".join(["?"] * len(accounts))
|
||||
txo_query = f"""
|
||||
SELECT tx.txid, txo.txoid, tx.raw, tx.height, txo.position as nout, tx.is_verified, txo.amount FROM txo
|
||||
INNER JOIN account_address USING (address)
|
||||
LEFT JOIN txi USING (txoid)
|
||||
INNER JOIN tx USING (txid)
|
||||
WHERE txo.txo_type=0 AND txi.txoid IS NULL AND tx.txid IS NOT NULL AND NOT txo.is_reserved
|
||||
AND txo.amount >= ? AND txo.amount < ?
|
||||
"""
|
||||
if accounts:
|
||||
txo_query += f"""
|
||||
AND account_address.account {'= ?' if len(accounts_fmt) == 1 else 'IN (' + accounts_fmt + ')'}
|
||||
"""
|
||||
txo_query += """
|
||||
ORDER BY txo.amount ASC, tx.height DESC
|
||||
"""
|
||||
# prefer confirmed, but save unconfirmed utxos from this selection in case they are needed
|
||||
unconfirmed = []
|
||||
for row in transaction.execute(txo_query, (floor, ceiling, *accounts)):
|
||||
(txid, txoid, raw, height, nout, verified, amount) = row.values()
|
||||
# verified or non verified transactions were found- reset the gap count
|
||||
# multiple txos can come from the same tx, only decode it once and cache
|
||||
if txid not in decoded_transactions:
|
||||
# cache the decoded transaction
|
||||
decoded_transactions[txid] = Transaction(raw)
|
||||
decoded_tx = decoded_transactions[txid]
|
||||
# save the unconfirmed txo for possible use later, if still needed
|
||||
if verified:
|
||||
# add the txo to the reservation, minus the fee for including it
|
||||
reserved_amount += amount
|
||||
reserved_amount -= Input.spend(decoded_tx.outputs[nout]).size * fee_per_byte
|
||||
# mark it as reserved
|
||||
result[(raw, height, verified)].append(nout)
|
||||
reserved.append(txoid)
|
||||
# if we've reserved enough, return
|
||||
if reserved_amount >= amount_to_reserve:
|
||||
return reserved_amount
|
||||
else:
|
||||
unconfirmed.append((txid, txoid, raw, height, nout, verified, amount))
|
||||
# we're popping the items, so to get them in the order they were seen they are reversed
|
||||
unconfirmed.reverse()
|
||||
# add available unconfirmed txos if any were previously found
|
||||
while unconfirmed and reserved_amount < amount_to_reserve:
|
||||
(txid, txoid, raw, height, nout, verified, amount) = unconfirmed.pop()
|
||||
# it's already decoded
|
||||
decoded_tx = decoded_transactions[txid]
|
||||
# add to the reserved amount
|
||||
reserved_amount += amount
|
||||
reserved_amount -= Input.spend(decoded_tx.outputs[nout]).size * fee_per_byte
|
||||
result[(raw, height, verified)].append(nout)
|
||||
reserved.append(txoid)
|
||||
return reserved_amount
|
||||
|
||||
|
||||
def get_and_reserve_spendable_utxos(transaction: sqlite3.Connection, accounts: List, amount_to_reserve: int, floor: int,
|
||||
fee_per_byte: int, set_reserved: bool, return_insufficient_funds: bool,
|
||||
base_multiplier: int = 100):
|
||||
txs = defaultdict(list)
|
||||
decoded_transactions = {}
|
||||
reserved = []
|
||||
|
||||
reserved_dewies = 0
|
||||
multiplier = base_multiplier
|
||||
gap_count = 0
|
||||
|
||||
while reserved_dewies < amount_to_reserve and gap_count < 5 and floor * multiplier < SQLITE_MAX_INTEGER:
|
||||
previous_reserved_dewies = reserved_dewies
|
||||
reserved_dewies = _get_spendable_utxos(
|
||||
transaction, accounts, decoded_transactions, txs, reserved, amount_to_reserve, reserved_dewies,
|
||||
floor, floor * multiplier, fee_per_byte
|
||||
)
|
||||
floor *= multiplier
|
||||
if previous_reserved_dewies == reserved_dewies:
|
||||
gap_count += 1
|
||||
multiplier **= 2
|
||||
else:
|
||||
gap_count = 0
|
||||
multiplier = base_multiplier
|
||||
|
||||
# reserve the accumulated txos if enough were found
|
||||
if reserved_dewies >= amount_to_reserve:
|
||||
if set_reserved:
|
||||
transaction.executemany("UPDATE txo SET is_reserved = ? WHERE txoid = ?",
|
||||
[(True, txoid) for txoid in reserved]).fetchall()
|
||||
return txs
|
||||
# return_insufficient_funds and set_reserved are used for testing
|
||||
return txs if return_insufficient_funds else {}
|
||||
|
||||
|
||||
class Database(SQLiteMixin):
|
||||
|
||||
SCHEMA_VERSION = "1.3"
|
||||
SCHEMA_VERSION = "1.5"
|
||||
|
||||
PRAGMAS = """
|
||||
pragma journal_mode=WAL;
|
||||
@@ -505,6 +704,9 @@ class Database(SQLiteMixin):
|
||||
row['txo_type'] = TXO_TYPES['stream']
|
||||
elif txo.is_support:
|
||||
row['txo_type'] = TXO_TYPES['support']
|
||||
support = txo.can_decode_support
|
||||
if support and support.is_signed:
|
||||
row['channel_id'] = support.signing_channel_id
|
||||
elif txo.purchase is not None:
|
||||
row['txo_type'] = TXO_TYPES['purchase']
|
||||
row['claim_id'] = txo.purchased_claim_id
|
||||
@@ -579,7 +781,7 @@ class Database(SQLiteMixin):
|
||||
return self.db.run(__many)
|
||||
|
||||
async def reserve_outputs(self, txos, is_reserved=True):
|
||||
txoids = ((is_reserved, txo.id) for txo in txos)
|
||||
txoids = [(is_reserved, txo.id) for txo in txos]
|
||||
await self.db.executemany("UPDATE txo SET is_reserved = ? WHERE txoid = ?", txoids)
|
||||
|
||||
async def release_outputs(self, txos):
|
||||
@@ -591,6 +793,20 @@ class Database(SQLiteMixin):
|
||||
# 2. update address histories removing deleted TXs
|
||||
return True
|
||||
|
||||
async def get_spendable_utxos(self, ledger, reserve_amount, accounts: Optional[Iterable], min_amount: int = 1,
|
||||
fee_per_byte: int = 50, set_reserved: bool = True,
|
||||
return_insufficient_funds: bool = False) -> List:
|
||||
to_spend = await self.db.run(
|
||||
get_and_reserve_spendable_utxos, tuple(account.id for account in accounts), reserve_amount, min_amount,
|
||||
fee_per_byte, set_reserved, return_insufficient_funds
|
||||
)
|
||||
txos = []
|
||||
for (raw, height, verified), positions in to_spend.items():
|
||||
tx = Transaction(raw, height=height, is_verified=verified)
|
||||
for nout in positions:
|
||||
txos.append(tx.outputs[nout].get_estimator(ledger))
|
||||
return txos
|
||||
|
||||
async def select_transactions(self, cols, accounts=None, read_only=False, **constraints):
|
||||
if not {'txid', 'txid__in'}.intersection(constraints):
|
||||
assert accounts, "'accounts' argument required when no 'txid' constraint is present"
|
||||
@@ -749,16 +965,18 @@ class Database(SQLiteMixin):
|
||||
sql.append("LEFT JOIN txi ON (txi.position=0 AND txi.txid=txo.txid)")
|
||||
return await self.db.execute_fetchall(*query(' '.join(sql), **constraints), read_only=read_only)
|
||||
|
||||
async def get_txos(self, wallet=None, no_tx=False, read_only=False, **constraints):
|
||||
async def get_txos(self, wallet=None, no_tx=False, no_channel_info=False, read_only=False, **constraints):
|
||||
include_is_spent = constraints.get('include_is_spent', False)
|
||||
include_is_my_input = constraints.get('include_is_my_input', False)
|
||||
include_is_my_output = constraints.pop('include_is_my_output', False)
|
||||
include_received_tips = constraints.pop('include_received_tips', False)
|
||||
|
||||
select_columns = [
|
||||
"tx.txid, raw, tx.height, tx.position as tx_position, tx.is_verified, "
|
||||
"tx.txid, tx.height, tx.position as tx_position, tx.is_verified, "
|
||||
"txo_type, txo.position as txo_position, amount, script"
|
||||
]
|
||||
if not no_tx:
|
||||
select_columns.append("raw")
|
||||
|
||||
my_accounts = {a.public_key.address for a in wallet.accounts} if wallet else set()
|
||||
my_accounts_sql = ""
|
||||
@@ -836,32 +1054,33 @@ class Database(SQLiteMixin):
|
||||
txo.received_tips = row['received_tips']
|
||||
txos.append(txo)
|
||||
|
||||
channel_ids = set()
|
||||
for txo in txos:
|
||||
if txo.is_claim and txo.can_decode_claim:
|
||||
if txo.claim.is_signed:
|
||||
channel_ids.add(txo.claim.signing_channel_id)
|
||||
if txo.claim.is_channel and wallet:
|
||||
for account in wallet.accounts:
|
||||
private_key = account.get_channel_private_key(
|
||||
txo.claim.channel.public_key_bytes
|
||||
)
|
||||
if private_key:
|
||||
txo.private_key = private_key
|
||||
break
|
||||
|
||||
if channel_ids:
|
||||
channels = {
|
||||
txo.claim_id: txo for txo in
|
||||
(await self.get_channels(
|
||||
wallet=wallet,
|
||||
claim_id__in=channel_ids,
|
||||
read_only=read_only
|
||||
))
|
||||
}
|
||||
if not no_channel_info:
|
||||
channel_ids = set()
|
||||
for txo in txos:
|
||||
if txo.is_claim and txo.can_decode_claim:
|
||||
txo.channel = channels.get(txo.claim.signing_channel_id, None)
|
||||
if txo.claim.is_signed:
|
||||
channel_ids.add(txo.claim.signing_channel_id)
|
||||
if txo.claim.is_channel and wallet:
|
||||
for account in wallet.accounts:
|
||||
private_key = await account.get_channel_private_key(
|
||||
txo.claim.channel.public_key_bytes
|
||||
)
|
||||
if private_key:
|
||||
txo.private_key = private_key
|
||||
break
|
||||
|
||||
if channel_ids:
|
||||
channels = {
|
||||
txo.claim_id: txo for txo in
|
||||
(await self.get_channels(
|
||||
wallet=wallet,
|
||||
claim_id__in=channel_ids,
|
||||
read_only=read_only
|
||||
))
|
||||
}
|
||||
for txo in txos:
|
||||
if txo.is_claim and txo.can_decode_claim:
|
||||
txo.channel = channels.get(txo.claim.signing_channel_id, None)
|
||||
|
||||
return txos
|
||||
|
||||
@@ -1068,13 +1287,16 @@ class Database(SQLiteMixin):
|
||||
self.constrain_collections(constraints)
|
||||
return self.get_utxo_count(**constraints)
|
||||
|
||||
async def release_all_outputs(self, account):
|
||||
await self.db.execute_fetchall(
|
||||
"UPDATE txo SET is_reserved = 0 WHERE"
|
||||
" is_reserved = 1 AND txo.address IN ("
|
||||
" SELECT address from account_address WHERE account = ?"
|
||||
" )", (account.public_key.address, )
|
||||
)
|
||||
async def release_all_outputs(self, account=None):
|
||||
if account is None:
|
||||
await self.db.execute_fetchall("UPDATE txo SET is_reserved = 0 WHERE is_reserved = 1")
|
||||
else:
|
||||
await self.db.execute_fetchall(
|
||||
"UPDATE txo SET is_reserved = 0 WHERE"
|
||||
" is_reserved = 1 AND txo.address IN ("
|
||||
" SELECT address from account_address WHERE account = ?"
|
||||
" )", (account.public_key.address, )
|
||||
)
|
||||
|
||||
def get_supports_summary(self, read_only=False, **constraints):
|
||||
return self.get_txos(
|
||||
|
||||
+44
-47
@@ -5,7 +5,6 @@ import asyncio
|
||||
import logging
|
||||
import zlib
|
||||
from datetime import date
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
|
||||
from io import BytesIO
|
||||
from typing import Optional, Iterator, Tuple, Callable
|
||||
@@ -42,23 +41,22 @@ class Headers:
|
||||
validate_difficulty: bool = True
|
||||
|
||||
def __init__(self, path) -> None:
|
||||
if path == ':memory:':
|
||||
self.io = BytesIO()
|
||||
self.io = None
|
||||
self.path = path
|
||||
self._size: Optional[int] = None
|
||||
self.chunk_getter: Optional[Callable] = None
|
||||
self.executor = ThreadPoolExecutor(1)
|
||||
self.known_missing_checkpointed_chunks = set()
|
||||
self.check_chunk_lock = asyncio.Lock()
|
||||
|
||||
async def open(self):
|
||||
if not self.executor:
|
||||
self.executor = ThreadPoolExecutor(1)
|
||||
self.io = BytesIO()
|
||||
if self.path != ':memory:':
|
||||
if not os.path.exists(self.path):
|
||||
self.io = open(self.path, 'w+b')
|
||||
else:
|
||||
self.io = open(self.path, 'r+b')
|
||||
def _readit():
|
||||
if os.path.exists(self.path):
|
||||
with open(self.path, 'r+b') as header_file:
|
||||
self.io.seek(0)
|
||||
self.io.write(header_file.read())
|
||||
await asyncio.get_event_loop().run_in_executor(None, _readit)
|
||||
bytes_size = self.io.seek(0, os.SEEK_END)
|
||||
self._size = bytes_size // self.header_size
|
||||
max_checkpointed_height = max(self.checkpoints.keys() or [-1]) + 1000
|
||||
@@ -72,10 +70,14 @@ class Headers:
|
||||
await self.get_all_missing_headers()
|
||||
|
||||
async def close(self):
|
||||
if self.executor:
|
||||
self.executor.shutdown()
|
||||
self.executor = None
|
||||
self.io.close()
|
||||
if self.io is not None:
|
||||
def _close():
|
||||
flags = 'r+b' if os.path.exists(self.path) else 'w+b'
|
||||
with open(self.path, flags) as header_file:
|
||||
header_file.write(self.io.getbuffer())
|
||||
await asyncio.get_event_loop().run_in_executor(None, _close)
|
||||
self.io.close()
|
||||
self.io = None
|
||||
|
||||
@staticmethod
|
||||
def serialize(header):
|
||||
@@ -135,28 +137,30 @@ class Headers:
|
||||
except struct.error:
|
||||
raise IndexError(f"failed to get {height}, at {len(self)}")
|
||||
|
||||
def estimated_timestamp(self, height):
|
||||
def estimated_timestamp(self, height, try_real_headers=True):
|
||||
if height <= 0:
|
||||
return
|
||||
if try_real_headers and self.has_header(height):
|
||||
offset = height * self.header_size
|
||||
return struct.unpack('<I', self.io.getbuffer()[offset + 100: offset + 104])[0]
|
||||
return int(self.first_block_timestamp + (height * self.timestamp_average_offset))
|
||||
|
||||
def estimated_julian_day(self, height):
|
||||
return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height)))
|
||||
return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height, False)))
|
||||
|
||||
async def get_raw_header(self, height) -> bytes:
|
||||
if self.chunk_getter:
|
||||
await self.ensure_chunk_at(height)
|
||||
if not 0 <= height <= self.height:
|
||||
raise IndexError(f"{height} is out of bounds, current height: {self.height}")
|
||||
return await asyncio.get_running_loop().run_in_executor(self.executor, self._read, height)
|
||||
return self._read(height)
|
||||
|
||||
def _read(self, height, count=1):
|
||||
self.io.seek(height * self.header_size, os.SEEK_SET)
|
||||
return self.io.read(self.header_size * count)
|
||||
offset = height * self.header_size
|
||||
return bytes(self.io.getbuffer()[offset: offset + self.header_size * count])
|
||||
|
||||
def chunk_hash(self, start, count):
|
||||
self.io.seek(start * self.header_size, os.SEEK_SET)
|
||||
return self.hash_header(self.io.read(count * self.header_size)).decode()
|
||||
return self.hash_header(self._read(start, count)).decode()
|
||||
|
||||
async def ensure_checkpointed_size(self):
|
||||
max_checkpointed_height = max(self.checkpoints.keys() or [-1])
|
||||
@@ -165,7 +169,7 @@ class Headers:
|
||||
|
||||
async def ensure_chunk_at(self, height):
|
||||
async with self.check_chunk_lock:
|
||||
if await self.has_header(height):
|
||||
if self.has_header(height):
|
||||
log.debug("has header %s", height)
|
||||
return
|
||||
return await self.fetch_chunk(height)
|
||||
@@ -179,7 +183,7 @@ class Headers:
|
||||
)
|
||||
chunk_hash = self.hash_header(chunk).decode()
|
||||
if self.checkpoints.get(start) == chunk_hash:
|
||||
await asyncio.get_running_loop().run_in_executor(self.executor, self._write, start, chunk)
|
||||
self._write(start, chunk)
|
||||
if start in self.known_missing_checkpointed_chunks:
|
||||
self.known_missing_checkpointed_chunks.remove(start)
|
||||
return
|
||||
@@ -189,27 +193,23 @@ class Headers:
|
||||
f"Checkpoint mismatch at height {start}. Expected {self.checkpoints[start]}, but got {chunk_hash} instead."
|
||||
)
|
||||
|
||||
async def has_header(self, height):
|
||||
def has_header(self, height):
|
||||
normalized_height = (height // 1000) * 1000
|
||||
if normalized_height in self.checkpoints:
|
||||
return normalized_height not in self.known_missing_checkpointed_chunks
|
||||
|
||||
def _has_header(height):
|
||||
empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d'
|
||||
all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b'
|
||||
return self.chunk_hash(height, 1) not in (empty, all_zeroes)
|
||||
return await asyncio.get_running_loop().run_in_executor(self.executor, _has_header, height)
|
||||
empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d'
|
||||
all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b'
|
||||
return self.chunk_hash(height, 1) not in (empty, all_zeroes)
|
||||
|
||||
async def get_all_missing_headers(self):
|
||||
# Heavy operation done in one optimized shot
|
||||
def _io_checkall():
|
||||
for chunk_height, expected_hash in reversed(list(self.checkpoints.items())):
|
||||
if chunk_height in self.known_missing_checkpointed_chunks:
|
||||
continue
|
||||
if self.chunk_hash(chunk_height, 1000) != expected_hash:
|
||||
self.known_missing_checkpointed_chunks.add(chunk_height)
|
||||
return self.known_missing_checkpointed_chunks
|
||||
return await asyncio.get_running_loop().run_in_executor(self.executor, _io_checkall)
|
||||
for chunk_height, expected_hash in reversed(list(self.checkpoints.items())):
|
||||
if chunk_height in self.known_missing_checkpointed_chunks:
|
||||
continue
|
||||
if self.chunk_hash(chunk_height, 1000) != expected_hash:
|
||||
self.known_missing_checkpointed_chunks.add(chunk_height)
|
||||
return self.known_missing_checkpointed_chunks
|
||||
|
||||
@property
|
||||
def height(self) -> int:
|
||||
@@ -241,7 +241,7 @@ class Headers:
|
||||
bail = True
|
||||
chunk = chunk[:(height-e.height)*self.header_size]
|
||||
if chunk:
|
||||
added += await asyncio.get_running_loop().run_in_executor(self.executor, self._write, height, chunk)
|
||||
added += self._write(height, chunk)
|
||||
if bail:
|
||||
break
|
||||
return added
|
||||
@@ -306,9 +306,7 @@ class Headers:
|
||||
previous_header_hash = fail = None
|
||||
batch_size = 36
|
||||
for height in range(start_height, self.height, batch_size):
|
||||
headers = await asyncio.get_running_loop().run_in_executor(
|
||||
self.executor, self._read, height, batch_size
|
||||
)
|
||||
headers = self._read(height, batch_size)
|
||||
if len(headers) % self.header_size != 0:
|
||||
headers = headers[:(len(headers) // self.header_size) * self.header_size]
|
||||
for header_hash, header in self._iterate_headers(height, headers):
|
||||
@@ -324,12 +322,11 @@ class Headers:
|
||||
assert start_height > 0 and height == start_height
|
||||
if fail:
|
||||
log.warning("Header file corrupted at height %s, truncating it.", height - 1)
|
||||
def __truncate(at_height):
|
||||
self.io.seek(max(0, (at_height - 1)) * self.header_size, os.SEEK_SET)
|
||||
self.io.truncate()
|
||||
self.io.flush()
|
||||
self._size = self.io.seek(0, os.SEEK_END) // self.header_size
|
||||
return await asyncio.get_running_loop().run_in_executor(self.executor, __truncate, height)
|
||||
self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET)
|
||||
self.io.truncate()
|
||||
self.io.flush()
|
||||
self._size = self.io.seek(0, os.SEEK_END) // self.header_size
|
||||
return
|
||||
previous_header_hash = header_hash
|
||||
|
||||
@classmethod
|
||||
|
||||
+282
-163
@@ -1,8 +1,8 @@
|
||||
import os
|
||||
import copy
|
||||
import time
|
||||
import asyncio
|
||||
import logging
|
||||
from io import StringIO
|
||||
from datetime import datetime
|
||||
from functools import partial
|
||||
from operator import itemgetter
|
||||
@@ -10,11 +10,11 @@ from collections import defaultdict
|
||||
from binascii import hexlify, unhexlify
|
||||
from typing import Dict, Tuple, Type, Iterable, List, Optional, DefaultDict, NamedTuple
|
||||
|
||||
import pylru
|
||||
from lbry.schema.result import Outputs, INVALID, NOT_FOUND
|
||||
from lbry.schema.url import URL
|
||||
from lbry.crypto.hash import hash160, double_sha256, sha256
|
||||
from lbry.crypto.base58 import Base58
|
||||
from lbry.utils import LRUCacheWithMetrics
|
||||
|
||||
from .tasks import TaskGroup
|
||||
from .database import Database
|
||||
@@ -29,7 +29,6 @@ from .constants import TXO_TYPES, CLAIM_TYPES, COIN, NULL_HASH32
|
||||
from .bip32 import PubKey, PrivateKey
|
||||
from .coinselection import CoinSelector
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
LedgerType = Type['BaseLedger']
|
||||
@@ -124,7 +123,6 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
self.network: Network = self.config.get('network') or Network(self)
|
||||
self.network.on_header.listen(self.receive_header)
|
||||
self.network.on_status.listen(self.process_status_update)
|
||||
self.network.on_connected.listen(self.join_network)
|
||||
|
||||
self.accounts = []
|
||||
self.fee_per_byte: int = self.config.get('fee_per_byte', self.default_fee_per_byte)
|
||||
@@ -157,18 +155,19 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
self._on_ready_controller = StreamController()
|
||||
self.on_ready = self._on_ready_controller.stream
|
||||
|
||||
self._tx_cache = pylru.lrucache(100000)
|
||||
self._tx_cache = LRUCacheWithMetrics(self.config.get("tx_cache_size", 1024), metric_name='tx')
|
||||
self._update_tasks = TaskGroup()
|
||||
self._other_tasks = TaskGroup() # that we dont need to start
|
||||
self._utxo_reservation_lock = asyncio.Lock()
|
||||
self._header_processing_lock = asyncio.Lock()
|
||||
self._address_update_locks: DefaultDict[str, asyncio.Lock] = defaultdict(asyncio.Lock)
|
||||
self._history_lock = asyncio.Lock()
|
||||
|
||||
self.coin_selection_strategy = None
|
||||
self._known_addresses_out_of_sync = set()
|
||||
|
||||
self.fee_per_name_char = self.config.get('fee_per_name_char', self.default_fee_per_name_char)
|
||||
self._balance_cache = pylru.lrucache(100000)
|
||||
self._balance_cache = LRUCacheWithMetrics(2 ** 15)
|
||||
|
||||
@classmethod
|
||||
def get_id(cls):
|
||||
@@ -232,7 +231,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
async def get_effective_amount_estimators(self, funding_accounts: Iterable[Account]):
|
||||
estimators = []
|
||||
for account in funding_accounts:
|
||||
utxos = await account.get_utxos()
|
||||
utxos = await account.get_utxos(no_tx=True, no_channel_info=True)
|
||||
for utxo in utxos:
|
||||
estimators.append(utxo.get_estimator(self))
|
||||
return estimators
|
||||
@@ -243,11 +242,15 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
def get_address_count(self, **constraints):
|
||||
return self.db.get_address_count(**constraints)
|
||||
|
||||
async def get_spendable_utxos(self, amount: int, funding_accounts):
|
||||
async def get_spendable_utxos(self, amount: int, funding_accounts: Optional[Iterable['Account']], min_amount=1):
|
||||
min_amount = min(amount // 10, min_amount)
|
||||
fee = Output.pay_pubkey_hash(COIN, NULL_HASH32).get_fee(self)
|
||||
selector = CoinSelector(amount, fee)
|
||||
async with self._utxo_reservation_lock:
|
||||
if self.coin_selection_strategy == 'sqlite':
|
||||
return await self.db.get_spendable_utxos(self, amount + fee, funding_accounts, min_amount=min_amount,
|
||||
fee_per_byte=self.fee_per_byte)
|
||||
txos = await self.get_effective_amount_estimators(funding_accounts)
|
||||
fee = Output.pay_pubkey_hash(COIN, NULL_HASH32).get_fee(self)
|
||||
selector = CoinSelector(amount, fee)
|
||||
spendables = selector.select(txos, self.coin_selection_strategy)
|
||||
if spendables:
|
||||
await self.reserve_outputs(s.txo for s in spendables)
|
||||
@@ -325,7 +328,10 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
await self.network.on_connected.first
|
||||
async with self._header_processing_lock:
|
||||
await self._update_tasks.add(self.initial_headers_sync())
|
||||
self.network.on_connected.listen(self.join_network)
|
||||
asyncio.ensure_future(self.join_network())
|
||||
await fully_synced
|
||||
await self.db.release_all_outputs()
|
||||
await asyncio.gather(*(a.maybe_migrate_certificates() for a in self.accounts))
|
||||
await asyncio.gather(*(a.save_max_gap() for a in self.accounts))
|
||||
if len(self.accounts) > 10:
|
||||
@@ -409,6 +415,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
"Blockchain Reorganization: attempting rewind to height %s from starting height %s",
|
||||
height, height+rewound
|
||||
)
|
||||
self._tx_cache.clear()
|
||||
|
||||
else:
|
||||
raise IndexError(f"headers.connect() returned negative number ({added})")
|
||||
@@ -473,21 +480,23 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
for address, remote_status in zip(batch, results):
|
||||
self._update_tasks.add(self.update_history(address, remote_status, address_manager))
|
||||
addresses_remaining = addresses_remaining[batch_size:]
|
||||
log.info("subscribed to %i/%i addresses on %s:%i", len(addresses) - len(addresses_remaining),
|
||||
len(addresses), *self.network.client.server_address_and_port)
|
||||
log.info(
|
||||
"finished subscribing to %i addresses on %s:%i", len(addresses),
|
||||
*self.network.client.server_address_and_port
|
||||
)
|
||||
if self.network.client and self.network.client.server_address_and_port:
|
||||
log.info("subscribed to %i/%i addresses on %s:%i", len(addresses) - len(addresses_remaining),
|
||||
len(addresses), *self.network.client.server_address_and_port)
|
||||
if self.network.client and self.network.client.server_address_and_port:
|
||||
log.info(
|
||||
"finished subscribing to %i addresses on %s:%i", len(addresses),
|
||||
*self.network.client.server_address_and_port
|
||||
)
|
||||
|
||||
def process_status_update(self, update):
|
||||
address, remote_status = update
|
||||
self._update_tasks.add(self.update_history(address, remote_status))
|
||||
|
||||
async def update_history(self, address, remote_status, address_manager: AddressManager = None):
|
||||
async def update_history(self, address, remote_status, address_manager: AddressManager = None,
|
||||
reattempt_update: bool = True):
|
||||
async with self._address_update_locks[address]:
|
||||
self._known_addresses_out_of_sync.discard(address)
|
||||
|
||||
local_status, local_history = await self.get_local_status_and_history(address)
|
||||
|
||||
if local_status == remote_status:
|
||||
@@ -497,60 +506,55 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
remote_history = list(map(itemgetter('tx_hash', 'height'), remote_history))
|
||||
we_need = set(remote_history) - set(local_history)
|
||||
if not we_need:
|
||||
remote_missing = set(local_history) - set(remote_history)
|
||||
if remote_missing:
|
||||
log.warning(
|
||||
"%i transactions we have for %s are not in the remote address history",
|
||||
len(remote_missing), address
|
||||
)
|
||||
return True
|
||||
|
||||
cache_tasks: List[asyncio.Task[Transaction]] = []
|
||||
synced_history = StringIO()
|
||||
loop = asyncio.get_running_loop()
|
||||
to_request = {}
|
||||
pending_synced_history = {}
|
||||
already_synced = set()
|
||||
|
||||
already_synced_offset = 0
|
||||
for i, (txid, remote_height) in enumerate(remote_history):
|
||||
if i < len(local_history) and local_history[i] == (txid, remote_height) and not cache_tasks:
|
||||
synced_history.write(f'{txid}:{remote_height}:')
|
||||
else:
|
||||
check_local = (txid, remote_height) not in we_need
|
||||
cache_tasks.append(loop.create_task(
|
||||
self.cache_transaction(txid, remote_height, check_local=check_local)
|
||||
))
|
||||
if i == already_synced_offset and i < len(local_history) and local_history[i] == (txid, remote_height):
|
||||
pending_synced_history[i] = f'{txid}:{remote_height}:'
|
||||
already_synced.add((txid, remote_height))
|
||||
already_synced_offset += 1
|
||||
continue
|
||||
|
||||
synced_txs = []
|
||||
for task in cache_tasks:
|
||||
tx = await task
|
||||
tx_indexes = {}
|
||||
|
||||
check_db_for_txos = []
|
||||
for txi in tx.inputs:
|
||||
if txi.txo_ref.txo is not None:
|
||||
continue
|
||||
cache_item = self._tx_cache.get(txi.txo_ref.tx_ref.id)
|
||||
if cache_item is not None:
|
||||
if cache_item.tx is None:
|
||||
await cache_item.has_tx.wait()
|
||||
assert cache_item.tx is not None
|
||||
txi.txo_ref = cache_item.tx.outputs[txi.txo_ref.position].ref
|
||||
else:
|
||||
check_db_for_txos.append(txi.txo_ref.id)
|
||||
for i, (txid, remote_height) in enumerate(remote_history):
|
||||
tx_indexes[txid] = i
|
||||
if (txid, remote_height) in already_synced:
|
||||
continue
|
||||
to_request[i] = (txid, remote_height)
|
||||
|
||||
referenced_txos = {} if not check_db_for_txos else {
|
||||
txo.id: txo for txo in await self.db.get_txos(
|
||||
txoid__in=check_db_for_txos, order_by='txo.txoid', no_tx=True
|
||||
)
|
||||
}
|
||||
|
||||
for txi in tx.inputs:
|
||||
if txi.txo_ref.txo is not None:
|
||||
continue
|
||||
referenced_txo = referenced_txos.get(txi.txo_ref.id)
|
||||
if referenced_txo is not None:
|
||||
txi.txo_ref = referenced_txo.ref
|
||||
|
||||
synced_history.write(f'{tx.id}:{tx.height}:')
|
||||
synced_txs.append(tx)
|
||||
|
||||
await self.db.save_transaction_io_batch(
|
||||
synced_txs, address, self.address_to_hash160(address), synced_history.getvalue()
|
||||
log.debug(
|
||||
"request %i transactions, %i/%i for %s are already synced", len(to_request), len(already_synced),
|
||||
len(remote_history), address
|
||||
)
|
||||
await asyncio.wait([
|
||||
self._on_transaction_controller.add(TransactionEvent(address, tx))
|
||||
for tx in synced_txs
|
||||
])
|
||||
remote_history_txids = set(txid for txid, _ in remote_history)
|
||||
async for tx in self.request_synced_transactions(to_request, remote_history_txids, address):
|
||||
pending_synced_history[tx_indexes[tx.id]] = f"{tx.id}:{tx.height}:"
|
||||
if len(pending_synced_history) % 100 == 0:
|
||||
log.info("Syncing address %s: %d/%d", address, len(pending_synced_history), len(to_request))
|
||||
log.info("Sync finished for address %s: %d/%d", address, len(pending_synced_history), len(to_request))
|
||||
|
||||
assert len(pending_synced_history) == len(remote_history), \
|
||||
f"{len(pending_synced_history)} vs {len(remote_history)}"
|
||||
synced_history = ""
|
||||
for remote_i, i in zip(range(len(remote_history)), sorted(pending_synced_history.keys())):
|
||||
assert i == remote_i, f"{i} vs {remote_i}"
|
||||
txid, height = remote_history[remote_i]
|
||||
if f"{txid}:{height}:" != pending_synced_history[i]:
|
||||
log.warning("history mismatch: %s vs %s", remote_history[remote_i], pending_synced_history[i])
|
||||
synced_history += pending_synced_history[i]
|
||||
await self.db.set_address_history(address, synced_history)
|
||||
|
||||
if address_manager is None:
|
||||
address_manager = await self.get_address_manager_for_address(address)
|
||||
@@ -559,73 +563,135 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
await address_manager.ensure_address_gap()
|
||||
|
||||
local_status, local_history = \
|
||||
await self.get_local_status_and_history(address, synced_history.getvalue())
|
||||
await self.get_local_status_and_history(address, synced_history)
|
||||
|
||||
if local_status != remote_status:
|
||||
if local_history == remote_history:
|
||||
log.warning(
|
||||
"%s has a synced history but a mismatched status", address
|
||||
)
|
||||
return True
|
||||
remote_set = set(remote_history)
|
||||
local_set = set(local_history)
|
||||
log.warning(
|
||||
"Wallet is out of sync after syncing. Remote: %s with %d items, local: %s with %d items",
|
||||
remote_status, len(remote_history), local_status, len(local_history)
|
||||
"%s is out of sync after syncing.\n"
|
||||
"Remote: %s with %d items (%i unique), local: %s with %d items (%i unique).\n"
|
||||
"Histories are mismatched on %i items.\n"
|
||||
"Local is missing\n"
|
||||
"%s\n"
|
||||
"Remote is missing\n"
|
||||
"%s\n"
|
||||
"******",
|
||||
address, remote_status, len(remote_history), len(remote_set),
|
||||
local_status, len(local_history), len(local_set), len(remote_set.symmetric_difference(local_set)),
|
||||
"\n".join([f"{txid} - {height}" for txid, height in local_set.difference(remote_set)]),
|
||||
"\n".join([f"{txid} - {height}" for txid, height in remote_set.difference(local_set)])
|
||||
)
|
||||
log.warning("local: %s", local_history)
|
||||
log.warning("remote: %s", remote_history)
|
||||
self._known_addresses_out_of_sync.add(address)
|
||||
return False
|
||||
else:
|
||||
log.debug("finished syncing transaction history for %s, %i known txs", address, len(local_history))
|
||||
return True
|
||||
|
||||
async def cache_transaction(self, txid, remote_height, check_local=True):
|
||||
cache_item = self._tx_cache.get(txid)
|
||||
if cache_item is None:
|
||||
cache_item = self._tx_cache[txid] = TransactionCacheItem()
|
||||
elif cache_item.tx is not None and \
|
||||
cache_item.tx.height >= remote_height and \
|
||||
(cache_item.tx.is_verified or remote_height < 1):
|
||||
return cache_item.tx # cached tx is already up-to-date
|
||||
|
||||
try:
|
||||
cache_item.pending_verifications += 1
|
||||
return await self._update_cache_item(cache_item, txid, remote_height, check_local)
|
||||
finally:
|
||||
cache_item.pending_verifications -= 1
|
||||
|
||||
async def _update_cache_item(self, cache_item, txid, remote_height, check_local=True):
|
||||
|
||||
async with cache_item.lock:
|
||||
|
||||
tx = cache_item.tx
|
||||
|
||||
if tx is None and check_local:
|
||||
# check local db
|
||||
tx = cache_item.tx = await self.db.get_transaction(txid=txid)
|
||||
|
||||
merkle = None
|
||||
if tx is None:
|
||||
# fetch from network
|
||||
_raw, merkle = await self.network.retriable_call(
|
||||
self.network.get_transaction_and_merkle, txid, remote_height
|
||||
)
|
||||
tx = Transaction(unhexlify(_raw), height=merkle.get('block_height'))
|
||||
cache_item.tx = tx # make sure it's saved before caching it
|
||||
await self.maybe_verify_transaction(tx, remote_height, merkle)
|
||||
return tx
|
||||
|
||||
async def maybe_verify_transaction(self, tx, remote_height, merkle=None):
|
||||
tx.height = remote_height
|
||||
cached = self._tx_cache.get(tx.id)
|
||||
if not cached:
|
||||
# cache txs looked up by transaction_show too
|
||||
cached = TransactionCacheItem()
|
||||
cached.tx = tx
|
||||
self._tx_cache[tx.id] = cached
|
||||
if 0 < remote_height < len(self.headers) and cached.pending_verifications <= 1:
|
||||
if 0 < remote_height < len(self.headers):
|
||||
# can't be tx.pending_verifications == 1 because we have to handle the transaction_show case
|
||||
if not merkle:
|
||||
merkle = await self.network.retriable_call(self.network.get_merkle, tx.id, remote_height)
|
||||
if 'merkle' not in merkle:
|
||||
return
|
||||
merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash)
|
||||
header = await self.headers.get(remote_height)
|
||||
tx.position = merkle['pos']
|
||||
tx.is_verified = merkle_root == header['merkle_root']
|
||||
return tx
|
||||
|
||||
async def request_transactions(self, to_request: Tuple[Tuple[str, int], ...], cached=False):
|
||||
batches = [[]]
|
||||
remote_heights = {}
|
||||
cache_hits = set()
|
||||
|
||||
for txid, height in sorted(to_request, key=lambda x: x[1]):
|
||||
if cached:
|
||||
cached_tx = self._tx_cache.get(txid)
|
||||
if cached_tx is not None:
|
||||
if cached_tx.tx is not None and cached_tx.tx.is_verified:
|
||||
cache_hits.add(txid)
|
||||
continue
|
||||
else:
|
||||
self._tx_cache[txid] = TransactionCacheItem()
|
||||
remote_heights[txid] = height
|
||||
if len(batches[-1]) == 100:
|
||||
batches.append([])
|
||||
batches[-1].append(txid)
|
||||
if not batches[-1]:
|
||||
batches.pop()
|
||||
if cached and cache_hits:
|
||||
yield {txid: self._tx_cache[txid].tx for txid in cache_hits}
|
||||
|
||||
for batch in batches:
|
||||
txs = await self._single_batch(batch, remote_heights)
|
||||
if cached:
|
||||
for txid, tx in txs.items():
|
||||
self._tx_cache[txid].tx = tx
|
||||
yield txs
|
||||
|
||||
async def request_synced_transactions(self, to_request, remote_history, address):
|
||||
async for txs in self.request_transactions(((txid, height) for txid, height in to_request.values())):
|
||||
for tx in txs.values():
|
||||
yield tx
|
||||
await self._sync_and_save_batch(address, remote_history, txs)
|
||||
|
||||
async def _single_batch(self, batch, remote_heights):
|
||||
heights = {remote_heights[txid] for txid in batch}
|
||||
unrestriced = 0 < min(heights) < max(heights) < max(self.headers.checkpoints or [0])
|
||||
batch_result = await self.network.retriable_call(self.network.get_transaction_batch, batch, not unrestriced)
|
||||
txs = {}
|
||||
for txid, (raw, merkle) in batch_result.items():
|
||||
remote_height = remote_heights[txid]
|
||||
tx = Transaction(unhexlify(raw), height=remote_height)
|
||||
txs[tx.id] = tx
|
||||
await self.maybe_verify_transaction(tx, remote_height, merkle)
|
||||
return txs
|
||||
|
||||
async def _sync_and_save_batch(self, address, remote_history, pending_txs):
|
||||
await asyncio.gather(*(self._sync(tx, remote_history, pending_txs) for tx in pending_txs.values()))
|
||||
await self.db.save_transaction_io_batch(
|
||||
pending_txs.values(), address, self.address_to_hash160(address), ""
|
||||
)
|
||||
while pending_txs:
|
||||
self._on_transaction_controller.add(TransactionEvent(address, pending_txs.popitem()[1]))
|
||||
|
||||
async def _sync(self, tx, remote_history, pending_txs):
|
||||
check_db_for_txos = {}
|
||||
for txi in tx.inputs:
|
||||
if txi.txo_ref.txo is not None:
|
||||
continue
|
||||
wanted_txid = txi.txo_ref.tx_ref.id
|
||||
if wanted_txid not in remote_history:
|
||||
continue
|
||||
if wanted_txid in pending_txs:
|
||||
txi.txo_ref = pending_txs[wanted_txid].outputs[txi.txo_ref.position].ref
|
||||
else:
|
||||
check_db_for_txos[txi] = txi.txo_ref.id
|
||||
|
||||
referenced_txos = {} if not check_db_for_txos else {
|
||||
txo.id: txo for txo in await self.db.get_txos(
|
||||
txoid__in=list(check_db_for_txos.values()), order_by='txo.txoid', no_tx=True
|
||||
)
|
||||
}
|
||||
|
||||
for txi in check_db_for_txos:
|
||||
if txi.txo_ref.id in referenced_txos:
|
||||
txi.txo_ref = referenced_txos[txi.txo_ref.id].ref
|
||||
else:
|
||||
tx_from_db = await self.db.get_transaction(txid=txi.txo_ref.tx_ref.id)
|
||||
if tx_from_db is None:
|
||||
log.warning("%s not on db, not on cache, but on remote history!", txi.txo_ref.id)
|
||||
else:
|
||||
txi.txo_ref = tx_from_db.outputs[txi.txo_ref.position].ref
|
||||
return tx
|
||||
|
||||
async def get_address_manager_for_address(self, address) -> Optional[AddressManager]:
|
||||
details = await self.db.get_address(address=address)
|
||||
@@ -639,6 +705,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
return self.network.broadcast(hexlify(tx.raw).decode())
|
||||
|
||||
async def wait(self, tx: Transaction, height=-1, timeout=1):
|
||||
timeout = timeout or 600 # after 10 minutes there is almost 0 hope
|
||||
addresses = set()
|
||||
for txi in tx.inputs:
|
||||
if txi.txo_ref.txo is not None:
|
||||
@@ -648,26 +715,40 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
for txo in tx.outputs:
|
||||
if txo.has_address:
|
||||
addresses.add(self.hash160_to_address(txo.pubkey_hash))
|
||||
start = int(time.perf_counter())
|
||||
while timeout and (int(time.perf_counter()) - start) <= timeout:
|
||||
if await self._wait_round(tx, height, addresses):
|
||||
return
|
||||
raise asyncio.TimeoutError('Timed out waiting for transaction.')
|
||||
|
||||
async def _wait_round(self, tx: Transaction, height: int, addresses: Iterable[str]):
|
||||
records = await self.db.get_addresses(address__in=addresses)
|
||||
_, pending = await asyncio.wait([
|
||||
self.on_transaction.where(partial(
|
||||
lambda a, e: a == e.address and e.tx.height >= height and e.tx.id == tx.id,
|
||||
address_record['address']
|
||||
)) for address_record in records
|
||||
], timeout=timeout)
|
||||
if pending:
|
||||
records = await self.db.get_addresses(address__in=addresses)
|
||||
for record in records:
|
||||
found = False
|
||||
local_history = (await self.get_local_status_and_history(
|
||||
record['address'], history=record['history']
|
||||
))[1] if record['history'] else []
|
||||
for txid, local_height in local_history:
|
||||
if txid == tx.id and local_height >= height:
|
||||
found = True
|
||||
if not found:
|
||||
print(record['history'], addresses, tx.id)
|
||||
raise asyncio.TimeoutError('Timed out waiting for transaction.')
|
||||
], timeout=1)
|
||||
if not pending:
|
||||
return True
|
||||
records = await self.db.get_addresses(address__in=addresses)
|
||||
for record in records:
|
||||
local_history = (await self.get_local_status_and_history(
|
||||
record['address'], history=record['history']
|
||||
))[1] if record['history'] else []
|
||||
for txid, local_height in local_history:
|
||||
if txid == tx.id:
|
||||
if local_height >= height:
|
||||
return True
|
||||
log.warning(
|
||||
"local history has higher height than remote for %s (%i vs %i)", txid,
|
||||
local_height, height
|
||||
)
|
||||
return False
|
||||
log.warning(
|
||||
"local history does not contain %s, requested height %i", tx.id, height
|
||||
)
|
||||
return False
|
||||
|
||||
async def _inflate_outputs(
|
||||
self, query, accounts,
|
||||
@@ -678,20 +759,31 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
include_received_tips=False) -> Tuple[List[Output], dict, int, int]:
|
||||
encoded_outputs = await query
|
||||
outputs = Outputs.from_base64(encoded_outputs or b'') # TODO: why is the server returning None?
|
||||
txs = []
|
||||
txs: List[Transaction] = []
|
||||
if len(outputs.txs) > 0:
|
||||
txs: List[Transaction] = await asyncio.gather(*(
|
||||
self.cache_transaction(*tx) for tx in outputs.txs
|
||||
))
|
||||
async for tx in self.request_transactions(tuple(outputs.txs), cached=True):
|
||||
txs.extend(tx.values())
|
||||
|
||||
txos, blocked = outputs.inflate(txs)
|
||||
_txos, blocked = outputs.inflate(txs)
|
||||
|
||||
txos = []
|
||||
for txo in _txos:
|
||||
if isinstance(txo, Output):
|
||||
# transactions and outputs are cached and shared between wallets
|
||||
# we don't want to leak informaion between wallet so we add the
|
||||
# wallet specific metadata on throw away copies of the txos
|
||||
txo = copy.copy(txo)
|
||||
channel = txo.channel
|
||||
txo.purchase_receipt = None
|
||||
txo.update_annotations(None)
|
||||
txo.channel = channel
|
||||
txos.append(txo)
|
||||
|
||||
includes = (
|
||||
include_purchase_receipt, include_is_my_output,
|
||||
include_sent_supports, include_sent_tips
|
||||
)
|
||||
if accounts and any(includes):
|
||||
copies = []
|
||||
receipts = {}
|
||||
if include_purchase_receipt:
|
||||
priced_claims = []
|
||||
@@ -708,55 +800,55 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
}
|
||||
for txo in txos:
|
||||
if isinstance(txo, Output) and txo.can_decode_claim:
|
||||
# transactions and outputs are cached and shared between wallets
|
||||
# we don't want to leak informaion between wallet so we add the
|
||||
# wallet specific metadata on throw away copies of the txos
|
||||
txo_copy = copy.copy(txo)
|
||||
copies.append(txo_copy)
|
||||
if include_purchase_receipt:
|
||||
txo_copy.purchase_receipt = receipts.get(txo.claim_id)
|
||||
txo.purchase_receipt = receipts.get(txo.claim_id)
|
||||
if include_is_my_output:
|
||||
mine = await self.db.get_txo_count(
|
||||
claim_id=txo.claim_id, txo_type__in=CLAIM_TYPES, is_my_output=True,
|
||||
is_spent=False, accounts=accounts
|
||||
)
|
||||
if mine:
|
||||
txo_copy.is_my_output = True
|
||||
txo.is_my_output = True
|
||||
else:
|
||||
txo_copy.is_my_output = False
|
||||
txo.is_my_output = False
|
||||
if include_sent_supports:
|
||||
supports = await self.db.get_txo_sum(
|
||||
claim_id=txo.claim_id, txo_type=TXO_TYPES['support'],
|
||||
is_my_input=True, is_my_output=True,
|
||||
is_spent=False, accounts=accounts
|
||||
)
|
||||
txo_copy.sent_supports = supports
|
||||
txo.sent_supports = supports
|
||||
if include_sent_tips:
|
||||
tips = await self.db.get_txo_sum(
|
||||
claim_id=txo.claim_id, txo_type=TXO_TYPES['support'],
|
||||
is_my_input=True, is_my_output=False,
|
||||
accounts=accounts
|
||||
)
|
||||
txo_copy.sent_tips = tips
|
||||
txo.sent_tips = tips
|
||||
if include_received_tips:
|
||||
tips = await self.db.get_txo_sum(
|
||||
claim_id=txo.claim_id, txo_type=TXO_TYPES['support'],
|
||||
is_my_input=False, is_my_output=True,
|
||||
accounts=accounts
|
||||
)
|
||||
txo_copy.received_tips = tips
|
||||
else:
|
||||
copies.append(txo)
|
||||
txos = copies
|
||||
txo.received_tips = tips
|
||||
return txos, blocked, outputs.offset, outputs.total
|
||||
|
||||
async def resolve(self, accounts, urls, **kwargs):
|
||||
resolve = partial(self.network.retriable_call, self.network.resolve)
|
||||
urls_copy = list(urls)
|
||||
async def resolve(self, accounts, urls, new_sdk_server=None, **kwargs):
|
||||
txos = []
|
||||
urls_copy = list(urls)
|
||||
if new_sdk_server:
|
||||
resolve = partial(self.network.new_resolve, new_sdk_server)
|
||||
else:
|
||||
resolve = partial(self.network.retriable_call, self.network.resolve)
|
||||
while urls_copy:
|
||||
batch, urls_copy = urls_copy[:500], urls_copy[500:]
|
||||
txos.extend((await self._inflate_outputs(resolve(batch), accounts, **kwargs))[0])
|
||||
batch, urls_copy = urls_copy[:100], urls_copy[100:]
|
||||
txos.extend(
|
||||
(await self._inflate_outputs(
|
||||
resolve(batch), accounts, **kwargs
|
||||
))[0]
|
||||
)
|
||||
|
||||
assert len(urls) == len(txos), "Mismatch between urls requested for resolve and responses received."
|
||||
result = {}
|
||||
for url, txo in zip(urls, txos):
|
||||
@@ -769,13 +861,20 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
result[url] = txo
|
||||
return result
|
||||
|
||||
async def sum_supports(self, new_sdk_server, **kwargs) -> List[Dict]:
|
||||
return await self.network.sum_supports(new_sdk_server, **kwargs)
|
||||
|
||||
async def claim_search(
|
||||
self, accounts, include_purchase_receipt=False, include_is_my_output=False,
|
||||
**kwargs) -> Tuple[List[Output], dict, int, int]:
|
||||
new_sdk_server=None, **kwargs) -> Tuple[List[Output], dict, int, int]:
|
||||
if new_sdk_server:
|
||||
claim_search = partial(self.network.new_claim_search, new_sdk_server)
|
||||
else:
|
||||
claim_search = self.network.claim_search
|
||||
return await self._inflate_outputs(
|
||||
self.network.claim_search(**kwargs), accounts,
|
||||
claim_search(**kwargs), accounts,
|
||||
include_purchase_receipt=include_purchase_receipt,
|
||||
include_is_my_output=include_is_my_output
|
||||
include_is_my_output=include_is_my_output,
|
||||
)
|
||||
|
||||
async def get_claim_by_claim_id(self, accounts, claim_id, **kwargs) -> Output:
|
||||
@@ -838,6 +937,11 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
return self.db.get_purchase_count(**constraints)
|
||||
|
||||
async def _resolve_for_local_results(self, accounts, txos):
|
||||
txos = await self._resolve_for_local_claim_results(accounts, txos)
|
||||
txos = await self._resolve_for_local_support_results(accounts, txos)
|
||||
return txos
|
||||
|
||||
async def _resolve_for_local_claim_results(self, accounts, txos):
|
||||
results = []
|
||||
response = await self.resolve(
|
||||
accounts, [txo.permanent_url for txo in txos if txo.can_decode_claim]
|
||||
@@ -853,6 +957,23 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
results.append(txo)
|
||||
return results
|
||||
|
||||
async def _resolve_for_local_support_results(self, accounts, txos):
|
||||
channel_ids = set()
|
||||
signed_support_txos = []
|
||||
for txo in txos:
|
||||
support = txo.can_decode_support
|
||||
if support and support.signing_channel_id:
|
||||
channel_ids.add(support.signing_channel_id)
|
||||
signed_support_txos.append(txo)
|
||||
if channel_ids:
|
||||
channels = {
|
||||
channel.claim_id: channel for channel in
|
||||
(await self.claim_search(accounts, claim_ids=list(channel_ids)))[0]
|
||||
}
|
||||
for txo in signed_support_txos:
|
||||
txo.channel = channels.get(txo.support.signing_channel_id)
|
||||
return txos
|
||||
|
||||
async def get_claims(self, resolve=False, **constraints):
|
||||
claims = await self.db.get_claims(**constraints)
|
||||
if resolve:
|
||||
@@ -881,7 +1002,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
return self.db.get_channel_count(**constraints)
|
||||
|
||||
async def resolve_collection(self, collection, offset=0, page_size=1):
|
||||
claim_ids = collection.claim.collection.claims.ids[offset:page_size+offset]
|
||||
claim_ids = collection.claim.collection.claims.ids[offset:page_size + offset]
|
||||
try:
|
||||
resolve_results, _, _, _ = await self.claim_search([], claim_ids=claim_ids)
|
||||
except Exception as err:
|
||||
@@ -930,7 +1051,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
'txid': tx.id,
|
||||
'timestamp': ts,
|
||||
'date': datetime.fromtimestamp(ts).isoformat(' ')[:-3] if tx.height > 0 else None,
|
||||
'confirmations': (headers.height+1) - tx.height if tx.height > 0 else 0,
|
||||
'confirmations': (headers.height + 1) - tx.height if tx.height > 0 else 0,
|
||||
'claim_info': [],
|
||||
'update_info': [],
|
||||
'support_info': [],
|
||||
@@ -940,7 +1061,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
is_my_inputs = all([txi.is_my_input for txi in tx.inputs])
|
||||
if is_my_inputs:
|
||||
# fees only matter if we are the ones paying them
|
||||
item['value'] = dewies_to_lbc(tx.net_account_balance+tx.fee)
|
||||
item['value'] = dewies_to_lbc(tx.net_account_balance + tx.fee)
|
||||
item['fee'] = dewies_to_lbc(-tx.fee)
|
||||
else:
|
||||
# someone else paid the fees
|
||||
@@ -963,13 +1084,13 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
if txi.txo_ref.txo is not None:
|
||||
other_txo = txi.txo_ref.txo
|
||||
if (other_txo.is_claim or other_txo.script.is_support_claim) \
|
||||
and other_txo.claim_id == txo.claim_id:
|
||||
and other_txo.claim_id == txo.claim_id:
|
||||
previous = other_txo
|
||||
break
|
||||
if previous is not None:
|
||||
item['update_info'].append({
|
||||
'address': txo.get_address(self),
|
||||
'balance_delta': dewies_to_lbc(previous.amount-txo.amount),
|
||||
'balance_delta': dewies_to_lbc(previous.amount - txo.amount),
|
||||
'amount': dewies_to_lbc(txo.amount),
|
||||
'claim_id': txo.claim_id,
|
||||
'claim_name': txo.claim_name,
|
||||
@@ -1047,7 +1168,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
for account in accounts:
|
||||
balance = self._balance_cache.get(account.id)
|
||||
if not balance:
|
||||
balance = self._balance_cache[account.id] =\
|
||||
balance = self._balance_cache[account.id] = \
|
||||
await account.get_detailed_balance(confirmations, reserved_subtotals=True)
|
||||
for key, value in balance.items():
|
||||
if key == 'reserved_subtotals':
|
||||
@@ -1057,7 +1178,6 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
result[key] += value
|
||||
return result
|
||||
|
||||
|
||||
class TestNetLedger(Ledger):
|
||||
network_name = 'testnet'
|
||||
pubkey_address_prefix = bytes((111,))
|
||||
@@ -1066,7 +1186,6 @@ class TestNetLedger(Ledger):
|
||||
extended_private_key_prefix = unhexlify('04358394')
|
||||
checkpoints = {}
|
||||
|
||||
|
||||
class RegTestLedger(Ledger):
|
||||
network_name = 'regtest'
|
||||
headers_class = UnvalidatedHeaders
|
||||
|
||||
@@ -184,6 +184,7 @@ class WalletManager:
|
||||
'auto_connect': True,
|
||||
'default_servers': config.lbryum_servers,
|
||||
'data_path': config.wallet_dir,
|
||||
'tx_cache_size': config.transaction_cache_size
|
||||
}
|
||||
|
||||
wallets_directory = os.path.join(config.wallet_dir, 'wallets')
|
||||
@@ -299,8 +300,8 @@ class WalletManager:
|
||||
async def broadcast_or_release(self, tx, blocking=False):
|
||||
try:
|
||||
await self.ledger.broadcast(tx)
|
||||
if blocking:
|
||||
await self.ledger.wait(tx, timeout=None)
|
||||
except:
|
||||
await self.ledger.release_tx(tx)
|
||||
raise
|
||||
if blocking:
|
||||
await self.ledger.wait(tx, timeout=None)
|
||||
+213
-177
@@ -1,25 +1,27 @@
|
||||
import logging
|
||||
import asyncio
|
||||
import json
|
||||
import socket
|
||||
from time import perf_counter
|
||||
from operator import itemgetter
|
||||
from collections import defaultdict
|
||||
from typing import Dict, Optional, Tuple
|
||||
import aiohttp
|
||||
|
||||
from lbry import __version__
|
||||
from lbry.utils import resolve_host
|
||||
from lbry.error import IncompatibleWalletServerError
|
||||
from lbry.wallet.rpc import RPCSession as BaseClientSession, Connector, RPCError, ProtocolError
|
||||
from lbry.wallet.stream import StreamController
|
||||
from lbry.wallet.server.udp import SPVStatusClientProtocol, SPVPong
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ClientSession(BaseClientSession):
|
||||
def __init__(self, *args, network, server, timeout=30, on_connect_callback=None, **kwargs):
|
||||
def __init__(self, *args, network: 'Network', server, timeout=30, **kwargs):
|
||||
self.network = network
|
||||
self.server = server
|
||||
super().__init__(*args, **kwargs)
|
||||
self._on_disconnect_controller = StreamController()
|
||||
self.on_disconnected = self._on_disconnect_controller.stream
|
||||
self.framer.max_size = self.max_errors = 1 << 32
|
||||
self.timeout = timeout
|
||||
self.max_seconds_idle = timeout * 2
|
||||
@@ -27,8 +29,6 @@ class ClientSession(BaseClientSession):
|
||||
self.connection_latency: Optional[float] = None
|
||||
self._response_samples = 0
|
||||
self.pending_amount = 0
|
||||
self._on_connect_cb = on_connect_callback or (lambda: None)
|
||||
self.trigger_urgent_reconnect = asyncio.Event()
|
||||
|
||||
@property
|
||||
def available(self):
|
||||
@@ -55,7 +55,7 @@ class ClientSession(BaseClientSession):
|
||||
|
||||
async def send_request(self, method, args=()):
|
||||
self.pending_amount += 1
|
||||
log.debug("send %s%s to %s:%i", method, tuple(args), *self.server)
|
||||
log.debug("send %s%s to %s:%i (%i timeout)", method, tuple(args), self.server[0], self.server[1], self.timeout)
|
||||
try:
|
||||
if method == 'server.version':
|
||||
return await self.send_timed_server_version_request(args, self.timeout)
|
||||
@@ -66,7 +66,7 @@ class ClientSession(BaseClientSession):
|
||||
log.debug("Time since last packet: %s", perf_counter() - self.last_packet_received)
|
||||
if (perf_counter() - self.last_packet_received) < self.timeout:
|
||||
continue
|
||||
log.info("timeout sending %s to %s:%i", method, *self.server)
|
||||
log.warning("timeout sending %s to %s:%i", method, *self.server)
|
||||
raise asyncio.TimeoutError
|
||||
if done:
|
||||
try:
|
||||
@@ -86,44 +86,12 @@ class ClientSession(BaseClientSession):
|
||||
self.synchronous_close()
|
||||
raise
|
||||
except asyncio.CancelledError:
|
||||
log.info("cancelled sending %s to %s:%i", method, *self.server)
|
||||
self.synchronous_close()
|
||||
log.warning("cancelled sending %s to %s:%i", method, *self.server)
|
||||
# self.synchronous_close()
|
||||
raise
|
||||
finally:
|
||||
self.pending_amount -= 1
|
||||
|
||||
async def ensure_session(self):
|
||||
# Handles reconnecting and maintaining a session alive
|
||||
# TODO: change to 'ping' on newer protocol (above 1.2)
|
||||
retry_delay = default_delay = 1.0
|
||||
while True:
|
||||
try:
|
||||
if self.is_closing():
|
||||
await self.create_connection(self.timeout)
|
||||
await self.ensure_server_version()
|
||||
self._on_connect_cb()
|
||||
if (perf_counter() - self.last_send) > self.max_seconds_idle or self.response_time is None:
|
||||
await self.ensure_server_version()
|
||||
retry_delay = default_delay
|
||||
except RPCError as e:
|
||||
await self.close()
|
||||
log.debug("Server error, ignoring for 1h: %s:%d -- %s", *self.server, e.message)
|
||||
retry_delay = 60 * 60
|
||||
except IncompatibleWalletServerError:
|
||||
await self.close()
|
||||
retry_delay = 60 * 60
|
||||
log.debug("Wallet server has an incompatible version, retrying in 1h: %s:%d", *self.server)
|
||||
except (asyncio.TimeoutError, OSError):
|
||||
await self.close()
|
||||
retry_delay = min(60, retry_delay * 2)
|
||||
log.debug("Wallet server timeout (retry in %s seconds): %s:%d", retry_delay, *self.server)
|
||||
try:
|
||||
await asyncio.wait_for(self.trigger_urgent_reconnect.wait(), timeout=retry_delay)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
finally:
|
||||
self.trigger_urgent_reconnect.clear()
|
||||
|
||||
async def ensure_server_version(self, required=None, timeout=3):
|
||||
required = required or self.network.PROTOCOL_VERSION
|
||||
response = await asyncio.wait_for(
|
||||
@@ -133,6 +101,25 @@ class ClientSession(BaseClientSession):
|
||||
raise IncompatibleWalletServerError(*self.server)
|
||||
return response
|
||||
|
||||
async def keepalive_loop(self, timeout=3, max_idle=60):
|
||||
try:
|
||||
while True:
|
||||
now = perf_counter()
|
||||
if min(self.last_send, self.last_packet_received) + max_idle < now:
|
||||
await asyncio.wait_for(
|
||||
self.send_request('server.ping', []), timeout=timeout
|
||||
)
|
||||
else:
|
||||
await asyncio.sleep(max(0, max_idle - (now - self.last_send)))
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError):
|
||||
log.warning("closing connection to %s:%i", *self.server)
|
||||
else:
|
||||
log.exception("lost connection to spv")
|
||||
finally:
|
||||
if not self.is_closing():
|
||||
self._close()
|
||||
|
||||
async def create_connection(self, timeout=6):
|
||||
connector = Connector(lambda: self, *self.server)
|
||||
start = perf_counter()
|
||||
@@ -144,13 +131,14 @@ class ClientSession(BaseClientSession):
|
||||
controller.add(request.args)
|
||||
|
||||
def connection_lost(self, exc):
|
||||
log.debug("Connection lost: %s:%d", *self.server)
|
||||
log.warning("Connection lost: %s:%d", *self.server)
|
||||
super().connection_lost(exc)
|
||||
self.response_time = None
|
||||
self.connection_latency = None
|
||||
self._response_samples = 0
|
||||
self.pending_amount = 0
|
||||
self._on_disconnect_controller.add(True)
|
||||
# self._on_disconnect_controller.add(True)
|
||||
if self.network:
|
||||
self.network.disconnect()
|
||||
|
||||
|
||||
class Network:
|
||||
@@ -160,10 +148,9 @@ class Network:
|
||||
|
||||
def __init__(self, ledger):
|
||||
self.ledger = ledger
|
||||
self.session_pool = SessionPool(network=self, timeout=self.config.get('connect_timeout', 6))
|
||||
self.client: Optional[ClientSession] = None
|
||||
self.server_features = None
|
||||
self._switch_task: Optional[asyncio.Task] = None
|
||||
# self._switch_task: Optional[asyncio.Task] = None
|
||||
self.running = False
|
||||
self.remote_height: int = 0
|
||||
self._concurrency = asyncio.Semaphore(16)
|
||||
@@ -182,55 +169,171 @@ class Network:
|
||||
'blockchain.address.subscribe': self._on_status_controller,
|
||||
}
|
||||
|
||||
self.aiohttp_session: Optional[aiohttp.ClientSession] = None
|
||||
self._urgent_need_reconnect = asyncio.Event()
|
||||
self._loop_task: Optional[asyncio.Task] = None
|
||||
self._keepalive_task: Optional[asyncio.Task] = None
|
||||
|
||||
@property
|
||||
def config(self):
|
||||
return self.ledger.config
|
||||
|
||||
async def switch_forever(self):
|
||||
while self.running:
|
||||
if self.is_connected:
|
||||
await self.client.on_disconnected.first
|
||||
self.server_features = None
|
||||
self.client = None
|
||||
continue
|
||||
self.client = await self.session_pool.wait_for_fastest_session()
|
||||
log.info("Switching to SPV wallet server: %s:%d", *self.client.server)
|
||||
try:
|
||||
self.server_features = await self.get_server_features()
|
||||
self._update_remote_height((await self.subscribe_headers(),))
|
||||
self._on_connected_controller.add(True)
|
||||
log.info("Subscribed to headers: %s:%d", *self.client.server)
|
||||
except (asyncio.TimeoutError, ConnectionError):
|
||||
log.info("Switching to %s:%d timed out, closing and retrying.", *self.client.server)
|
||||
self.client.synchronous_close()
|
||||
self.server_features = None
|
||||
self.client = None
|
||||
def disconnect(self):
|
||||
if self._keepalive_task and not self._keepalive_task.done():
|
||||
self._keepalive_task.cancel()
|
||||
self._keepalive_task = None
|
||||
|
||||
async def start(self):
|
||||
self.running = True
|
||||
self._switch_task = asyncio.ensure_future(self.switch_forever())
|
||||
# this may become unnecessary when there are no more bugs found,
|
||||
# but for now it helps understanding log reports
|
||||
self._switch_task.add_done_callback(lambda _: log.info("Wallet client switching task stopped."))
|
||||
self.session_pool.start(self.config['default_servers'])
|
||||
self.on_header.listen(self._update_remote_height)
|
||||
if not self.running:
|
||||
self.running = True
|
||||
self.aiohttp_session = aiohttp.ClientSession()
|
||||
self.on_header.listen(self._update_remote_height)
|
||||
self._loop_task = asyncio.create_task(self.network_loop())
|
||||
self._urgent_need_reconnect.set()
|
||||
|
||||
def loop_task_done_callback(f):
|
||||
try:
|
||||
f.result()
|
||||
except Exception:
|
||||
if self.running:
|
||||
log.exception("wallet server connection loop crashed")
|
||||
|
||||
self._loop_task.add_done_callback(loop_task_done_callback)
|
||||
|
||||
async def resolve_spv_dns(self):
|
||||
hostname_to_ip = {}
|
||||
ip_to_hostnames = defaultdict(list)
|
||||
|
||||
async def resolve_spv(server, port):
|
||||
try:
|
||||
server_addr = await resolve_host(server, port, 'udp')
|
||||
hostname_to_ip[server] = (server_addr, port)
|
||||
ip_to_hostnames[(server_addr, port)].append(server)
|
||||
except socket.error:
|
||||
log.warning("error looking up dns for spv server %s:%i", server, port)
|
||||
except Exception:
|
||||
log.exception("error looking up dns for spv server %s:%i", server, port)
|
||||
|
||||
# accumulate the dns results
|
||||
await asyncio.gather(*(resolve_spv(server, port) for (server, port) in self.config['default_servers']))
|
||||
return hostname_to_ip, ip_to_hostnames
|
||||
|
||||
async def get_n_fastest_spvs(self, n=5, timeout=3.0) -> Dict[Tuple[str, int], SPVPong]:
|
||||
loop = asyncio.get_event_loop()
|
||||
pong_responses = asyncio.Queue()
|
||||
connection = SPVStatusClientProtocol(pong_responses)
|
||||
sent_ping_timestamps = {}
|
||||
_, ip_to_hostnames = await self.resolve_spv_dns()
|
||||
log.info("%i possible spv servers to try (%i urls in config)", len(ip_to_hostnames),
|
||||
len(self.config['default_servers']))
|
||||
pongs = {}
|
||||
try:
|
||||
await loop.create_datagram_endpoint(lambda: connection, ('0.0.0.0', 0))
|
||||
# could raise OSError if it cant bind
|
||||
start = perf_counter()
|
||||
for server in ip_to_hostnames:
|
||||
connection.ping(server)
|
||||
sent_ping_timestamps[server] = perf_counter()
|
||||
while len(pongs) < n:
|
||||
(remote, ts), pong = await asyncio.wait_for(pong_responses.get(), timeout - (perf_counter() - start))
|
||||
latency = ts - start
|
||||
log.info("%s:%i has latency of %sms (available: %s, height: %i)",
|
||||
'/'.join(ip_to_hostnames[remote]), remote[1], round(latency * 1000, 2),
|
||||
pong.available, pong.height)
|
||||
|
||||
if pong.available:
|
||||
pongs[remote] = pong
|
||||
return pongs
|
||||
except asyncio.TimeoutError:
|
||||
if pongs:
|
||||
log.info("%i/%i probed spv servers are accepting connections", len(pongs), len(ip_to_hostnames))
|
||||
else:
|
||||
log.warning("%i spv status probes failed, retrying later. servers tried: %s",
|
||||
len(sent_ping_timestamps),
|
||||
', '.join('/'.join(hosts) + f' ({ip})' for ip, hosts in ip_to_hostnames.items()))
|
||||
return pongs
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
async def connect_to_fastest(self) -> Optional[ClientSession]:
|
||||
fastest_spvs = await self.get_n_fastest_spvs()
|
||||
for (host, port) in fastest_spvs:
|
||||
|
||||
client = ClientSession(network=self, server=(host, port))
|
||||
try:
|
||||
await client.create_connection()
|
||||
log.warning("Connected to spv server %s:%i", host, port)
|
||||
await client.ensure_server_version()
|
||||
return client
|
||||
except (asyncio.TimeoutError, ConnectionError, OSError, IncompatibleWalletServerError, RPCError):
|
||||
log.warning("Connecting to %s:%d failed", host, port)
|
||||
client._close()
|
||||
return
|
||||
|
||||
async def network_loop(self):
|
||||
sleep_delay = 30
|
||||
while self.running:
|
||||
await asyncio.wait(
|
||||
[asyncio.sleep(30), self._urgent_need_reconnect.wait()], return_when=asyncio.FIRST_COMPLETED
|
||||
)
|
||||
if self._urgent_need_reconnect.is_set():
|
||||
sleep_delay = 30
|
||||
self._urgent_need_reconnect.clear()
|
||||
if not self.is_connected:
|
||||
client = await self.connect_to_fastest()
|
||||
if not client:
|
||||
log.warning("failed to connect to any spv servers, retrying later")
|
||||
sleep_delay *= 2
|
||||
sleep_delay = min(sleep_delay, 300)
|
||||
continue
|
||||
log.debug("get spv server features %s:%i", *client.server)
|
||||
features = await client.send_request('server.features', [])
|
||||
self.client, self.server_features = client, features
|
||||
log.info("subscribe to headers %s:%i", *client.server)
|
||||
self._update_remote_height((await self.subscribe_headers(),))
|
||||
self._on_connected_controller.add(True)
|
||||
server_str = "%s:%i" % client.server
|
||||
log.info("maintaining connection to spv server %s", server_str)
|
||||
self._keepalive_task = asyncio.create_task(self.client.keepalive_loop())
|
||||
try:
|
||||
await asyncio.wait(
|
||||
[self._keepalive_task, self._urgent_need_reconnect.wait()],
|
||||
return_when=asyncio.FIRST_COMPLETED
|
||||
)
|
||||
if self._urgent_need_reconnect.is_set():
|
||||
log.warning("urgent reconnect needed")
|
||||
self._urgent_need_reconnect.clear()
|
||||
if self._keepalive_task and not self._keepalive_task.done():
|
||||
self._keepalive_task.cancel()
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
finally:
|
||||
self._keepalive_task = None
|
||||
self.client = None
|
||||
self.server_features = None
|
||||
log.warning("connection lost to %s", server_str)
|
||||
log.info("network loop finished")
|
||||
|
||||
async def stop(self):
|
||||
if self.running:
|
||||
self.running = False
|
||||
self._switch_task.cancel()
|
||||
self.session_pool.stop()
|
||||
self.running = False
|
||||
self.disconnect()
|
||||
if self._loop_task and not self._loop_task.done():
|
||||
self._loop_task.cancel()
|
||||
self._loop_task = None
|
||||
if self.aiohttp_session:
|
||||
await self.aiohttp_session.close()
|
||||
self.aiohttp_session = None
|
||||
|
||||
@property
|
||||
def is_connected(self):
|
||||
return self.client and not self.client.is_closing()
|
||||
|
||||
def rpc(self, list_or_method, args, restricted=True):
|
||||
session = self.client if restricted else self.session_pool.fastest_session
|
||||
if session and not session.is_closing():
|
||||
def rpc(self, list_or_method, args, restricted=True, session: Optional[ClientSession] = None):
|
||||
if session or self.is_connected:
|
||||
session = session or self.client
|
||||
return session.send_request(list_or_method, args)
|
||||
else:
|
||||
self.session_pool.trigger_nodelay_connect()
|
||||
self._urgent_need_reconnect.set()
|
||||
raise ConnectionError("Attempting to send rpc request when connection is not available.")
|
||||
|
||||
async def retriable_call(self, function, *args, **kwargs):
|
||||
@@ -238,14 +341,15 @@ class Network:
|
||||
while self.running:
|
||||
if not self.is_connected:
|
||||
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
|
||||
self._urgent_need_reconnect.set()
|
||||
await self.on_connected.first
|
||||
await self.session_pool.wait_for_fastest_session()
|
||||
try:
|
||||
return await function(*args, **kwargs)
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("Wallet server call timed out, retrying.")
|
||||
except ConnectionError:
|
||||
pass
|
||||
log.warning("connection error")
|
||||
|
||||
raise asyncio.CancelledError() # if we got here, we are shutting down
|
||||
|
||||
def _update_remote_height(self, header_args):
|
||||
@@ -256,6 +360,10 @@ class Network:
|
||||
restricted = known_height in (None, -1, 0) or 0 > known_height > self.remote_height - 10
|
||||
return self.rpc('blockchain.transaction.get', [tx_hash], restricted)
|
||||
|
||||
def get_transaction_batch(self, txids, restricted=True):
|
||||
# use any server if its old, otherwise restrict to who gave us the history
|
||||
return self.rpc('blockchain.transaction.get_batch', txids, restricted)
|
||||
|
||||
def get_transaction_and_merkle(self, tx_hash, known_height=None):
|
||||
# use any server if its old, otherwise restrict to who gave us the history
|
||||
restricted = known_height in (None, -1, 0) or 0 > known_height > self.remote_height - 10
|
||||
@@ -285,12 +393,13 @@ class Network:
|
||||
|
||||
async def subscribe_address(self, address, *addresses):
|
||||
addresses = list((address, ) + addresses)
|
||||
server_addr_and_port = self.client.server_address_and_port # on disconnect client will be None
|
||||
try:
|
||||
return await self.rpc('blockchain.address.subscribe', addresses, True)
|
||||
except asyncio.TimeoutError:
|
||||
log.warning(
|
||||
"timed out subscribing to addresses from %s:%i",
|
||||
*self.client.server_address_and_port
|
||||
*server_addr_and_port
|
||||
)
|
||||
# abort and cancel, we can't lose a subscription, it will happen again on reconnect
|
||||
if self.client:
|
||||
@@ -306,100 +415,27 @@ class Network:
|
||||
def get_claims_by_ids(self, claim_ids):
|
||||
return self.rpc('blockchain.claimtrie.getclaimsbyids', claim_ids)
|
||||
|
||||
def resolve(self, urls):
|
||||
return self.rpc('blockchain.claimtrie.resolve', urls)
|
||||
def resolve(self, urls, session_override=None):
|
||||
return self.rpc('blockchain.claimtrie.resolve', urls, False, session_override)
|
||||
|
||||
def claim_search(self, **kwargs):
|
||||
return self.rpc('blockchain.claimtrie.search', kwargs)
|
||||
def claim_search(self, session_override=None, **kwargs):
|
||||
return self.rpc('blockchain.claimtrie.search', kwargs, False, session_override)
|
||||
|
||||
async def new_resolve(self, server, urls):
|
||||
message = {"method": "resolve", "params": {"urls": urls, "protobuf": True}}
|
||||
async with self.aiohttp_session.post(server, json=message) as r:
|
||||
result = await r.json()
|
||||
return result['result']
|
||||
|
||||
class SessionPool:
|
||||
async def new_claim_search(self, server, **kwargs):
|
||||
kwargs['protobuf'] = True
|
||||
message = {"method": "claim_search", "params": kwargs}
|
||||
async with self.aiohttp_session.post(server, json=message) as r:
|
||||
result = await r.json()
|
||||
return result['result']
|
||||
|
||||
def __init__(self, network: Network, timeout: float):
|
||||
self.network = network
|
||||
self.sessions: Dict[ClientSession, Optional[asyncio.Task]] = dict()
|
||||
self.timeout = timeout
|
||||
self.new_connection_event = asyncio.Event()
|
||||
|
||||
@property
|
||||
def online(self):
|
||||
return any(not session.is_closing() for session in self.sessions)
|
||||
|
||||
@property
|
||||
def available_sessions(self):
|
||||
return (session for session in self.sessions if session.available)
|
||||
|
||||
@property
|
||||
def fastest_session(self):
|
||||
if not self.online:
|
||||
return None
|
||||
return min(
|
||||
[((session.response_time + session.connection_latency) * (session.pending_amount + 1), session)
|
||||
for session in self.available_sessions] or [(0, None)],
|
||||
key=itemgetter(0)
|
||||
)[1]
|
||||
|
||||
def _get_session_connect_callback(self, session: ClientSession):
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def callback():
|
||||
duplicate_connections = [
|
||||
s for s in self.sessions
|
||||
if s is not session and s.server_address_and_port == session.server_address_and_port
|
||||
]
|
||||
already_connected = None if not duplicate_connections else duplicate_connections[0]
|
||||
if already_connected:
|
||||
self.sessions.pop(session).cancel()
|
||||
session.synchronous_close()
|
||||
log.debug("wallet server %s resolves to the same server as %s, rechecking in an hour",
|
||||
session.server[0], already_connected.server[0])
|
||||
loop.call_later(3600, self._connect_session, session.server)
|
||||
return
|
||||
self.new_connection_event.set()
|
||||
log.info("connected to %s:%i", *session.server)
|
||||
|
||||
return callback
|
||||
|
||||
def _connect_session(self, server: Tuple[str, int]):
|
||||
session = None
|
||||
for s in self.sessions:
|
||||
if s.server == server:
|
||||
session = s
|
||||
break
|
||||
if not session:
|
||||
session = ClientSession(
|
||||
network=self.network, server=server
|
||||
)
|
||||
session._on_connect_cb = self._get_session_connect_callback(session)
|
||||
task = self.sessions.get(session, None)
|
||||
if not task or task.done():
|
||||
task = asyncio.create_task(session.ensure_session())
|
||||
task.add_done_callback(lambda _: self.ensure_connections())
|
||||
self.sessions[session] = task
|
||||
|
||||
def start(self, default_servers):
|
||||
for server in default_servers:
|
||||
self._connect_session(server)
|
||||
|
||||
def stop(self):
|
||||
for session, task in self.sessions.items():
|
||||
task.cancel()
|
||||
session.synchronous_close()
|
||||
self.sessions.clear()
|
||||
|
||||
def ensure_connections(self):
|
||||
for session in self.sessions:
|
||||
self._connect_session(session.server)
|
||||
|
||||
def trigger_nodelay_connect(self):
|
||||
# used when other parts of the system sees we might have internet back
|
||||
# bypasses the retry interval
|
||||
for session in self.sessions:
|
||||
session.trigger_urgent_reconnect.set()
|
||||
|
||||
async def wait_for_fastest_session(self):
|
||||
while not self.fastest_session:
|
||||
self.trigger_nodelay_connect()
|
||||
self.new_connection_event.clear()
|
||||
await self.new_connection_event.wait()
|
||||
return self.fastest_session
|
||||
async def sum_supports(self, server, **kwargs):
|
||||
message = {"method": "support_sum", "params": kwargs}
|
||||
async with self.aiohttp_session.post(server, json=message) as r:
|
||||
result = await r.json()
|
||||
return result['result']
|
||||
@@ -33,13 +33,16 @@ from asyncio import Event, CancelledError
|
||||
import logging
|
||||
import time
|
||||
from contextlib import suppress
|
||||
|
||||
from prometheus_client import Counter, Histogram
|
||||
from lbry.wallet.tasks import TaskGroup
|
||||
|
||||
from .jsonrpc import Request, JSONRPCConnection, JSONRPCv2, JSONRPC, Batch, Notification
|
||||
from .jsonrpc import RPCError, ProtocolError
|
||||
from .framing import BadMagicError, BadChecksumError, OversizedPayloadError, BitcoinFramer, NewlineFramer
|
||||
from lbry.wallet.server.prometheus import NOTIFICATION_COUNT, RESPONSE_TIMES, REQUEST_ERRORS_COUNT, RESET_CONNECTIONS
|
||||
|
||||
HISTOGRAM_BUCKETS = (
|
||||
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
|
||||
)
|
||||
|
||||
|
||||
class Connector:
|
||||
@@ -203,7 +206,7 @@ class SessionBase(asyncio.Protocol):
|
||||
"""
|
||||
return self._address
|
||||
|
||||
def peer_address_str(self):
|
||||
def peer_address_str(self, for_log=True):
|
||||
"""Returns the peer's IP address and port as a human-readable
|
||||
string."""
|
||||
if not self._address:
|
||||
@@ -372,10 +375,26 @@ class BatchRequest:
|
||||
raise BatchError(self)
|
||||
|
||||
|
||||
NAMESPACE = "wallet_server"
|
||||
|
||||
|
||||
class RPCSession(SessionBase):
|
||||
"""Base class for protocols where a message can lead to a response,
|
||||
for example JSON RPC."""
|
||||
|
||||
RESPONSE_TIMES = Histogram("response_time", "Response times", namespace=NAMESPACE,
|
||||
labelnames=("method", "version"), buckets=HISTOGRAM_BUCKETS)
|
||||
NOTIFICATION_COUNT = Counter("notification", "Number of notifications sent (for subscriptions)",
|
||||
namespace=NAMESPACE, labelnames=("method", "version"))
|
||||
REQUEST_ERRORS_COUNT = Counter(
|
||||
"request_error", "Number of requests that returned errors", namespace=NAMESPACE,
|
||||
labelnames=("method", "version")
|
||||
)
|
||||
RESET_CONNECTIONS = Counter(
|
||||
"reset_clients", "Number of reset connections by client version",
|
||||
namespace=NAMESPACE, labelnames=("version",)
|
||||
)
|
||||
|
||||
def __init__(self, *, framer=None, loop=None, connection=None):
|
||||
super().__init__(framer=framer, loop=loop)
|
||||
self.connection = connection or self.default_connection()
|
||||
@@ -388,7 +407,7 @@ class RPCSession(SessionBase):
|
||||
except MemoryError:
|
||||
self.logger.warning('received oversized message from %s:%s, dropping connection',
|
||||
self._address[0], self._address[1])
|
||||
RESET_CONNECTIONS.labels(version=self.client_version).inc()
|
||||
self.RESET_CONNECTIONS.labels(version=self.client_version).inc()
|
||||
self._close()
|
||||
return
|
||||
|
||||
@@ -422,7 +441,7 @@ class RPCSession(SessionBase):
|
||||
'internal server error')
|
||||
if isinstance(request, Request):
|
||||
message = request.send_result(result)
|
||||
RESPONSE_TIMES.labels(
|
||||
self.RESPONSE_TIMES.labels(
|
||||
method=request.method,
|
||||
version=self.client_version
|
||||
).observe(time.perf_counter() - start)
|
||||
@@ -430,7 +449,7 @@ class RPCSession(SessionBase):
|
||||
await self._send_message(message)
|
||||
if isinstance(result, Exception):
|
||||
self._bump_errors()
|
||||
REQUEST_ERRORS_COUNT.labels(
|
||||
self.REQUEST_ERRORS_COUNT.labels(
|
||||
method=request.method,
|
||||
version=self.client_version
|
||||
).inc()
|
||||
@@ -464,11 +483,17 @@ class RPCSession(SessionBase):
|
||||
raise result
|
||||
return result
|
||||
|
||||
async def send_notification(self, method, args=()):
|
||||
async def send_notification(self, method, args=()) -> bool:
|
||||
"""Send an RPC notification over the network."""
|
||||
message = self.connection.send_notification(Notification(method, args))
|
||||
NOTIFICATION_COUNT.labels(method=method, version=self.client_version).inc()
|
||||
await self._send_message(message)
|
||||
self.NOTIFICATION_COUNT.labels(method=method, version=self.client_version).inc()
|
||||
try:
|
||||
await self._send_message(message)
|
||||
return True
|
||||
except asyncio.TimeoutError:
|
||||
self.logger.info("timeout sending address notification to %s", self.peer_address_str(for_log=True))
|
||||
self.abort()
|
||||
return False
|
||||
|
||||
def send_batch(self, raise_errors=False):
|
||||
"""Return a BatchRequest. Intended to be used like so:
|
||||
|
||||
@@ -438,6 +438,17 @@ class OutputScript(Script):
|
||||
SUPPORT_CLAIM_OPCODES + PAY_SCRIPT_HASH.opcodes
|
||||
))
|
||||
|
||||
SUPPORT_CLAIM_DATA_OPCODES = (
|
||||
OP_SUPPORT_CLAIM, PUSH_SINGLE('claim_name'), PUSH_SINGLE('claim_id'), PUSH_SINGLE('support'),
|
||||
OP_2DROP, OP_2DROP
|
||||
)
|
||||
SUPPORT_CLAIM_DATA_PUBKEY = Template('support_claim+data+pay_pubkey_hash', (
|
||||
SUPPORT_CLAIM_DATA_OPCODES + PAY_PUBKEY_HASH.opcodes
|
||||
))
|
||||
SUPPORT_CLAIM_DATA_SCRIPT = Template('support_claim+data+pay_script_hash', (
|
||||
SUPPORT_CLAIM_DATA_OPCODES + PAY_SCRIPT_HASH.opcodes
|
||||
))
|
||||
|
||||
UPDATE_CLAIM_OPCODES = (
|
||||
OP_UPDATE_CLAIM, PUSH_SINGLE('claim_name'), PUSH_SINGLE('claim_id'), PUSH_SINGLE('claim'),
|
||||
OP_2DROP, OP_2DROP
|
||||
@@ -474,6 +485,8 @@ class OutputScript(Script):
|
||||
CLAIM_NAME_SCRIPT,
|
||||
SUPPORT_CLAIM_PUBKEY,
|
||||
SUPPORT_CLAIM_SCRIPT,
|
||||
SUPPORT_CLAIM_DATA_PUBKEY,
|
||||
SUPPORT_CLAIM_DATA_SCRIPT,
|
||||
UPDATE_CLAIM_PUBKEY,
|
||||
UPDATE_CLAIM_SCRIPT,
|
||||
SELL_CLAIM, SELL_SCRIPT,
|
||||
@@ -527,6 +540,16 @@ class OutputScript(Script):
|
||||
'pubkey_hash': pubkey_hash
|
||||
})
|
||||
|
||||
@classmethod
|
||||
def pay_support_data_pubkey_hash(
|
||||
cls, claim_name: bytes, claim_id: bytes, support, pubkey_hash: bytes):
|
||||
return cls(template=cls.SUPPORT_CLAIM_DATA_PUBKEY, values={
|
||||
'claim_name': claim_name,
|
||||
'claim_id': claim_id,
|
||||
'support': support,
|
||||
'pubkey_hash': pubkey_hash
|
||||
})
|
||||
|
||||
@classmethod
|
||||
def sell_script(cls, price):
|
||||
return cls(template=cls.SELL_SCRIPT, values={
|
||||
@@ -575,6 +598,10 @@ class OutputScript(Script):
|
||||
def is_support_claim(self):
|
||||
return self.template.name.startswith('support_claim+')
|
||||
|
||||
@property
|
||||
def is_support_claim_data(self):
|
||||
return self.template.name.startswith('support_claim+data+')
|
||||
|
||||
@property
|
||||
def is_sell_claim(self):
|
||||
return self.template.name.startswith('sell_claim+')
|
||||
|
||||
@@ -3,6 +3,7 @@ import asyncio
|
||||
from struct import pack, unpack
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
from typing import Optional
|
||||
from prometheus_client import Gauge, Histogram
|
||||
import lbry
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.wallet.server.db.writer import SQLDB
|
||||
@@ -10,7 +11,7 @@ from lbry.wallet.server.daemon import DaemonError
|
||||
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
|
||||
from lbry.wallet.server.util import chunks, class_logger
|
||||
from lbry.wallet.server.leveldb import FlushData
|
||||
from lbry.wallet.server.prometheus import BLOCK_COUNT, BLOCK_UPDATE_TIMES, REORG_COUNT
|
||||
from lbry.wallet.server.udp import StatusServer
|
||||
|
||||
|
||||
class Prefetcher:
|
||||
@@ -129,6 +130,12 @@ class ChainError(Exception):
|
||||
"""Raised on error processing blocks."""
|
||||
|
||||
|
||||
NAMESPACE = "wallet_server"
|
||||
HISTOGRAM_BUCKETS = (
|
||||
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
|
||||
)
|
||||
|
||||
|
||||
class BlockProcessor:
|
||||
"""Process blocks and update the DB state to match.
|
||||
|
||||
@@ -136,6 +143,16 @@ class BlockProcessor:
|
||||
Coordinate backing up in case of chain reorganisations.
|
||||
"""
|
||||
|
||||
block_count_metric = Gauge(
|
||||
"block_count", "Number of processed blocks", namespace=NAMESPACE
|
||||
)
|
||||
block_update_time_metric = Histogram(
|
||||
"block_time", "Block update times", namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS
|
||||
)
|
||||
reorg_count_metric = Gauge(
|
||||
"reorg_count", "Number of reorgs", namespace=NAMESPACE
|
||||
)
|
||||
|
||||
def __init__(self, env, db, daemon, notifications):
|
||||
self.env = env
|
||||
self.db = db
|
||||
@@ -155,7 +172,8 @@ class BlockProcessor:
|
||||
|
||||
# Caches of unflushed items.
|
||||
self.headers = []
|
||||
self.tx_hashes = []
|
||||
self.block_hashes = []
|
||||
self.block_txs = []
|
||||
self.undo_infos = []
|
||||
|
||||
# UTXO cache
|
||||
@@ -167,6 +185,8 @@ class BlockProcessor:
|
||||
self.state_lock = asyncio.Lock()
|
||||
|
||||
self.search_cache = {}
|
||||
self.history_cache = {}
|
||||
self.status_server = StatusServer()
|
||||
|
||||
async def run_in_thread_with_lock(self, func, *args):
|
||||
# Run in a thread to prevent blocking. Shielded so that
|
||||
@@ -197,10 +217,13 @@ class BlockProcessor:
|
||||
await self.run_in_thread_with_lock(self.advance_blocks, blocks)
|
||||
for cache in self.search_cache.values():
|
||||
cache.clear()
|
||||
self.history_cache.clear()
|
||||
self.notifications.notified_mempool_txs.clear()
|
||||
await self._maybe_flush()
|
||||
processed_time = time.perf_counter() - start
|
||||
BLOCK_COUNT.set(self.height)
|
||||
BLOCK_UPDATE_TIMES.observe(processed_time)
|
||||
self.block_count_metric.set(self.height)
|
||||
self.block_update_time_metric.observe(processed_time)
|
||||
self.status_server.set_height(self.db.fs_height, self.db.db_tip)
|
||||
if not self.db.first_sync:
|
||||
s = '' if len(blocks) == 1 else 's'
|
||||
self.logger.info('processed {:,d} block{} in {:.1f}s'.format(len(blocks), s, processed_time))
|
||||
@@ -228,12 +251,12 @@ class BlockProcessor:
|
||||
self.logger.info('chain reorg detected')
|
||||
else:
|
||||
self.logger.info(f'faking a reorg of {count:,d} blocks')
|
||||
await self.flush(True)
|
||||
|
||||
|
||||
async def get_raw_blocks(last_height, hex_hashes):
|
||||
heights = range(last_height, last_height - len(hex_hashes), -1)
|
||||
try:
|
||||
blocks = [self.db.read_raw_block(height) for height in heights]
|
||||
blocks = [await self.db.read_raw_block(height) for height in heights]
|
||||
self.logger.info(f'read {len(blocks)} blocks from disk')
|
||||
return blocks
|
||||
except FileNotFoundError:
|
||||
@@ -245,17 +268,28 @@ class BlockProcessor:
|
||||
self.touched.discard(None)
|
||||
self.db.flush_backup(self.flush_data(), self.touched)
|
||||
|
||||
start, last, hashes = await self.reorg_hashes(count)
|
||||
# Reverse and convert to hex strings.
|
||||
hashes = [hash_to_hex_str(hash) for hash in reversed(hashes)]
|
||||
for hex_hashes in chunks(hashes, 50):
|
||||
raw_blocks = await get_raw_blocks(last, hex_hashes)
|
||||
await self.run_in_thread_with_lock(self.backup_blocks, raw_blocks)
|
||||
await self.run_in_thread_with_lock(flush_backup)
|
||||
last -= len(raw_blocks)
|
||||
await self.run_in_thread_with_lock(self.db.sql.delete_claims_above_height, self.height)
|
||||
await self.prefetcher.reset_height(self.height)
|
||||
REORG_COUNT.inc()
|
||||
try:
|
||||
await self.flush(True)
|
||||
|
||||
start, last, hashes = await self.reorg_hashes(count)
|
||||
# Reverse and convert to hex strings.
|
||||
hashes = [hash_to_hex_str(hash) for hash in reversed(hashes)]
|
||||
self.logger.info("reorg %i block hashes", len(hashes))
|
||||
for hex_hashes in chunks(hashes, 50):
|
||||
raw_blocks = await get_raw_blocks(last, hex_hashes)
|
||||
self.logger.info("got %i raw blocks", len(raw_blocks))
|
||||
await self.run_in_thread_with_lock(self.backup_blocks, raw_blocks)
|
||||
await self.run_in_thread_with_lock(flush_backup)
|
||||
last -= len(raw_blocks)
|
||||
|
||||
await self.run_in_thread_with_lock(self.db.sql.delete_claims_above_height, self.height)
|
||||
await self.prefetcher.reset_height(self.height)
|
||||
self.reorg_count_metric.inc()
|
||||
except:
|
||||
self.logger.exception("boom")
|
||||
raise
|
||||
finally:
|
||||
self.logger.info("done with reorg")
|
||||
|
||||
async def reorg_hashes(self, count):
|
||||
"""Return a pair (start, last, hashes) of blocks to back up during a
|
||||
@@ -318,8 +352,8 @@ class BlockProcessor:
|
||||
def flush_data(self):
|
||||
"""The data for a flush. The lock must be taken."""
|
||||
assert self.state_lock.locked()
|
||||
return FlushData(self.height, self.tx_count, self.headers,
|
||||
self.tx_hashes, self.undo_infos, self.utxo_cache,
|
||||
return FlushData(self.height, self.tx_count, self.headers, self.block_hashes,
|
||||
self.block_txs, self.undo_infos, self.utxo_cache,
|
||||
self.db_deletes, self.tip)
|
||||
|
||||
async def flush(self, flush_utxos):
|
||||
@@ -333,11 +367,9 @@ class BlockProcessor:
|
||||
# performed on the DB.
|
||||
if self._caught_up_event.is_set():
|
||||
await self.flush(True)
|
||||
elif time.time() > self.next_cache_check:
|
||||
flush_arg = self.check_cache_size()
|
||||
if flush_arg is not None:
|
||||
await self.flush(flush_arg)
|
||||
self.next_cache_check = time.time() + 30
|
||||
elif time.perf_counter() > self.next_cache_check:
|
||||
await self.flush(True)
|
||||
self.next_cache_check = time.perf_counter() + 30
|
||||
|
||||
def check_cache_size(self):
|
||||
"""Flush a cache if it gets too big."""
|
||||
@@ -376,7 +408,8 @@ class BlockProcessor:
|
||||
for block in blocks:
|
||||
height += 1
|
||||
undo_info = self.advance_txs(
|
||||
height, block.transactions, self.coin.electrum_header(block.header, height)
|
||||
height, block.transactions, self.coin.electrum_header(block.header, height),
|
||||
self.coin.header_hash(block.header)
|
||||
)
|
||||
if height >= min_height:
|
||||
self.undo_infos.append((undo_info, height))
|
||||
@@ -387,25 +420,26 @@ class BlockProcessor:
|
||||
self.headers.extend(headers)
|
||||
self.tip = self.coin.header_hash(headers[-1])
|
||||
|
||||
def advance_txs(self, height, txs, header):
|
||||
self.tx_hashes.append(b''.join(tx_hash for tx, tx_hash in txs))
|
||||
def advance_txs(self, height, txs, header, block_hash):
|
||||
self.block_hashes.append(block_hash)
|
||||
self.block_txs.append((b''.join(tx_hash for tx, tx_hash in txs), [tx.raw for tx, _ in txs]))
|
||||
|
||||
# Use local vars for speed in the loops
|
||||
undo_info = []
|
||||
tx_num = self.tx_count
|
||||
script_hashX = self.coin.hashX_from_script
|
||||
s_pack = pack
|
||||
hashXs_by_tx = []
|
||||
|
||||
# Use local vars for speed in the loops
|
||||
put_utxo = self.utxo_cache.__setitem__
|
||||
spend_utxo = self.spend_utxo
|
||||
undo_info_append = undo_info.append
|
||||
update_touched = self.touched.update
|
||||
hashXs_by_tx = []
|
||||
append_hashXs = hashXs_by_tx.append
|
||||
append_hashX_by_tx = hashXs_by_tx.append
|
||||
hashX_from_script = self.coin.hashX_from_script
|
||||
|
||||
for tx, tx_hash in txs:
|
||||
hashXs = []
|
||||
append_hashX = hashXs.append
|
||||
tx_numb = s_pack('<I', tx_num)
|
||||
tx_numb = pack('<I', tx_num)
|
||||
|
||||
# Spend the inputs
|
||||
for txin in tx.inputs:
|
||||
@@ -418,18 +452,17 @@ class BlockProcessor:
|
||||
# Add the new UTXOs
|
||||
for idx, txout in enumerate(tx.outputs):
|
||||
# Get the hashX. Ignore unspendable outputs
|
||||
hashX = script_hashX(txout.pk_script)
|
||||
hashX = hashX_from_script(txout.pk_script)
|
||||
if hashX:
|
||||
append_hashX(hashX)
|
||||
put_utxo(tx_hash + s_pack('<H', idx),
|
||||
hashX + tx_numb + s_pack('<Q', txout.value))
|
||||
put_utxo(tx_hash + pack('<H', idx), hashX + tx_numb + pack('<Q', txout.value))
|
||||
|
||||
append_hashXs(hashXs)
|
||||
append_hashX_by_tx(hashXs)
|
||||
update_touched(hashXs)
|
||||
self.db.total_transactions.append(tx_hash)
|
||||
tx_num += 1
|
||||
|
||||
self.db.history.add_unflushed(hashXs_by_tx, self.tx_count)
|
||||
|
||||
self.tx_count = tx_num
|
||||
self.db.tx_counts.append(tx_num)
|
||||
|
||||
@@ -446,6 +479,7 @@ class BlockProcessor:
|
||||
|
||||
coin = self.coin
|
||||
for raw_block in raw_blocks:
|
||||
self.logger.info("backup block %i", self.height)
|
||||
# Check and update self.tip
|
||||
block = coin.block(raw_block, self.height)
|
||||
header_hash = coin.header_hash(block.header)
|
||||
@@ -471,20 +505,16 @@ class BlockProcessor:
|
||||
|
||||
# Use local vars for speed in the loops
|
||||
s_pack = pack
|
||||
put_utxo = self.utxo_cache.__setitem__
|
||||
spend_utxo = self.spend_utxo
|
||||
script_hashX = self.coin.hashX_from_script
|
||||
touched = self.touched
|
||||
undo_entry_len = 12 + HASHX_LEN
|
||||
|
||||
for tx, tx_hash in reversed(txs):
|
||||
for idx, txout in enumerate(tx.outputs):
|
||||
# Spend the TX outputs. Be careful with unspendable
|
||||
# outputs - we didn't save those in the first place.
|
||||
hashX = script_hashX(txout.pk_script)
|
||||
hashX = self.coin.hashX_from_script(txout.pk_script)
|
||||
if hashX:
|
||||
cache_value = spend_utxo(tx_hash, idx)
|
||||
touched.add(cache_value[:-12])
|
||||
cache_value = self.spend_utxo(tx_hash, idx)
|
||||
self.touched.add(cache_value[:-12])
|
||||
|
||||
# Restore the inputs
|
||||
for txin in reversed(tx.inputs):
|
||||
@@ -492,9 +522,10 @@ class BlockProcessor:
|
||||
continue
|
||||
n -= undo_entry_len
|
||||
undo_item = undo_info[n:n + undo_entry_len]
|
||||
put_utxo(txin.prev_hash + s_pack('<H', txin.prev_idx),
|
||||
undo_item)
|
||||
touched.add(undo_item[:-12])
|
||||
self.utxo_cache[txin.prev_hash + s_pack('<H', txin.prev_idx)] = undo_item
|
||||
self.touched.add(undo_item[:-12])
|
||||
|
||||
self.db.total_transactions.pop()
|
||||
|
||||
assert n == 0
|
||||
self.tx_count -= len(txs)
|
||||
@@ -573,13 +604,17 @@ class BlockProcessor:
|
||||
prefix = b'h' + tx_hash[:4] + idx_packed
|
||||
candidates = {db_key: hashX for db_key, hashX
|
||||
in self.db.utxo_db.iterator(prefix=prefix)}
|
||||
|
||||
for hdb_key, hashX in candidates.items():
|
||||
tx_num_packed = hdb_key[-4:]
|
||||
|
||||
if len(candidates) > 1:
|
||||
|
||||
tx_num, = unpack('<I', tx_num_packed)
|
||||
hash, height = self.db.fs_tx_hash(tx_num)
|
||||
try:
|
||||
hash, height = self.db.fs_tx_hash(tx_num)
|
||||
except IndexError:
|
||||
self.logger.error("data integrity error for hashx history: %s missing tx #%s (%s:%s)",
|
||||
hashX.hex(), tx_num, hash_to_hex_str(tx_hash), tx_idx)
|
||||
continue
|
||||
if hash != tx_hash:
|
||||
assert hash is not None # Should always be found
|
||||
continue
|
||||
@@ -588,12 +623,17 @@ class BlockProcessor:
|
||||
# Value: the UTXO value as a 64-bit unsigned integer
|
||||
udb_key = b'u' + hashX + hdb_key[-6:]
|
||||
utxo_value_packed = self.db.utxo_db.get(udb_key)
|
||||
if utxo_value_packed:
|
||||
# Remove both entries for this UTXO
|
||||
self.db_deletes.append(hdb_key)
|
||||
self.db_deletes.append(udb_key)
|
||||
return hashX + tx_num_packed + utxo_value_packed
|
||||
if utxo_value_packed is None:
|
||||
self.logger.warning(
|
||||
"%s:%s is not found in UTXO db for %s", hash_to_hex_str(tx_hash), tx_idx, hash_to_hex_str(hashX)
|
||||
)
|
||||
raise ChainError(f"{hash_to_hex_str(tx_hash)}:{tx_idx} is not found in UTXO db for {hash_to_hex_str(hashX)}")
|
||||
# Remove both entries for this UTXO
|
||||
self.db_deletes.append(hdb_key)
|
||||
self.db_deletes.append(udb_key)
|
||||
return hashX + tx_num_packed + utxo_value_packed
|
||||
|
||||
self.logger.error('UTXO {hash_to_hex_str(tx_hash)} / {tx_idx} not found in "h" table')
|
||||
raise ChainError('UTXO {} / {:,d} not found in "h" table'
|
||||
.format(hash_to_hex_str(tx_hash), tx_idx))
|
||||
|
||||
@@ -606,7 +646,7 @@ class BlockProcessor:
|
||||
self._caught_up_event.set()
|
||||
await self.blocks_event.wait()
|
||||
self.blocks_event.clear()
|
||||
if self.reorg_count:
|
||||
if self.reorg_count: # this could only happen by calling the reorg rpc
|
||||
await self.reorg_chain(self.reorg_count)
|
||||
self.reorg_count = 0
|
||||
else:
|
||||
@@ -645,9 +685,11 @@ class BlockProcessor:
|
||||
disk before exiting, as otherwise a significant amount of work
|
||||
could be lost.
|
||||
"""
|
||||
|
||||
self._caught_up_event = caught_up_event
|
||||
try:
|
||||
await self._first_open_dbs()
|
||||
self.status_server.set_height(self.db.fs_height, self.db.db_tip)
|
||||
await asyncio.wait([
|
||||
self.prefetcher.main_loop(self.height),
|
||||
self._process_prefetched_blocks()
|
||||
@@ -658,6 +700,7 @@ class BlockProcessor:
|
||||
self.logger.exception("Block processing failed!")
|
||||
raise
|
||||
finally:
|
||||
self.status_server.stop()
|
||||
# Shut down block processing
|
||||
self.logger.info('flushing to DB for a clean shutdown...')
|
||||
await self.flush(True)
|
||||
@@ -677,7 +720,6 @@ class BlockProcessor:
|
||||
|
||||
|
||||
class Timer:
|
||||
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
self.total = 0
|
||||
@@ -752,10 +794,11 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
self.timer.run(self.sql.execute, self.sql.SEARCH_INDEXES, timer_name='executing SEARCH_INDEXES')
|
||||
if self.env.individual_tag_indexes:
|
||||
self.timer.run(self.sql.execute, self.sql.TAG_INDEXES, timer_name='executing TAG_INDEXES')
|
||||
self.timer.run(self.sql.execute, self.sql.LANGUAGE_INDEXES, timer_name='executing LANGUAGE_INDEXES')
|
||||
|
||||
def advance_txs(self, height, txs, header):
|
||||
def advance_txs(self, height, txs, header, block_hash):
|
||||
timer = self.timer.sub_timers['advance_blocks']
|
||||
undo = timer.run(super().advance_txs, height, txs, header, timer_name='super().advance_txs')
|
||||
undo = timer.run(super().advance_txs, height, txs, header, block_hash, timer_name='super().advance_txs')
|
||||
timer.run(self.sql.advance_txs, height, txs, header, self.daemon.cached_height(), forward_timer=True)
|
||||
if (height % 10000 == 0 or not self.db.first_sync) and self.logger.isEnabledFor(10):
|
||||
self.timer.show(height=height)
|
||||
|
||||
@@ -3,14 +3,14 @@ import itertools
|
||||
import json
|
||||
import time
|
||||
from functools import wraps
|
||||
from pylru import lrucache
|
||||
|
||||
import aiohttp
|
||||
|
||||
from prometheus_client import Gauge, Histogram
|
||||
from lbry.utils import LRUCacheWithMetrics
|
||||
from lbry.wallet.rpc.jsonrpc import RPCError
|
||||
from lbry.wallet.server.util import hex_to_bytes, class_logger
|
||||
from lbry.wallet.rpc import JSONRPC
|
||||
from lbry.wallet.server.prometheus import LBRYCRD_REQUEST_TIMES, LBRYCRD_PENDING_COUNT
|
||||
|
||||
|
||||
class DaemonError(Exception):
|
||||
"""Raised when the daemon returns an error in its results."""
|
||||
@@ -24,12 +24,23 @@ class WorkQueueFullError(Exception):
|
||||
"""Internal - when the daemon's work queue is full."""
|
||||
|
||||
|
||||
NAMESPACE = "wallet_server"
|
||||
|
||||
|
||||
class Daemon:
|
||||
"""Handles connections to a daemon at the given URL."""
|
||||
|
||||
WARMING_UP = -28
|
||||
id_counter = itertools.count()
|
||||
|
||||
lbrycrd_request_time_metric = Histogram(
|
||||
"lbrycrd_request", "lbrycrd requests count", namespace=NAMESPACE, labelnames=("method",)
|
||||
)
|
||||
lbrycrd_pending_count_metric = Gauge(
|
||||
"lbrycrd_pending_count", "Number of lbrycrd rpcs that are in flight", namespace=NAMESPACE,
|
||||
labelnames=("method",)
|
||||
)
|
||||
|
||||
def __init__(self, coin, url, max_workqueue=10, init_retry=0.25,
|
||||
max_retry=4.0):
|
||||
self.coin = coin
|
||||
@@ -43,8 +54,8 @@ class Daemon:
|
||||
self._height = None
|
||||
self.available_rpcs = {}
|
||||
self.connector = aiohttp.TCPConnector()
|
||||
self._block_hash_cache = lrucache(100000)
|
||||
self._block_cache = lrucache(10000)
|
||||
self._block_hash_cache = LRUCacheWithMetrics(100000)
|
||||
self._block_cache = LRUCacheWithMetrics(2 ** 16, metric_name='block', namespace=NAMESPACE)
|
||||
|
||||
async def close(self):
|
||||
if self.connector:
|
||||
@@ -129,7 +140,7 @@ class Daemon:
|
||||
while True:
|
||||
try:
|
||||
for method in methods:
|
||||
LBRYCRD_PENDING_COUNT.labels(method=method).inc()
|
||||
self.lbrycrd_pending_count_metric.labels(method=method).inc()
|
||||
result = await self._send_data(data)
|
||||
result = processor(result)
|
||||
if on_good_message:
|
||||
@@ -154,7 +165,7 @@ class Daemon:
|
||||
on_good_message = 'running normally'
|
||||
finally:
|
||||
for method in methods:
|
||||
LBRYCRD_PENDING_COUNT.labels(method=method).dec()
|
||||
self.lbrycrd_pending_count_metric.labels(method=method).dec()
|
||||
await asyncio.sleep(retry)
|
||||
retry = max(min(self.max_retry, retry * 2), self.init_retry)
|
||||
|
||||
@@ -175,7 +186,7 @@ class Daemon:
|
||||
if params:
|
||||
payload['params'] = params
|
||||
result = await self._send(payload, processor)
|
||||
LBRYCRD_REQUEST_TIMES.labels(method=method).observe(time.perf_counter() - start)
|
||||
self.lbrycrd_request_time_metric.labels(method=method).observe(time.perf_counter() - start)
|
||||
return result
|
||||
|
||||
async def _send_vector(self, method, params_iterable, replace_errs=False):
|
||||
@@ -200,7 +211,7 @@ class Daemon:
|
||||
result = []
|
||||
if payload:
|
||||
result = await self._send(payload, processor)
|
||||
LBRYCRD_REQUEST_TIMES.labels(method=method).observe(time.perf_counter()-start)
|
||||
self.lbrycrd_request_time_metric.labels(method=method).observe(time.perf_counter() - start)
|
||||
return result
|
||||
|
||||
async def _is_rpc_available(self, method):
|
||||
@@ -238,8 +249,10 @@ class Daemon:
|
||||
|
||||
async def deserialised_block(self, hex_hash):
|
||||
"""Return the deserialised block with the given hex hash."""
|
||||
if not self._block_cache.get(hex_hash):
|
||||
self._block_cache[hex_hash] = await self._send_single('getblock', (hex_hash, True))
|
||||
if hex_hash not in self._block_cache:
|
||||
block = await self._send_single('getblock', (hex_hash, True))
|
||||
self._block_cache[hex_hash] = block
|
||||
return block
|
||||
return self._block_cache[hex_hash]
|
||||
|
||||
async def raw_blocks(self, hex_hashes):
|
||||
|
||||
+401
-202
@@ -1,7 +1,8 @@
|
||||
CLAIM_TYPES = {
|
||||
'stream': 1,
|
||||
'channel': 2,
|
||||
'repost': 3
|
||||
'repost': 3,
|
||||
'collection': 4,
|
||||
}
|
||||
|
||||
STREAM_TYPES = {
|
||||
@@ -10,212 +11,410 @@ STREAM_TYPES = {
|
||||
'image': 3,
|
||||
'document': 4,
|
||||
'binary': 5,
|
||||
'model': 6
|
||||
'model': 6,
|
||||
}
|
||||
|
||||
# 9/21/2020
|
||||
MOST_USED_TAGS = {
|
||||
"gaming",
|
||||
"people & blogs",
|
||||
"entertainment",
|
||||
"music",
|
||||
"pop culture",
|
||||
"education",
|
||||
"technology",
|
||||
"blockchain",
|
||||
"news",
|
||||
"funny",
|
||||
"science & technology",
|
||||
"learning",
|
||||
"gameplay",
|
||||
"news & politics",
|
||||
"comedy",
|
||||
"bitcoin",
|
||||
"beliefs",
|
||||
"nature",
|
||||
"art",
|
||||
"economics",
|
||||
"film & animation",
|
||||
"lets play",
|
||||
"games",
|
||||
"sports",
|
||||
"howto & style",
|
||||
"game",
|
||||
"cryptocurrency",
|
||||
"playstation 4",
|
||||
"automotive",
|
||||
"crypto",
|
||||
"mature",
|
||||
"sony interactive entertainment",
|
||||
"walkthrough",
|
||||
"tutorial",
|
||||
"video game",
|
||||
"weapons",
|
||||
"playthrough",
|
||||
"pc",
|
||||
"anime",
|
||||
"how to",
|
||||
"btc",
|
||||
"fun",
|
||||
"ethereum",
|
||||
"food",
|
||||
"travel & events",
|
||||
"minecraft",
|
||||
"science",
|
||||
"autos & vehicles",
|
||||
"play",
|
||||
"politics",
|
||||
"commentary",
|
||||
"twitch",
|
||||
"ps4live",
|
||||
"love",
|
||||
"ps4",
|
||||
"nonprofits & activism",
|
||||
"ps4share",
|
||||
"fortnite",
|
||||
"xbox",
|
||||
"porn",
|
||||
"video games",
|
||||
"trump",
|
||||
"español",
|
||||
"money",
|
||||
"music video",
|
||||
"nintendo",
|
||||
"movie",
|
||||
"coronavirus",
|
||||
"donald trump",
|
||||
"steam",
|
||||
"trailer",
|
||||
"android",
|
||||
"podcast",
|
||||
"xbox one",
|
||||
"survival",
|
||||
"audio",
|
||||
"linux",
|
||||
"travel",
|
||||
"funny moments",
|
||||
"litecoin",
|
||||
"animation",
|
||||
"gamer",
|
||||
"lets",
|
||||
"playstation",
|
||||
"bitcoin news",
|
||||
"history",
|
||||
"xxx",
|
||||
"fox news",
|
||||
"dance",
|
||||
"god",
|
||||
"adventure",
|
||||
"liberal",
|
||||
"2020",
|
||||
"horror",
|
||||
"government",
|
||||
"freedom",
|
||||
"reaction",
|
||||
"meme",
|
||||
"photography",
|
||||
"truth",
|
||||
"health",
|
||||
"lbry",
|
||||
"family",
|
||||
"online",
|
||||
"eth",
|
||||
"crypto news",
|
||||
"diy",
|
||||
"trading",
|
||||
"gold",
|
||||
"memes",
|
||||
"world",
|
||||
"space",
|
||||
"lol",
|
||||
"covid-19",
|
||||
"rpg",
|
||||
"humor",
|
||||
"democrat",
|
||||
"film",
|
||||
"call of duty",
|
||||
"tech",
|
||||
"religion",
|
||||
"conspiracy",
|
||||
"rap",
|
||||
"cnn",
|
||||
"hangoutsonair",
|
||||
"unboxing",
|
||||
"fiction",
|
||||
"conservative",
|
||||
"cars",
|
||||
"hoa",
|
||||
"epic",
|
||||
"programming",
|
||||
"progressive",
|
||||
"cryptocurrency news",
|
||||
"classical",
|
||||
"jesus",
|
||||
"movies",
|
||||
"book",
|
||||
"ps3",
|
||||
"republican",
|
||||
"fitness",
|
||||
"books",
|
||||
"multiplayer",
|
||||
"animals",
|
||||
"pokemon",
|
||||
"bitcoin price",
|
||||
"facebook",
|
||||
"sharefactory",
|
||||
"criptomonedas",
|
||||
"cod",
|
||||
"bible",
|
||||
"business",
|
||||
"stream",
|
||||
"comics",
|
||||
"how",
|
||||
"fail",
|
||||
"nsfw",
|
||||
"new music",
|
||||
"satire",
|
||||
"pets & animals",
|
||||
"computer",
|
||||
"classical music",
|
||||
"indie",
|
||||
"musica",
|
||||
"msnbc",
|
||||
"fps",
|
||||
"mod",
|
||||
"sport",
|
||||
"sony",
|
||||
"ripple",
|
||||
"auto",
|
||||
"rock",
|
||||
"marvel",
|
||||
"complete",
|
||||
"mining",
|
||||
"political",
|
||||
"mobile",
|
||||
"pubg",
|
||||
"hip hop",
|
||||
"flat earth",
|
||||
"xbox 360",
|
||||
"reviews",
|
||||
"vlogging",
|
||||
"latest news",
|
||||
"hack",
|
||||
"tarot",
|
||||
"iphone",
|
||||
"media",
|
||||
"cute",
|
||||
"christian",
|
||||
"free speech",
|
||||
"trap",
|
||||
"war",
|
||||
"remix",
|
||||
"ios",
|
||||
"xrp",
|
||||
"spirituality",
|
||||
"song",
|
||||
"league of legends",
|
||||
"cat"
|
||||
}
|
||||
|
||||
MATURE_TAGS = [
|
||||
'nsfw', 'porn', 'xxx', 'mature', 'adult', 'sex'
|
||||
]
|
||||
|
||||
|
||||
def normalize_tag(tag):
|
||||
return tag.replace(" ", "_").replace("&", "and").replace("-", "_")
|
||||
|
||||
|
||||
COMMON_TAGS = {
|
||||
"gaming": "gaming",
|
||||
"people & blogs": "people_and_blogs",
|
||||
"pop culture": "pop_culture",
|
||||
"entertainment": "entertainment",
|
||||
"technology": "technology",
|
||||
"music": "music",
|
||||
"funny": "funny",
|
||||
"education": "education",
|
||||
"learning": "learning",
|
||||
"news": "news",
|
||||
"gameplay": "gameplay",
|
||||
"science & technology": "science_and_technology",
|
||||
"playstation 4": "playstation_4",
|
||||
"beliefs": "beliefs",
|
||||
"nature": "nature",
|
||||
"news & politics": "news_and_politics",
|
||||
"comedy": "comedy",
|
||||
"games": "games",
|
||||
"sony interactive entertainment": "sony_interactive_entertainment",
|
||||
"film & animation": "film_and_animation",
|
||||
"game": "game",
|
||||
"howto & style": "howto_and_style",
|
||||
"weapons": "weapons",
|
||||
"blockchain": "blockchain",
|
||||
"video game": "video_game",
|
||||
"sports": "sports",
|
||||
"walkthrough": "walkthrough",
|
||||
"ps4live": "ps4live",
|
||||
"art": "art",
|
||||
"pc": "pc",
|
||||
"economics": "economics",
|
||||
"automotive": "automotive",
|
||||
"minecraft": "minecraft",
|
||||
"playthrough": "playthrough",
|
||||
"ps4share": "ps4share",
|
||||
"tutorial": "tutorial",
|
||||
"play": "play",
|
||||
"twitch": "twitch",
|
||||
"how to": "how_to",
|
||||
"ps4": "ps4",
|
||||
"bitcoin": "bitcoin",
|
||||
"fortnite": "fortnite",
|
||||
"commentary": "commentary",
|
||||
"lets play": "lets_play",
|
||||
"fun": "fun",
|
||||
"politics": "politics",
|
||||
"xbox": "xbox",
|
||||
"autos & vehicles": "autos_and_vehicles",
|
||||
"travel & events": "travel_and_events",
|
||||
"food": "food",
|
||||
"science": "science",
|
||||
"mature": "mature",
|
||||
"xbox one": "xbox_one",
|
||||
"liberal": "liberal",
|
||||
"democrat": "democrat",
|
||||
"progressive": "progressive",
|
||||
"survival": "survival",
|
||||
"nonprofits & activism": "nonprofits_and_activism",
|
||||
"cryptocurrency": "cryptocurrency",
|
||||
"playstation": "playstation",
|
||||
"nintendo": "nintendo",
|
||||
"government": "government",
|
||||
"steam": "steam",
|
||||
"podcast": "podcast",
|
||||
"horror": "horror",
|
||||
"conservative": "conservative",
|
||||
"reaction": "reaction",
|
||||
"trailer": "trailer",
|
||||
"love": "love",
|
||||
"cnn": "cnn",
|
||||
"republican": "republican",
|
||||
"gamer": "gamer",
|
||||
"political": "political",
|
||||
"hangoutsonair": "hangoutsonair",
|
||||
"hoa": "hoa",
|
||||
"msnbc": "msnbc",
|
||||
"cbs": "cbs",
|
||||
"donald trump": "donald_trump",
|
||||
"fiction": "fiction",
|
||||
"fox news": "fox_news",
|
||||
"anime": "anime",
|
||||
"crypto": "crypto",
|
||||
"ethereum": "ethereum",
|
||||
"call of duty": "call_of_duty",
|
||||
"multiplayer": "multiplayer",
|
||||
"android": "android",
|
||||
"epic": "epic",
|
||||
"rpg": "rpg",
|
||||
"adventure": "adventure",
|
||||
"secular talk": "secular_talk",
|
||||
"btc": "btc",
|
||||
"atheist": "atheist",
|
||||
"atheism": "atheism",
|
||||
"ps3": "ps3",
|
||||
"video games": "video_games",
|
||||
"cod": "cod",
|
||||
"agnostic": "agnostic",
|
||||
"movie": "movie",
|
||||
"online": "online",
|
||||
"fps": "fps",
|
||||
"mod": "mod",
|
||||
"reviews": "reviews",
|
||||
"sharefactory": "sharefactory",
|
||||
"world": "world",
|
||||
"space": "space",
|
||||
"hilarious": "hilarious",
|
||||
"stream": "stream",
|
||||
"lol": "lol",
|
||||
"sony": "sony",
|
||||
"god": "god",
|
||||
"lets": "lets",
|
||||
"dance": "dance",
|
||||
"pvp": "pvp",
|
||||
"tech": "tech",
|
||||
"zombies": "zombies",
|
||||
"pokemon": "pokemon",
|
||||
"fail": "fail",
|
||||
"xbox 360": "xbox_360",
|
||||
"film": "film",
|
||||
"unboxing": "unboxing",
|
||||
"animation": "animation",
|
||||
"travel": "travel",
|
||||
"money": "money",
|
||||
"wwe": "wwe",
|
||||
"how": "how",
|
||||
"mods": "mods",
|
||||
"pubg": "pubg",
|
||||
"indie": "indie",
|
||||
"strategy": "strategy",
|
||||
"history": "history",
|
||||
"rap": "rap",
|
||||
"ios": "ios",
|
||||
"sony computer entertainment": "sony_computer_entertainment",
|
||||
"mobile": "mobile",
|
||||
"trump": "trump",
|
||||
"flat earth": "flat_earth",
|
||||
"hack": "hack",
|
||||
"trap": "trap",
|
||||
"fox": "fox",
|
||||
"vlogging": "vlogging",
|
||||
"news radio": "news_radio",
|
||||
"humor": "humor",
|
||||
"facebook": "facebook",
|
||||
"edm": "edm",
|
||||
"fitness": "fitness",
|
||||
"vaping": "vaping",
|
||||
"hip hop": "hip_hop",
|
||||
"secular": "secular",
|
||||
"jesus": "jesus",
|
||||
"vape": "vape",
|
||||
"song": "song",
|
||||
"remix": "remix",
|
||||
"guitar": "guitar",
|
||||
"daily": "daily",
|
||||
"mining": "mining",
|
||||
"diy": "diy",
|
||||
"videogame": "videogame",
|
||||
"pets & animals": "pets_and_animals",
|
||||
"funny moments": "funny_moments",
|
||||
"religion": "religion",
|
||||
"death": "death",
|
||||
"media": "media",
|
||||
"nbc": "nbc",
|
||||
"war": "war",
|
||||
"freedom": "freedom",
|
||||
"viral": "viral",
|
||||
"meme": "meme",
|
||||
"family": "family",
|
||||
"gold": "gold",
|
||||
"photography": "photography",
|
||||
"chill": "chill",
|
||||
"zombie": "zombie",
|
||||
"computer": "computer",
|
||||
"sniper": "sniper",
|
||||
"bible": "bible",
|
||||
"linux": "linux",
|
||||
"overwatch": "overwatch",
|
||||
"pro": "pro",
|
||||
"dragon": "dragon",
|
||||
"litecoin": "litecoin",
|
||||
"gta": "gta",
|
||||
"iphone": "iphone",
|
||||
"house": "house",
|
||||
"bass": "bass",
|
||||
"bitcoin news": "bitcoin_news",
|
||||
"wii": "wii",
|
||||
"crash": "crash",
|
||||
"league of legends": "league_of_legends",
|
||||
"grand theft auto v": "grand_theft_auto_v",
|
||||
"mario": "mario",
|
||||
"mmorpg": "mmorpg",
|
||||
"satire": "satire",
|
||||
"fire": "fire",
|
||||
"racing": "racing",
|
||||
"apple": "apple",
|
||||
"health": "health",
|
||||
"instrumental": "instrumental",
|
||||
"destiny": "destiny",
|
||||
"truth": "truth",
|
||||
"race": "race"
|
||||
tag: normalize_tag(tag) for tag in list(MOST_USED_TAGS)
|
||||
}
|
||||
|
||||
INDEXED_LANGUAGES = [
|
||||
'none',
|
||||
'en',
|
||||
'aa',
|
||||
'ab',
|
||||
'ae',
|
||||
'af',
|
||||
'ak',
|
||||
'am',
|
||||
'an',
|
||||
'ar',
|
||||
'as',
|
||||
'av',
|
||||
'ay',
|
||||
'az',
|
||||
'ba',
|
||||
'be',
|
||||
'bg',
|
||||
'bh',
|
||||
'bi',
|
||||
'bm',
|
||||
'bn',
|
||||
'bo',
|
||||
'br',
|
||||
'bs',
|
||||
'ca',
|
||||
'ce',
|
||||
'ch',
|
||||
'co',
|
||||
'cr',
|
||||
'cs',
|
||||
'cu',
|
||||
'cv',
|
||||
'cy',
|
||||
'da',
|
||||
'de',
|
||||
'dv',
|
||||
'dz',
|
||||
'ee',
|
||||
'el',
|
||||
'eo',
|
||||
'es',
|
||||
'et',
|
||||
'eu',
|
||||
'fa',
|
||||
'ff',
|
||||
'fi',
|
||||
'fj',
|
||||
'fo',
|
||||
'fr',
|
||||
'fy',
|
||||
'ga',
|
||||
'gd',
|
||||
'gl',
|
||||
'gn',
|
||||
'gu',
|
||||
'gv',
|
||||
'ha',
|
||||
'he',
|
||||
'hi',
|
||||
'ho',
|
||||
'hr',
|
||||
'ht',
|
||||
'hu',
|
||||
'hy',
|
||||
'hz',
|
||||
'ia',
|
||||
'id',
|
||||
'ie',
|
||||
'ig',
|
||||
'ii',
|
||||
'ik',
|
||||
'io',
|
||||
'is',
|
||||
'it',
|
||||
'iu',
|
||||
'ja',
|
||||
'jv',
|
||||
'ka',
|
||||
'kg',
|
||||
'ki',
|
||||
'kj',
|
||||
'kk',
|
||||
'kl',
|
||||
'km',
|
||||
'kn',
|
||||
'ko',
|
||||
'kr',
|
||||
'ks',
|
||||
'ku',
|
||||
'kv',
|
||||
'kw',
|
||||
'ky',
|
||||
'la',
|
||||
'lb',
|
||||
'lg',
|
||||
'li',
|
||||
'ln',
|
||||
'lo',
|
||||
'lt',
|
||||
'lu',
|
||||
'lv',
|
||||
'mg',
|
||||
'mh',
|
||||
'mi',
|
||||
'mk',
|
||||
'ml',
|
||||
'mn',
|
||||
'mr',
|
||||
'ms',
|
||||
'mt',
|
||||
'my',
|
||||
'na',
|
||||
'nb',
|
||||
'nd',
|
||||
'ne',
|
||||
'ng',
|
||||
'nl',
|
||||
'nn',
|
||||
'no',
|
||||
'nr',
|
||||
'nv',
|
||||
'ny',
|
||||
'oc',
|
||||
'oj',
|
||||
'om',
|
||||
'or',
|
||||
'os',
|
||||
'pa',
|
||||
'pi',
|
||||
'pl',
|
||||
'ps',
|
||||
'pt',
|
||||
'qu',
|
||||
'rm',
|
||||
'rn',
|
||||
'ro',
|
||||
'ru',
|
||||
'rw',
|
||||
'sa',
|
||||
'sc',
|
||||
'sd',
|
||||
'se',
|
||||
'sg',
|
||||
'si',
|
||||
'sk',
|
||||
'sl',
|
||||
'sm',
|
||||
'sn',
|
||||
'so',
|
||||
'sq',
|
||||
'sr',
|
||||
'ss',
|
||||
'st',
|
||||
'su',
|
||||
'sv',
|
||||
'sw',
|
||||
'ta',
|
||||
'te',
|
||||
'tg',
|
||||
'th',
|
||||
'ti',
|
||||
'tk',
|
||||
'tl',
|
||||
'tn',
|
||||
'to',
|
||||
'tr',
|
||||
'ts',
|
||||
'tt',
|
||||
'tw',
|
||||
'ty',
|
||||
'ug',
|
||||
'uk',
|
||||
'ur',
|
||||
'uz',
|
||||
've',
|
||||
'vi',
|
||||
'vo',
|
||||
'wa',
|
||||
'wo',
|
||||
'xh',
|
||||
'yi',
|
||||
'yo',
|
||||
'za',
|
||||
'zh',
|
||||
'zu'
|
||||
]
|
||||
@@ -18,7 +18,7 @@ from lbry.schema.tags import clean_tags
|
||||
from lbry.schema.result import Outputs, Censor
|
||||
from lbry.wallet import Ledger, RegTestLedger
|
||||
|
||||
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
|
||||
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS, INDEXED_LANGUAGES
|
||||
from .full_text_search import FTS_ORDER_BY
|
||||
|
||||
|
||||
@@ -39,7 +39,7 @@ ATTRIBUTE_ARRAY_MAX_LENGTH = 100
|
||||
INTEGER_PARAMS = {
|
||||
'height', 'creation_height', 'activation_height', 'expiration_height',
|
||||
'timestamp', 'creation_timestamp', 'duration', 'release_time', 'fee_amount',
|
||||
'tx_position', 'channel_join', 'reposted',
|
||||
'tx_position', 'channel_join', 'reposted', 'limit_claims_per_channel',
|
||||
'amount', 'effective_amount', 'support_amount',
|
||||
'trending_group', 'trending_mixed',
|
||||
'trending_local', 'trending_global',
|
||||
@@ -96,8 +96,8 @@ class ReaderState:
|
||||
def get_resolve_censor(self) -> Censor:
|
||||
return Censor(self.blocked_streams, self.blocked_channels)
|
||||
|
||||
def get_search_censor(self) -> Censor:
|
||||
return Censor(self.filtered_streams, self.filtered_channels)
|
||||
def get_search_censor(self, limit_claims_per_channel: int) -> Censor:
|
||||
return Censor(self.filtered_streams, self.filtered_channels, limit_claims_per_channel)
|
||||
|
||||
|
||||
ctx: ContextVar[Optional[ReaderState]] = ContextVar('ctx')
|
||||
@@ -283,7 +283,7 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
|
||||
channel_ids = constraints.pop('channel_ids')
|
||||
if channel_ids:
|
||||
constraints['claim.channel_hash__in'] = {
|
||||
unhexlify(cid)[::-1] for cid in channel_ids
|
||||
unhexlify(cid)[::-1] for cid in channel_ids if cid
|
||||
}
|
||||
if 'not_channel_ids' in constraints:
|
||||
not_channel_ids = constraints.pop('not_channel_ids')
|
||||
@@ -421,12 +421,13 @@ def search(constraints) -> Tuple[List, List, int, int, Censor]:
|
||||
assert set(constraints).issubset(SEARCH_PARAMS), \
|
||||
f"Search query contains invalid arguments: {set(constraints).difference(SEARCH_PARAMS)}"
|
||||
total = None
|
||||
limit_claims_per_channel = constraints.pop('limit_claims_per_channel', None)
|
||||
if not constraints.pop('no_totals', False):
|
||||
total = count_claims(**constraints)
|
||||
constraints['offset'] = abs(constraints.get('offset', 0))
|
||||
constraints['limit'] = min(abs(constraints.get('limit', 10)), 50)
|
||||
context = ctx.get()
|
||||
search_censor = context.get_search_censor()
|
||||
search_censor = context.get_search_censor(limit_claims_per_channel)
|
||||
txo_rows = search_claims(search_censor, **constraints)
|
||||
extra_txo_rows = _get_referenced_rows(txo_rows, search_censor.censored.keys())
|
||||
return txo_rows, extra_txo_rows, constraints['offset'], total, search_censor
|
||||
@@ -535,6 +536,18 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
|
||||
AND tag IN ({values})
|
||||
)
|
||||
"""
|
||||
elif attr == 'language':
|
||||
indexed_languages = any_items & set(INDEXED_LANGUAGES)
|
||||
if indexed_languages:
|
||||
any_items -= indexed_languages
|
||||
for language in indexed_languages:
|
||||
any_queries[f'#_any_common_languages_{language}'] = f"""
|
||||
EXISTS(
|
||||
SELECT 1 FROM language INDEXED BY language_{language}_idx
|
||||
WHERE {CLAIM_HASH_OR_REPOST_HASH_SQL}=language.claim_hash
|
||||
AND language = '{language}'
|
||||
)
|
||||
"""
|
||||
|
||||
if any_items:
|
||||
|
||||
@@ -547,10 +560,10 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
|
||||
if for_count or attr == 'tag':
|
||||
if attr == 'tag':
|
||||
any_queries[f'#_any_{attr}'] = f"""
|
||||
(claim.claim_type != {CLAIM_TYPES['repost']}
|
||||
((claim.claim_type != {CLAIM_TYPES['repost']}
|
||||
AND claim.claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) OR
|
||||
(claim.claim_type == {CLAIM_TYPES['repost']} AND
|
||||
claim.reposted_claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values})))
|
||||
claim.reposted_claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))))
|
||||
"""
|
||||
else:
|
||||
any_queries[f'#_any_{attr}'] = f"""
|
||||
@@ -606,10 +619,10 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
|
||||
if for_count:
|
||||
if attr == 'tag':
|
||||
constraints[f'#_not_{attr}'] = f"""
|
||||
(claim.claim_type != {CLAIM_TYPES['repost']}
|
||||
AND claim.claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) AND
|
||||
((claim.claim_type != {CLAIM_TYPES['repost']}
|
||||
AND claim.claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) OR
|
||||
(claim.claim_type == {CLAIM_TYPES['repost']} AND
|
||||
claim.reposted_claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values})))
|
||||
claim.reposted_claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))))
|
||||
"""
|
||||
else:
|
||||
constraints[f'#_not_{attr}'] = f"""
|
||||
|
||||
@@ -1,18 +1,17 @@
|
||||
"""
|
||||
Delayed AR with variable decay rate.
|
||||
|
||||
The spike height function is also simpler.
|
||||
AR-like trending with a delayed effect and a faster
|
||||
decay rate for high valued claims.
|
||||
"""
|
||||
|
||||
import copy
|
||||
import math
|
||||
import time
|
||||
import apsw
|
||||
|
||||
# Half life in blocks *for lower LBC claims* (it's shorter for whale claims)
|
||||
HALF_LIFE = 200
|
||||
|
||||
# Whale threshold (higher -> less DB writing)
|
||||
WHALE_THRESHOLD = 3.0
|
||||
# Whale threshold, in LBC (higher -> less DB writing)
|
||||
WHALE_THRESHOLD = 10000.0
|
||||
|
||||
# Decay coefficient per block
|
||||
DECAY = 0.5**(1.0/HALF_LIFE)
|
||||
@@ -38,6 +37,7 @@ def install(connection):
|
||||
Install the trending algorithm.
|
||||
"""
|
||||
check_trending_values(connection)
|
||||
trending_data.initialise(connection.cursor())
|
||||
|
||||
if TRENDING_LOG:
|
||||
f = open("trending_variable_decay.log", "a")
|
||||
@@ -46,7 +46,6 @@ def install(connection):
|
||||
# Stub
|
||||
CREATE_TREND_TABLE = ""
|
||||
|
||||
|
||||
def check_trending_values(connection):
|
||||
"""
|
||||
If the trending values appear to be based on the zscore algorithm,
|
||||
@@ -60,19 +59,304 @@ def check_trending_values(connection):
|
||||
break
|
||||
|
||||
if needs_reset:
|
||||
print("Resetting some columns. This might take a while...", flush=True, end="")
|
||||
print("Resetting some columns. This might take a while...", flush=True,
|
||||
end="")
|
||||
c.execute(""" BEGIN;
|
||||
UPDATE claim SET trending_group = 0;
|
||||
UPDATE claim SET trending_mixed = 0;
|
||||
UPDATE claim SET trending_global = 0;
|
||||
UPDATE claim SET trending_local = 0;
|
||||
COMMIT;""")
|
||||
print("done.")
|
||||
|
||||
|
||||
def spike_height(x, x_old):
|
||||
|
||||
|
||||
def trending_log(s):
|
||||
"""
|
||||
Compute the size of a trending spike (normed - constant units).
|
||||
Log a string to the log file
|
||||
"""
|
||||
if TRENDING_LOG:
|
||||
fout = open("trending_variable_decay.log", "a")
|
||||
fout.write(s)
|
||||
fout.flush()
|
||||
fout.close()
|
||||
|
||||
|
||||
def trending_unit(height):
|
||||
"""
|
||||
Return the trending score unit at a given height.
|
||||
"""
|
||||
# Round to the beginning of a SAVE_INTERVAL batch of blocks.
|
||||
_height = height - (height % SAVE_INTERVAL)
|
||||
return 1.0/DECAY**(height % RENORM_INTERVAL)
|
||||
|
||||
|
||||
class TrendingDB:
|
||||
"""
|
||||
An in-memory database of trending scores
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.conn = apsw.Connection(":memory:")
|
||||
self.cursor = self.conn.cursor()
|
||||
self.initialised = False
|
||||
self.write_needed = set()
|
||||
|
||||
def execute(self, query, *args, **kwargs):
|
||||
return self.cursor.execute(query, *args, **kwargs)
|
||||
|
||||
def executemany(self, query, *args, **kwargs):
|
||||
return self.cursor.executemany(query, *args, **kwargs)
|
||||
|
||||
def begin(self):
|
||||
self.execute("BEGIN;")
|
||||
|
||||
def commit(self):
|
||||
self.execute("COMMIT;")
|
||||
|
||||
def initialise(self, db):
|
||||
"""
|
||||
Pass in claims.db
|
||||
"""
|
||||
if self.initialised:
|
||||
return
|
||||
|
||||
trending_log("Initialising trending database...")
|
||||
|
||||
# The need for speed
|
||||
self.execute("PRAGMA JOURNAL_MODE=OFF;")
|
||||
self.execute("PRAGMA SYNCHRONOUS=0;")
|
||||
|
||||
self.begin()
|
||||
|
||||
# Create the tables
|
||||
self.execute("""
|
||||
CREATE TABLE IF NOT EXISTS claims
|
||||
(claim_hash BYTES PRIMARY KEY,
|
||||
lbc REAL NOT NULL DEFAULT 0.0,
|
||||
trending_score REAL NOT NULL DEFAULT 0.0)
|
||||
WITHOUT ROWID;""")
|
||||
|
||||
self.execute("""
|
||||
CREATE TABLE IF NOT EXISTS spikes
|
||||
(id INTEGER PRIMARY KEY,
|
||||
claim_hash BYTES NOT NULL,
|
||||
height INTEGER NOT NULL,
|
||||
mass REAL NOT NULL,
|
||||
FOREIGN KEY (claim_hash)
|
||||
REFERENCES claims (claim_hash));""")
|
||||
|
||||
# Clear out any existing data
|
||||
self.execute("DELETE FROM claims;")
|
||||
self.execute("DELETE FROM spikes;")
|
||||
|
||||
# Create indexes
|
||||
self.execute("CREATE INDEX idx1 ON spikes (claim_hash, height, mass);")
|
||||
self.execute("CREATE INDEX idx2 ON spikes (claim_hash, height, mass DESC);")
|
||||
self.execute("CREATE INDEX idx3 on claims (lbc DESC, claim_hash, trending_score);")
|
||||
|
||||
# Import data from claims.db
|
||||
for row in db.execute("""
|
||||
SELECT claim_hash,
|
||||
1E-8*(amount + support_amount) AS lbc,
|
||||
trending_mixed
|
||||
FROM claim;
|
||||
"""):
|
||||
self.execute("INSERT INTO claims VALUES (?, ?, ?);", row)
|
||||
self.commit()
|
||||
|
||||
self.initialised = True
|
||||
trending_log("done.\n")
|
||||
|
||||
def apply_spikes(self, height):
|
||||
"""
|
||||
Apply spikes that are due. This occurs inside a transaction.
|
||||
"""
|
||||
|
||||
spikes = []
|
||||
unit = trending_unit(height)
|
||||
for row in self.execute("""
|
||||
SELECT SUM(mass), claim_hash FROM spikes
|
||||
WHERE height = ?
|
||||
GROUP BY claim_hash;
|
||||
""", (height, )):
|
||||
spikes.append((row[0]*unit, row[1]))
|
||||
self.write_needed.add(row[1])
|
||||
|
||||
self.executemany("""
|
||||
UPDATE claims
|
||||
SET trending_score = (trending_score + ?)
|
||||
WHERE claim_hash = ?;
|
||||
""", spikes)
|
||||
self.execute("DELETE FROM spikes WHERE height = ?;", (height, ))
|
||||
|
||||
|
||||
def decay_whales(self, height):
|
||||
"""
|
||||
Occurs inside transaction.
|
||||
"""
|
||||
if height % SAVE_INTERVAL != 0:
|
||||
return
|
||||
|
||||
whales = self.execute("""
|
||||
SELECT trending_score, lbc, claim_hash
|
||||
FROM claims
|
||||
WHERE lbc >= ?;
|
||||
""", (WHALE_THRESHOLD, )).fetchall()
|
||||
whales2 = []
|
||||
for whale in whales:
|
||||
trending, lbc, claim_hash = whale
|
||||
|
||||
# Overall multiplication factor for decay rate
|
||||
# At WHALE_THRESHOLD, this is 1
|
||||
# At 10*WHALE_THRESHOLD, it is 3
|
||||
decay_rate_factor = 1.0 + 2.0*math.log10(lbc/WHALE_THRESHOLD)
|
||||
|
||||
# The -1 is because this is just the *extra* part being applied
|
||||
factor = (DECAY**SAVE_INTERVAL)**(decay_rate_factor - 1.0)
|
||||
|
||||
# Decay
|
||||
trending *= factor
|
||||
whales2.append((trending, claim_hash))
|
||||
self.write_needed.add(claim_hash)
|
||||
|
||||
self.executemany("UPDATE claims SET trending_score=? WHERE claim_hash=?;",
|
||||
whales2)
|
||||
|
||||
|
||||
def renorm(self, height):
|
||||
"""
|
||||
Renormalise trending scores. Occurs inside a transaction.
|
||||
"""
|
||||
|
||||
if height % RENORM_INTERVAL == 0:
|
||||
threshold = 1.0E-3/DECAY_PER_RENORM
|
||||
for row in self.execute("""SELECT claim_hash FROM claims
|
||||
WHERE ABS(trending_score) >= ?;""",
|
||||
(threshold, )):
|
||||
self.write_needed.add(row[0])
|
||||
|
||||
self.execute("""UPDATE claims SET trending_score = ?*trending_score
|
||||
WHERE ABS(trending_score) >= ?;""",
|
||||
(DECAY_PER_RENORM, threshold))
|
||||
|
||||
def write_to_claims_db(self, db, height):
|
||||
"""
|
||||
Write changed trending scores to claims.db.
|
||||
"""
|
||||
if height % SAVE_INTERVAL != 0:
|
||||
return
|
||||
|
||||
rows = self.execute(f"""
|
||||
SELECT trending_score, claim_hash
|
||||
FROM claims
|
||||
WHERE claim_hash IN
|
||||
({','.join('?' for _ in self.write_needed)});
|
||||
""", self.write_needed).fetchall()
|
||||
|
||||
db.executemany("""UPDATE claim SET trending_mixed = ?
|
||||
WHERE claim_hash = ?;""", rows)
|
||||
|
||||
# Clear list of claims needing to be written to claims.db
|
||||
self.write_needed = set()
|
||||
|
||||
|
||||
def update(self, db, height, recalculate_claim_hashes):
|
||||
"""
|
||||
Update trending scores.
|
||||
Input is a cursor to claims.db, the block height, and the list of
|
||||
claims that changed.
|
||||
"""
|
||||
assert self.initialised
|
||||
|
||||
self.begin()
|
||||
self.renorm(height)
|
||||
|
||||
# Fetch changed/new claims from claims.db
|
||||
for row in db.execute(f"""
|
||||
SELECT claim_hash,
|
||||
1E-8*(amount + support_amount) AS lbc
|
||||
FROM claim
|
||||
WHERE claim_hash IN
|
||||
({','.join('?' for _ in recalculate_claim_hashes)});
|
||||
""", recalculate_claim_hashes):
|
||||
claim_hash, lbc = row
|
||||
|
||||
# Insert into trending db if it does not exist
|
||||
self.execute("""
|
||||
INSERT INTO claims (claim_hash)
|
||||
VALUES (?)
|
||||
ON CONFLICT (claim_hash) DO NOTHING;""",
|
||||
(claim_hash, ))
|
||||
|
||||
# See if it was an LBC change
|
||||
old = self.execute("SELECT * FROM claims WHERE claim_hash=?;",
|
||||
(claim_hash, )).fetchone()
|
||||
lbc_old = old[1]
|
||||
|
||||
# Save new LBC value into trending db
|
||||
self.execute("UPDATE claims SET lbc = ? WHERE claim_hash = ?;",
|
||||
(lbc, claim_hash))
|
||||
|
||||
if lbc > lbc_old:
|
||||
|
||||
# Schedule a future spike
|
||||
delay = min(int((lbc + 1E-8)**0.4), HALF_LIFE)
|
||||
spike = (claim_hash, height + delay, spike_mass(lbc, lbc_old))
|
||||
self.execute("""INSERT INTO spikes
|
||||
(claim_hash, height, mass)
|
||||
VALUES (?, ?, ?);""", spike)
|
||||
|
||||
elif lbc < lbc_old:
|
||||
|
||||
# Subtract from future spikes
|
||||
penalty = spike_mass(lbc_old, lbc)
|
||||
spikes = self.execute("""
|
||||
SELECT * FROM spikes
|
||||
WHERE claim_hash = ?
|
||||
ORDER BY height ASC, mass DESC;
|
||||
""", (claim_hash, )).fetchall()
|
||||
for spike in spikes:
|
||||
spike_id, mass = spike[0], spike[3]
|
||||
|
||||
if mass > penalty:
|
||||
# The entire penalty merely reduces this spike
|
||||
self.execute("UPDATE spikes SET mass=? WHERE id=?;",
|
||||
(mass - penalty, spike_id))
|
||||
penalty = 0.0
|
||||
else:
|
||||
# Removing this spike entirely accounts for some (or
|
||||
# all) of the penalty, then move on to other spikes
|
||||
self.execute("DELETE FROM spikes WHERE id=?;",
|
||||
(spike_id, ))
|
||||
penalty -= mass
|
||||
|
||||
# If penalty remains, that's a negative spike to be applied
|
||||
# immediately.
|
||||
if penalty > 0.0:
|
||||
self.execute("""
|
||||
INSERT INTO spikes (claim_hash, height, mass)
|
||||
VALUES (?, ?, ?);""",
|
||||
(claim_hash, height, -penalty))
|
||||
|
||||
self.apply_spikes(height)
|
||||
self.decay_whales(height)
|
||||
self.commit()
|
||||
|
||||
self.write_to_claims_db(db, height)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# The "global" instance to work with
|
||||
# pylint: disable=C0103
|
||||
trending_data = TrendingDB()
|
||||
|
||||
def spike_mass(x, x_old):
|
||||
"""
|
||||
Compute the mass of a trending spike (normed - constant units).
|
||||
x_old = old LBC value
|
||||
x = new LBC value
|
||||
"""
|
||||
|
||||
# Sign of trending spike
|
||||
@@ -89,155 +373,16 @@ def spike_height(x, x_old):
|
||||
return sign*mag
|
||||
|
||||
|
||||
def run(db, height, final_height, recalculate_claim_hashes):
|
||||
if height < final_height - 5*HALF_LIFE:
|
||||
trending_log(f"Skipping trending calculations at block {height}.\n")
|
||||
return
|
||||
|
||||
def get_time_boost(height):
|
||||
"""
|
||||
Return the time boost at a given height.
|
||||
"""
|
||||
return 1.0/DECAY**(height % RENORM_INTERVAL)
|
||||
|
||||
|
||||
def trending_log(s):
|
||||
"""
|
||||
Log a string.
|
||||
"""
|
||||
if TRENDING_LOG:
|
||||
fout = open("trending_variable_decay.log", "a")
|
||||
fout.write(s)
|
||||
fout.flush()
|
||||
fout.close()
|
||||
|
||||
class TrendingData:
|
||||
"""
|
||||
An object of this class holds trending data
|
||||
"""
|
||||
def __init__(self):
|
||||
|
||||
# Dict from claim id to some trending info.
|
||||
# Units are TIME VARIABLE in here
|
||||
self.claims = {}
|
||||
|
||||
# Claims with >= WHALE_THRESHOLD LBC total amount
|
||||
self.whales = set([])
|
||||
|
||||
# Have all claims been read from db yet?
|
||||
self.initialised = False
|
||||
|
||||
# List of pending spikes.
|
||||
# Units are CONSTANT in here
|
||||
self.pending_spikes = []
|
||||
|
||||
def insert_claim_from_load(self, height, claim_hash, trending_score, total_amount):
|
||||
assert not self.initialised
|
||||
self.claims[claim_hash] = {"trending_score": trending_score,
|
||||
"total_amount": total_amount,
|
||||
"changed": False}
|
||||
|
||||
if trending_score >= WHALE_THRESHOLD*get_time_boost(height):
|
||||
self.add_whale(claim_hash)
|
||||
|
||||
def add_whale(self, claim_hash):
|
||||
self.whales.add(claim_hash)
|
||||
|
||||
def apply_spikes(self, height):
|
||||
"""
|
||||
Apply all pending spikes that are due at this height.
|
||||
Apply with time boost ON.
|
||||
"""
|
||||
time_boost = get_time_boost(height)
|
||||
|
||||
for spike in self.pending_spikes:
|
||||
if spike["height"] > height:
|
||||
# Ignore
|
||||
pass
|
||||
if spike["height"] == height:
|
||||
# Apply
|
||||
self.claims[spike["claim_hash"]]["trending_score"] += time_boost*spike["size"]
|
||||
self.claims[spike["claim_hash"]]["changed"] = True
|
||||
|
||||
if self.claims[spike["claim_hash"]]["trending_score"] >= WHALE_THRESHOLD*time_boost:
|
||||
self.add_whale(spike["claim_hash"])
|
||||
if spike["claim_hash"] in self.whales and \
|
||||
self.claims[spike["claim_hash"]]["trending_score"] < WHALE_THRESHOLD*time_boost:
|
||||
self.whales.remove(spike["claim_hash"])
|
||||
|
||||
|
||||
# Keep only future spikes
|
||||
self.pending_spikes = [s for s in self.pending_spikes \
|
||||
if s["height"] > height]
|
||||
|
||||
|
||||
|
||||
|
||||
def update_claim(self, height, claim_hash, total_amount):
|
||||
"""
|
||||
Update trending data for a claim, given its new total amount.
|
||||
"""
|
||||
assert self.initialised
|
||||
|
||||
# Extract existing total amount and trending score
|
||||
# or use starting values if the claim is new
|
||||
if claim_hash in self.claims:
|
||||
old_state = copy.deepcopy(self.claims[claim_hash])
|
||||
else:
|
||||
old_state = {"trending_score": 0.0,
|
||||
"total_amount": 0.0,
|
||||
"changed": False}
|
||||
|
||||
# Calculate LBC change
|
||||
change = total_amount - old_state["total_amount"]
|
||||
|
||||
# Modify data if there was an LBC change
|
||||
if change != 0.0:
|
||||
spike = spike_height(total_amount,
|
||||
old_state["total_amount"])
|
||||
delay = min(int((total_amount + 1E-8)**0.4), HALF_LIFE)
|
||||
|
||||
if change < 0.0:
|
||||
|
||||
# How big would the spike be for the inverse movement?
|
||||
reverse_spike = spike_height(old_state["total_amount"], total_amount)
|
||||
|
||||
# Remove that much spike from future pending ones
|
||||
for future_spike in self.pending_spikes:
|
||||
if future_spike["claim_hash"] == claim_hash:
|
||||
if reverse_spike >= future_spike["size"]:
|
||||
reverse_spike -= future_spike["size"]
|
||||
future_spike["size"] = 0.0
|
||||
elif reverse_spike > 0.0:
|
||||
future_spike["size"] -= reverse_spike
|
||||
reverse_spike = 0.0
|
||||
|
||||
delay = 0
|
||||
spike = -reverse_spike
|
||||
|
||||
self.pending_spikes.append({"height": height + delay,
|
||||
"claim_hash": claim_hash,
|
||||
"size": spike})
|
||||
|
||||
self.claims[claim_hash] = {"total_amount": total_amount,
|
||||
"trending_score": old_state["trending_score"],
|
||||
"changed": False}
|
||||
|
||||
def process_whales(self, height):
|
||||
"""
|
||||
Whale claims decay faster.
|
||||
"""
|
||||
if height % SAVE_INTERVAL != 0:
|
||||
return
|
||||
|
||||
for claim_hash in self.whales:
|
||||
trending_normed = self.claims[claim_hash]["trending_score"]/get_time_boost(height)
|
||||
|
||||
# Overall multiplication factor for decay rate
|
||||
decay_rate_factor = trending_normed/WHALE_THRESHOLD
|
||||
|
||||
# The -1 is because this is just the *extra* part being applied
|
||||
factor = (DECAY**SAVE_INTERVAL)**(decay_rate_factor - 1.0)
|
||||
# print(claim_hash, trending_normed, decay_rate_factor)
|
||||
self.claims[claim_hash]["trending_score"] *= factor
|
||||
self.claims[claim_hash]["changed"] = True
|
||||
|
||||
start = time.time()
|
||||
trending_log(f"Calculating variable_decay trending at block {height}.\n")
|
||||
trending_data.update(db, height, recalculate_claim_hashes)
|
||||
end = time.time()
|
||||
trending_log(f"Trending operations took {end - start} seconds.\n\n")
|
||||
|
||||
def test_trending():
|
||||
"""
|
||||
@@ -260,12 +405,12 @@ def test_trending():
|
||||
COMMIT;
|
||||
""")
|
||||
|
||||
# Initialise trending data before anything happens with the claims
|
||||
trending_data.initialise(db)
|
||||
|
||||
# Insert initial states of claims
|
||||
everything = {"huge_whale": 0.01,
|
||||
"huge_whale_botted": 0.01,
|
||||
"medium_whale": 0.01,
|
||||
"small_whale": 0.01,
|
||||
"minnow": 0.01}
|
||||
everything = {"huge_whale": 0.01, "medium_whale": 0.01, "small_whale": 0.01,
|
||||
"huge_whale_botted": 0.01, "minnow": 0.01}
|
||||
|
||||
def to_list_of_tuples(stuff):
|
||||
l = []
|
||||
@@ -277,13 +422,17 @@ def test_trending():
|
||||
INSERT INTO claim (claim_hash, amount) VALUES (?, 1E8*?);
|
||||
""", to_list_of_tuples(everything))
|
||||
|
||||
# Process block zero
|
||||
height = 0
|
||||
run(db, height, height, everything.keys())
|
||||
|
||||
# Save trajectories for plotting
|
||||
trajectories = {}
|
||||
for key in trending_data.claims:
|
||||
trajectories[key] = [trending_data.claims[key]["trending_score"]]
|
||||
for row in trending_data.execute("""
|
||||
SELECT claim_hash, trending_score
|
||||
FROM claims;
|
||||
"""):
|
||||
trajectories[row[0]] = [row[1]/trending_unit(height)]
|
||||
|
||||
# Main loop
|
||||
for height in range(1, 1000):
|
||||
@@ -312,119 +461,24 @@ def test_trending():
|
||||
# Call run()
|
||||
run(db, height, height, everything.keys())
|
||||
|
||||
for key in trending_data.claims:
|
||||
trajectories[key].append(trending_data.claims[key]["trending_score"]\
|
||||
/get_time_boost(height))
|
||||
# Append current trending scores to trajectories
|
||||
for row in db.execute("""
|
||||
SELECT claim_hash, trending_mixed
|
||||
FROM claim;
|
||||
"""):
|
||||
trajectories[row[0]].append(row[1]/trending_unit(height))
|
||||
|
||||
dbc.close()
|
||||
|
||||
# pylint: disable=C0415
|
||||
import matplotlib.pyplot as plt
|
||||
for key in trending_data.claims:
|
||||
for key in trajectories:
|
||||
plt.plot(trajectories[key], label=key)
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
|
||||
# One global instance
|
||||
# pylint: disable=C0103
|
||||
trending_data = TrendingData()
|
||||
|
||||
def run(db, height, final_height, recalculate_claim_hashes):
|
||||
|
||||
if height < final_height - 5*HALF_LIFE:
|
||||
trending_log("Skipping variable_decay trending at block {h}.\n".format(h=height))
|
||||
return
|
||||
|
||||
start = time.time()
|
||||
|
||||
trending_log("Calculating variable_decay trending at block {h}.\n".format(h=height))
|
||||
trending_log(" Length of trending data = {l}.\n"\
|
||||
.format(l=len(trending_data.claims)))
|
||||
|
||||
# Renormalise trending scores and mark all as having changed
|
||||
if height % RENORM_INTERVAL == 0:
|
||||
trending_log(" Renormalising trending scores...")
|
||||
|
||||
keys = trending_data.claims.keys()
|
||||
trending_data.whales = set([])
|
||||
for key in keys:
|
||||
if trending_data.claims[key]["trending_score"] != 0.0:
|
||||
trending_data.claims[key]["trending_score"] *= DECAY_PER_RENORM
|
||||
trending_data.claims[key]["changed"] = True
|
||||
|
||||
# Tiny becomes zero
|
||||
if abs(trending_data.claims[key]["trending_score"]) < 1E-3:
|
||||
trending_data.claims[key]["trending_score"] = 0.0
|
||||
|
||||
# Re-mark whales
|
||||
if trending_data.claims[key]["trending_score"] >= WHALE_THRESHOLD*get_time_boost(height):
|
||||
trending_data.add_whale(key)
|
||||
|
||||
trending_log("done.\n")
|
||||
|
||||
|
||||
# Regular message.
|
||||
trending_log(" Reading total_amounts from db and updating"\
|
||||
+ " trending scores in RAM...")
|
||||
|
||||
# Update claims from db
|
||||
if not trending_data.initialised:
|
||||
|
||||
trending_log("initial load...")
|
||||
# On fresh launch
|
||||
for row in db.execute("""
|
||||
SELECT claim_hash, trending_mixed,
|
||||
(amount + support_amount)
|
||||
AS total_amount
|
||||
FROM claim;
|
||||
"""):
|
||||
trending_data.insert_claim_from_load(height, row[0], row[1], 1E-8*row[2])
|
||||
trending_data.initialised = True
|
||||
else:
|
||||
for row in db.execute(f"""
|
||||
SELECT claim_hash,
|
||||
(amount + support_amount)
|
||||
AS total_amount
|
||||
FROM claim
|
||||
WHERE claim_hash IN
|
||||
({','.join('?' for _ in recalculate_claim_hashes)});
|
||||
""", recalculate_claim_hashes):
|
||||
trending_data.update_claim(height, row[0], 1E-8*row[1])
|
||||
|
||||
# Apply pending spikes
|
||||
trending_data.apply_spikes(height)
|
||||
|
||||
trending_log("done.\n")
|
||||
|
||||
|
||||
# Write trending scores to DB
|
||||
if height % SAVE_INTERVAL == 0:
|
||||
|
||||
trending_log(" Finding and processing whales...")
|
||||
trending_log(str(len(trending_data.whales)) + " whales found...")
|
||||
trending_data.process_whales(height)
|
||||
trending_log("done.\n")
|
||||
|
||||
trending_log(" Writing trending scores to db...")
|
||||
|
||||
the_list = []
|
||||
keys = trending_data.claims.keys()
|
||||
|
||||
for key in keys:
|
||||
if trending_data.claims[key]["changed"]:
|
||||
the_list.append((trending_data.claims[key]["trending_score"], key))
|
||||
trending_data.claims[key]["changed"] = False
|
||||
|
||||
trending_log("{n} scores to write...".format(n=len(the_list)))
|
||||
|
||||
db.executemany("UPDATE claim SET trending_mixed=? WHERE claim_hash=?;",
|
||||
the_list)
|
||||
|
||||
trending_log("done.\n")
|
||||
|
||||
trending_log("Trending operations took {time} seconds.\n\n"\
|
||||
.format(time=time.time() - start))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -6,7 +6,6 @@ from decimal import Decimal
|
||||
from collections import namedtuple
|
||||
from multiprocessing import Manager
|
||||
from binascii import unhexlify
|
||||
|
||||
from lbry.wallet.server.leveldb import LevelDB
|
||||
from lbry.wallet.server.util import class_logger
|
||||
from lbry.wallet.database import query, constraints_to_sql
|
||||
@@ -19,7 +18,7 @@ from lbry.wallet.server.db.canonical import register_canonical_functions
|
||||
from lbry.wallet.server.db.full_text_search import update_full_text_search, CREATE_FULL_TEXT_SEARCH, first_sync_finished
|
||||
from lbry.wallet.server.db.trending import TRENDING_ALGORITHMS
|
||||
|
||||
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
|
||||
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS, INDEXED_LANGUAGES
|
||||
|
||||
|
||||
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
|
||||
@@ -117,6 +116,15 @@ class SQLDB:
|
||||
create unique index if not exists tag_claim_hash_tag_idx on tag (claim_hash, tag);
|
||||
"""
|
||||
|
||||
CREATE_LANGUAGE_TABLE = """
|
||||
create table if not exists language (
|
||||
language text not null,
|
||||
claim_hash bytes not null,
|
||||
height integer not null
|
||||
);
|
||||
create unique index if not exists language_claim_hash_language_idx on language (claim_hash, language);
|
||||
"""
|
||||
|
||||
CREATE_CLAIMTRIE_TABLE = """
|
||||
create table if not exists claimtrie (
|
||||
normalized text primary key,
|
||||
@@ -174,12 +182,18 @@ class SQLDB:
|
||||
for tag_value, tag_key in COMMON_TAGS.items()
|
||||
)
|
||||
|
||||
LANGUAGE_INDEXES = '\n'.join(
|
||||
f"create unique index if not exists language_{language}_idx on language (language, claim_hash) WHERE language='{language}';"
|
||||
for language in INDEXED_LANGUAGES
|
||||
)
|
||||
|
||||
CREATE_TABLES_QUERY = (
|
||||
CREATE_CLAIM_TABLE +
|
||||
CREATE_FULL_TEXT_SEARCH +
|
||||
CREATE_SUPPORT_TABLE +
|
||||
CREATE_CLAIMTRIE_TABLE +
|
||||
CREATE_TAG_TABLE
|
||||
CREATE_TAG_TABLE +
|
||||
CREATE_LANGUAGE_TABLE
|
||||
)
|
||||
|
||||
def __init__(
|
||||
@@ -305,7 +319,7 @@ class SQLDB:
|
||||
self.execute('commit;')
|
||||
|
||||
def _upsertable_claims(self, txos: List[Output], header, clear_first=False):
|
||||
claim_hashes, claims, tags = set(), [], {}
|
||||
claim_hashes, claims, tags, languages = set(), [], {}, {}
|
||||
for txo in txos:
|
||||
tx = txo.tx_ref.tx
|
||||
|
||||
@@ -316,6 +330,13 @@ class SQLDB:
|
||||
#self.logger.exception(f"Could not decode claim name for {tx.id}:{txo.position}.")
|
||||
continue
|
||||
|
||||
language = 'none'
|
||||
try:
|
||||
if txo.claim.is_stream and txo.claim.stream.languages:
|
||||
language = txo.claim.stream.languages[0].language
|
||||
except:
|
||||
pass
|
||||
|
||||
claim_hash = txo.claim_hash
|
||||
claim_hashes.add(claim_hash)
|
||||
claim_record = {
|
||||
@@ -372,6 +393,10 @@ class SQLDB:
|
||||
claim_record['reposted_claim_hash'] = claim.repost.reference.claim_hash
|
||||
elif claim.is_channel:
|
||||
claim_record['claim_type'] = CLAIM_TYPES['channel']
|
||||
elif claim.is_collection:
|
||||
claim_record['claim_type'] = CLAIM_TYPES['collection']
|
||||
|
||||
languages[(language, claim_hash)] = (language, claim_hash, tx.height)
|
||||
|
||||
for tag in clean_tags(claim.message.tags):
|
||||
tags[(tag, claim_hash)] = (tag, claim_hash, tx.height)
|
||||
@@ -383,6 +408,10 @@ class SQLDB:
|
||||
self.executemany(
|
||||
"INSERT OR IGNORE INTO tag (tag, claim_hash, height) VALUES (?, ?, ?)", tags.values()
|
||||
)
|
||||
if languages:
|
||||
self.executemany(
|
||||
"INSERT OR IGNORE INTO language (language, claim_hash, height) VALUES (?, ?, ?)", languages.values()
|
||||
)
|
||||
|
||||
return claims
|
||||
|
||||
|
||||
@@ -37,7 +37,9 @@ class Env:
|
||||
self.obsolete(['UTXO_MB', 'HIST_MB', 'NETWORK'])
|
||||
self.db_dir = self.required('DB_DIRECTORY')
|
||||
self.db_engine = self.default('DB_ENGINE', 'leveldb')
|
||||
self.trending_algorithms = list(set(self.default('TRENDING_ALGORITHMS', 'zscore').split(' ')))
|
||||
self.trending_algorithms = [
|
||||
trending for trending in set(self.default('TRENDING_ALGORITHMS', 'zscore').split(' ')) if trending
|
||||
]
|
||||
self.max_query_workers = self.integer('MAX_QUERY_WORKERS', None)
|
||||
self.individual_tag_indexes = self.boolean('INDIVIDUAL_TAG_INDEXES', True)
|
||||
self.track_metrics = self.boolean('TRACK_METRICS', False)
|
||||
|
||||
@@ -67,17 +67,17 @@ def hash160(x):
|
||||
return ripemd160(sha256(x))
|
||||
|
||||
|
||||
def hash_to_hex_str(x):
|
||||
def hash_to_hex_str(x: bytes) -> str:
|
||||
"""Convert a big-endian binary hash to displayed hex string.
|
||||
|
||||
Display form of a binary hash is reversed and converted to hex.
|
||||
"""
|
||||
return bytes(reversed(x)).hex()
|
||||
return x[::-1].hex()
|
||||
|
||||
|
||||
def hex_str_to_hash(x):
|
||||
def hex_str_to_hash(x: str) -> bytes:
|
||||
"""Convert a displayed hex string to a binary hash."""
|
||||
return bytes(reversed(hex_to_bytes(x)))
|
||||
return hex_to_bytes(x)[::-1]
|
||||
|
||||
|
||||
class Base58Error(Exception):
|
||||
|
||||
@@ -173,20 +173,20 @@ class History:
|
||||
|
||||
self.logger.info(f'backing up removed {nremoves:,d} history entries')
|
||||
|
||||
def get_txnums(self, hashX, limit=1000):
|
||||
"""Generator that returns an unpruned, sorted list of tx_nums in the
|
||||
history of a hashX. Includes both spending and receiving
|
||||
transactions. By default yields at most 1000 entries. Set
|
||||
limit to None to get them all. """
|
||||
limit = util.resolve_limit(limit)
|
||||
for key, hist in self.db.iterator(prefix=hashX):
|
||||
a = array.array('I')
|
||||
a.frombytes(hist)
|
||||
for tx_num in a:
|
||||
if limit == 0:
|
||||
return
|
||||
yield tx_num
|
||||
limit -= 1
|
||||
# def get_txnums(self, hashX, limit=1000):
|
||||
# """Generator that returns an unpruned, sorted list of tx_nums in the
|
||||
# history of a hashX. Includes both spending and receiving
|
||||
# transactions. By default yields at most 1000 entries. Set
|
||||
# limit to None to get them all. """
|
||||
# limit = util.resolve_limit(limit)
|
||||
# for key, hist in self.db.iterator(prefix=hashX):
|
||||
# a = array.array('I')
|
||||
# a.frombytes(hist)
|
||||
# for tx_num in a:
|
||||
# if limit == 0:
|
||||
# return
|
||||
# yield tx_num
|
||||
# limit -= 1
|
||||
|
||||
#
|
||||
# History compaction
|
||||
|
||||
+231
-122
@@ -14,15 +14,17 @@ import array
|
||||
import ast
|
||||
import os
|
||||
import time
|
||||
import zlib
|
||||
import typing
|
||||
from typing import Optional, List, Tuple, Iterable
|
||||
from asyncio import sleep
|
||||
from bisect import bisect_right
|
||||
from collections import namedtuple
|
||||
from functools import partial
|
||||
from glob import glob
|
||||
from struct import pack, unpack
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
import attr
|
||||
|
||||
from lbry.utils import LRUCacheWithMetrics
|
||||
from lbry.wallet.server import util
|
||||
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
|
||||
from lbry.wallet.server.merkle import Merkle, MerkleCache
|
||||
@@ -32,6 +34,13 @@ from lbry.wallet.server.history import History
|
||||
|
||||
|
||||
UTXO = namedtuple("UTXO", "tx_num tx_pos tx_hash height value")
|
||||
HEADER_PREFIX = b'H'
|
||||
TX_COUNT_PREFIX = b'T'
|
||||
TX_HASH_PREFIX = b'X'
|
||||
TX_PREFIX = b'B'
|
||||
TX_NUM_PREFIX = b'N'
|
||||
BLOCK_HASH_PREFIX = b'C'
|
||||
|
||||
|
||||
|
||||
@attr.s(slots=True)
|
||||
@@ -39,7 +48,8 @@ class FlushData:
|
||||
height = attr.ib()
|
||||
tx_count = attr.ib()
|
||||
headers = attr.ib()
|
||||
block_tx_hashes = attr.ib()
|
||||
block_hashes = attr.ib()
|
||||
block_txs = attr.ib()
|
||||
# The following are flushed to the UTXO DB if undo_infos is not None
|
||||
undo_infos = attr.ib()
|
||||
adds = attr.ib()
|
||||
@@ -63,15 +73,7 @@ class LevelDB:
|
||||
self.logger = util.class_logger(__name__, self.__class__.__name__)
|
||||
self.env = env
|
||||
self.coin = env.coin
|
||||
self.executor = ThreadPoolExecutor(max(1, os.cpu_count() - 1))
|
||||
|
||||
# Setup block header size handlers
|
||||
if self.coin.STATIC_BLOCK_HEADERS:
|
||||
self.header_offset = self.coin.static_header_offset
|
||||
self.header_len = self.coin.static_header_len
|
||||
else:
|
||||
self.header_offset = self.dynamic_header_offset
|
||||
self.header_len = self.dynamic_header_len
|
||||
self.executor = None
|
||||
|
||||
self.logger.info(f'switching current directory to {env.db_dir}')
|
||||
|
||||
@@ -79,6 +81,7 @@ class LevelDB:
|
||||
self.history = History()
|
||||
self.utxo_db = None
|
||||
self.tx_counts = None
|
||||
self.headers = None
|
||||
self.last_flush = time.time()
|
||||
|
||||
self.logger.info(f'using {self.env.db_engine} for DB backend')
|
||||
@@ -87,60 +90,107 @@ class LevelDB:
|
||||
self.merkle = Merkle()
|
||||
self.header_mc = MerkleCache(self.merkle, self.fs_block_hashes)
|
||||
|
||||
path = partial(os.path.join, self.env.db_dir)
|
||||
self.headers_file = util.LogicalFile(path('meta/headers'), 2, 16000000)
|
||||
self.tx_counts_file = util.LogicalFile(path('meta/txcounts'), 2, 2000000)
|
||||
self.hashes_file = util.LogicalFile(path('meta/hashes'), 4, 16000000)
|
||||
if not self.coin.STATIC_BLOCK_HEADERS:
|
||||
self.headers_offsets_file = util.LogicalFile(
|
||||
path('meta/headers_offsets'), 2, 16000000)
|
||||
self.headers_db = None
|
||||
self.tx_db = None
|
||||
|
||||
self._tx_and_merkle_cache = LRUCacheWithMetrics(2 ** 17, metric_name='tx_and_merkle', namespace="wallet_server")
|
||||
self.total_transactions = None
|
||||
|
||||
async def _read_tx_counts(self):
|
||||
if self.tx_counts is not None:
|
||||
return
|
||||
# tx_counts[N] has the cumulative number of txs at the end of
|
||||
# height N. So tx_counts[0] is 1 - the genesis coinbase
|
||||
size = (self.db_height + 1) * 4
|
||||
tx_counts = self.tx_counts_file.read(0, size)
|
||||
assert len(tx_counts) == size
|
||||
|
||||
def get_counts():
|
||||
return tuple(
|
||||
util.unpack_be_uint64(tx_count)
|
||||
for tx_count in self.tx_db.iterator(prefix=TX_COUNT_PREFIX, include_key=False)
|
||||
)
|
||||
|
||||
tx_counts = await asyncio.get_event_loop().run_in_executor(self.executor, get_counts)
|
||||
assert len(tx_counts) == self.db_height + 1, f"{len(tx_counts)} vs {self.db_height + 1}"
|
||||
self.tx_counts = array.array('I', tx_counts)
|
||||
|
||||
if self.tx_counts:
|
||||
assert self.db_tx_count == self.tx_counts[-1]
|
||||
assert self.db_tx_count == self.tx_counts[-1], \
|
||||
f"{self.db_tx_count} vs {self.tx_counts[-1]} ({len(self.tx_counts)} counts)"
|
||||
else:
|
||||
assert self.db_tx_count == 0
|
||||
|
||||
async def _open_dbs(self, for_sync, compacting):
|
||||
assert self.utxo_db is None
|
||||
async def _read_txids(self):
|
||||
def get_txids():
|
||||
return list(self.tx_db.iterator(prefix=TX_HASH_PREFIX, include_key=False))
|
||||
|
||||
# First UTXO DB
|
||||
self.utxo_db = self.db_class('utxo', for_sync)
|
||||
if self.utxo_db.is_new:
|
||||
self.logger.info('created new database')
|
||||
self.logger.info('creating metadata directory')
|
||||
os.mkdir(os.path.join(self.env.db_dir, 'meta'))
|
||||
coin_path = os.path.join(self.env.db_dir, 'meta', 'COIN')
|
||||
start = time.perf_counter()
|
||||
self.logger.info("loading txids")
|
||||
txids = await asyncio.get_event_loop().run_in_executor(self.executor, get_txids)
|
||||
assert len(txids) == len(self.tx_counts) == 0 or len(txids) == self.tx_counts[-1]
|
||||
self.total_transactions = txids
|
||||
ts = time.perf_counter() - start
|
||||
self.logger.info("loaded %i txids in %ss", len(self.total_transactions), round(ts, 4))
|
||||
|
||||
async def _read_headers(self):
|
||||
if self.headers is not None:
|
||||
return
|
||||
|
||||
def get_headers():
|
||||
return [
|
||||
header for header in self.headers_db.iterator(prefix=HEADER_PREFIX, include_key=False)
|
||||
]
|
||||
|
||||
headers = await asyncio.get_event_loop().run_in_executor(self.executor, get_headers)
|
||||
assert len(headers) - 1 == self.db_height, f"{len(headers)} vs {self.db_height}"
|
||||
self.headers = headers
|
||||
|
||||
async def _open_dbs(self, for_sync, compacting):
|
||||
if self.executor is None:
|
||||
self.executor = ThreadPoolExecutor(1)
|
||||
coin_path = os.path.join(self.env.db_dir, 'COIN')
|
||||
if not os.path.isfile(coin_path):
|
||||
with util.open_file(coin_path, create=True) as f:
|
||||
f.write(f'ElectrumX databases and metadata for '
|
||||
f'{self.coin.NAME} {self.coin.NET}'.encode())
|
||||
if not self.coin.STATIC_BLOCK_HEADERS:
|
||||
self.headers_offsets_file.write(0, bytes(8))
|
||||
else:
|
||||
self.logger.info(f'opened UTXO DB (for sync: {for_sync})')
|
||||
|
||||
assert self.headers_db is None
|
||||
self.headers_db = self.db_class('headers', for_sync)
|
||||
if self.headers_db.is_new:
|
||||
self.logger.info('created new headers db')
|
||||
self.logger.info(f'opened headers DB (for sync: {for_sync})')
|
||||
|
||||
assert self.tx_db is None
|
||||
self.tx_db = self.db_class('tx', for_sync)
|
||||
if self.tx_db.is_new:
|
||||
self.logger.info('created new tx db')
|
||||
self.logger.info(f'opened tx DB (for sync: {for_sync})')
|
||||
|
||||
assert self.utxo_db is None
|
||||
# First UTXO DB
|
||||
self.utxo_db = self.db_class('utxo', for_sync)
|
||||
if self.utxo_db.is_new:
|
||||
self.logger.info('created new utxo db')
|
||||
self.logger.info(f'opened utxo db (for sync: {for_sync})')
|
||||
self.read_utxo_state()
|
||||
|
||||
# Then history DB
|
||||
self.utxo_flush_count = self.history.open_db(self.db_class, for_sync,
|
||||
self.utxo_flush_count,
|
||||
compacting)
|
||||
self.utxo_flush_count = self.history.open_db(
|
||||
self.db_class, for_sync, self.utxo_flush_count, compacting
|
||||
)
|
||||
self.clear_excess_undo_info()
|
||||
|
||||
# Read TX counts (requires meta directory)
|
||||
await self._read_tx_counts()
|
||||
if self.total_transactions is None:
|
||||
await self._read_txids()
|
||||
await self._read_headers()
|
||||
|
||||
def close(self):
|
||||
self.utxo_db.close()
|
||||
self.history.close_db()
|
||||
self.headers_db.close()
|
||||
self.tx_db.close()
|
||||
self.executor.shutdown(wait=True)
|
||||
self.executor = None
|
||||
|
||||
async def open_for_compacting(self):
|
||||
await self._open_dbs(True, True)
|
||||
@@ -152,18 +202,28 @@ class LevelDB:
|
||||
synchronization. When serving clients we want the open files for
|
||||
serving network connections.
|
||||
"""
|
||||
self.logger.info("opened for sync")
|
||||
await self._open_dbs(True, False)
|
||||
|
||||
async def open_for_serving(self):
|
||||
"""Open the databases for serving. If they are already open they are
|
||||
closed first.
|
||||
"""
|
||||
self.logger.info('closing DBs to re-open for serving')
|
||||
if self.utxo_db:
|
||||
self.logger.info('closing DBs to re-open for serving')
|
||||
self.utxo_db.close()
|
||||
self.history.close_db()
|
||||
self.utxo_db = None
|
||||
if self.headers_db:
|
||||
self.headers_db.close()
|
||||
self.headers_db = None
|
||||
if self.tx_db:
|
||||
self.tx_db.close()
|
||||
self.tx_db = None
|
||||
|
||||
await self._open_dbs(False, False)
|
||||
self.logger.info("opened for serving")
|
||||
|
||||
# Header merkle cache
|
||||
|
||||
@@ -185,7 +245,7 @@ class LevelDB:
|
||||
assert flush_data.height == self.fs_height == self.db_height
|
||||
assert flush_data.tip == self.db_tip
|
||||
assert not flush_data.headers
|
||||
assert not flush_data.block_tx_hashes
|
||||
assert not flush_data.block_txs
|
||||
assert not flush_data.adds
|
||||
assert not flush_data.deletes
|
||||
assert not flush_data.undo_infos
|
||||
@@ -243,36 +303,50 @@ class LevelDB:
|
||||
"""
|
||||
prior_tx_count = (self.tx_counts[self.fs_height]
|
||||
if self.fs_height >= 0 else 0)
|
||||
assert len(flush_data.block_tx_hashes) == len(flush_data.headers)
|
||||
assert len(flush_data.block_txs) == len(flush_data.headers)
|
||||
assert flush_data.height == self.fs_height + len(flush_data.headers)
|
||||
assert flush_data.tx_count == (self.tx_counts[-1] if self.tx_counts
|
||||
else 0)
|
||||
assert len(self.tx_counts) == flush_data.height + 1
|
||||
hashes = b''.join(flush_data.block_tx_hashes)
|
||||
flush_data.block_tx_hashes.clear()
|
||||
assert len(hashes) % 32 == 0
|
||||
assert len(hashes) // 32 == flush_data.tx_count - prior_tx_count
|
||||
assert len(
|
||||
b''.join(hashes for hashes, _ in flush_data.block_txs)
|
||||
) // 32 == flush_data.tx_count - prior_tx_count
|
||||
|
||||
# Write the headers, tx counts, and tx hashes
|
||||
start_time = time.time()
|
||||
height_start = self.fs_height + 1
|
||||
offset = self.header_offset(height_start)
|
||||
self.headers_file.write(offset, b''.join(flush_data.headers))
|
||||
self.fs_update_header_offsets(offset, height_start, flush_data.headers)
|
||||
# Write the headers
|
||||
start_time = time.perf_counter()
|
||||
|
||||
with self.headers_db.write_batch() as batch:
|
||||
batch_put = batch.put
|
||||
for i, header in enumerate(flush_data.headers):
|
||||
batch_put(HEADER_PREFIX + util.pack_be_uint64(self.fs_height + i + 1), header)
|
||||
self.headers.append(header)
|
||||
flush_data.headers.clear()
|
||||
|
||||
offset = height_start * self.tx_counts.itemsize
|
||||
self.tx_counts_file.write(offset,
|
||||
self.tx_counts[height_start:].tobytes())
|
||||
offset = prior_tx_count * 32
|
||||
self.hashes_file.write(offset, hashes)
|
||||
height_start = self.fs_height + 1
|
||||
tx_num = prior_tx_count
|
||||
|
||||
with self.tx_db.write_batch() as batch:
|
||||
batch_put = batch.put
|
||||
for block_hash, (tx_hashes, txs) in zip(flush_data.block_hashes, flush_data.block_txs):
|
||||
tx_count = self.tx_counts[height_start]
|
||||
batch_put(BLOCK_HASH_PREFIX + util.pack_be_uint64(height_start), block_hash[::-1])
|
||||
batch_put(TX_COUNT_PREFIX + util.pack_be_uint64(height_start), util.pack_be_uint64(tx_count))
|
||||
height_start += 1
|
||||
offset = 0
|
||||
while offset < len(tx_hashes):
|
||||
batch_put(TX_HASH_PREFIX + util.pack_be_uint64(tx_num), tx_hashes[offset:offset+32])
|
||||
batch_put(TX_NUM_PREFIX + tx_hashes[offset:offset+32], util.pack_be_uint64(tx_num))
|
||||
batch_put(TX_PREFIX + tx_hashes[offset:offset+32], txs[offset // 32])
|
||||
tx_num += 1
|
||||
offset += 32
|
||||
|
||||
flush_data.block_txs.clear()
|
||||
flush_data.block_hashes.clear()
|
||||
|
||||
self.fs_height = flush_data.height
|
||||
self.fs_tx_count = flush_data.tx_count
|
||||
|
||||
if self.utxo_db.for_sync:
|
||||
elapsed = time.time() - start_time
|
||||
self.logger.info(f'flushed filesystem data in {elapsed:.2f}s')
|
||||
elapsed = time.perf_counter() - start_time
|
||||
self.logger.info(f'flushed filesystem data in {elapsed:.2f}s')
|
||||
|
||||
def flush_history(self):
|
||||
self.history.flush()
|
||||
@@ -331,7 +405,7 @@ class LevelDB:
|
||||
def flush_backup(self, flush_data, touched):
|
||||
"""Like flush_dbs() but when backing up. All UTXOs are flushed."""
|
||||
assert not flush_data.headers
|
||||
assert not flush_data.block_tx_hashes
|
||||
assert not flush_data.block_txs
|
||||
assert flush_data.height < self.db_height
|
||||
self.history.assert_flushed()
|
||||
|
||||
@@ -350,43 +424,23 @@ class LevelDB:
|
||||
f'{elapsed:.1f}s. Height {flush_data.height:,d} '
|
||||
f'txs: {flush_data.tx_count:,d} ({tx_delta:+,d})')
|
||||
|
||||
def fs_update_header_offsets(self, offset_start, height_start, headers):
|
||||
if self.coin.STATIC_BLOCK_HEADERS:
|
||||
return
|
||||
offset = offset_start
|
||||
offsets = []
|
||||
for h in headers:
|
||||
offset += len(h)
|
||||
offsets.append(pack("<Q", offset))
|
||||
# For each header we get the offset of the next header, hence we
|
||||
# start writing from the next height
|
||||
pos = (height_start + 1) * 8
|
||||
self.headers_offsets_file.write(pos, b''.join(offsets))
|
||||
|
||||
def dynamic_header_offset(self, height):
|
||||
assert not self.coin.STATIC_BLOCK_HEADERS
|
||||
offset, = unpack('<Q', self.headers_offsets_file.read(height * 8, 8))
|
||||
return offset
|
||||
|
||||
def dynamic_header_len(self, height):
|
||||
return self.dynamic_header_offset(height + 1)\
|
||||
- self.dynamic_header_offset(height)
|
||||
|
||||
def backup_fs(self, height, tx_count):
|
||||
"""Back up during a reorg. This just updates our pointers."""
|
||||
self.fs_height = height
|
||||
while self.fs_height > height:
|
||||
self.fs_height -= 1
|
||||
self.headers.pop()
|
||||
self.fs_tx_count = tx_count
|
||||
# Truncate header_mc: header count is 1 more than the height.
|
||||
self.header_mc.truncate(height + 1)
|
||||
|
||||
async def raw_header(self, height):
|
||||
def raw_header(self, height):
|
||||
"""Return the binary header at the given height."""
|
||||
header, n = await self.read_headers(height, 1)
|
||||
header, n = self.read_headers(height, 1)
|
||||
if n != 1:
|
||||
raise IndexError(f'height {height:,d} out of range')
|
||||
return header
|
||||
|
||||
async def read_headers(self, start_height, count):
|
||||
def read_headers(self, start_height, count) -> typing.Tuple[bytes, int]:
|
||||
"""Requires start_height >= 0, count >= 0. Reads as many headers as
|
||||
are available starting at start_height up to count. This
|
||||
would be zero if start_height is beyond self.db_height, for
|
||||
@@ -395,20 +449,15 @@ class LevelDB:
|
||||
Returns a (binary, n) pair where binary is the concatenated
|
||||
binary headers, and n is the count of headers returned.
|
||||
"""
|
||||
|
||||
if start_height < 0 or count < 0:
|
||||
raise self.DBError(f'{count:,d} headers starting at '
|
||||
f'{start_height:,d} not on disk')
|
||||
|
||||
def read_headers():
|
||||
# Read some from disk
|
||||
disk_count = max(0, min(count, self.db_height + 1 - start_height))
|
||||
if disk_count:
|
||||
offset = self.header_offset(start_height)
|
||||
size = self.header_offset(start_height + disk_count) - offset
|
||||
return self.headers_file.read(offset, size), disk_count
|
||||
return b'', 0
|
||||
|
||||
return await asyncio.get_event_loop().run_in_executor(self.executor, read_headers)
|
||||
disk_count = max(0, min(count, self.db_height + 1 - start_height))
|
||||
if disk_count:
|
||||
return b''.join(self.headers[start_height:start_height + disk_count]), disk_count
|
||||
return b'', 0
|
||||
|
||||
def fs_tx_hash(self, tx_num):
|
||||
"""Return a par (tx_hash, tx_height) for the given tx number.
|
||||
@@ -416,23 +465,60 @@ class LevelDB:
|
||||
If the tx_height is not on disk, returns (None, tx_height)."""
|
||||
tx_height = bisect_right(self.tx_counts, tx_num)
|
||||
if tx_height > self.db_height:
|
||||
tx_hash = None
|
||||
else:
|
||||
tx_hash = self.hashes_file.read(tx_num * 32, 32)
|
||||
return tx_hash, tx_height
|
||||
return None, tx_height
|
||||
return self.total_transactions[tx_num], tx_height
|
||||
|
||||
def _fs_transactions(self, txids: Iterable[str]):
|
||||
unpack_be_uint64 = util.unpack_be_uint64
|
||||
tx_counts = self.tx_counts
|
||||
tx_db_get = self.tx_db.get
|
||||
tx_cache = self._tx_and_merkle_cache
|
||||
|
||||
tx_infos = {}
|
||||
|
||||
for tx_hash in txids:
|
||||
cached_tx = tx_cache.get(tx_hash)
|
||||
if cached_tx:
|
||||
tx, merkle = cached_tx
|
||||
else:
|
||||
tx_hash_bytes = bytes.fromhex(tx_hash)[::-1]
|
||||
tx_num = tx_db_get(TX_NUM_PREFIX + tx_hash_bytes)
|
||||
tx = None
|
||||
tx_height = -1
|
||||
if tx_num is not None:
|
||||
tx_num = unpack_be_uint64(tx_num)
|
||||
tx_height = bisect_right(tx_counts, tx_num)
|
||||
if tx_height < self.db_height:
|
||||
tx = tx_db_get(TX_PREFIX + tx_hash_bytes)
|
||||
if tx_height == -1:
|
||||
merkle = {
|
||||
'block_height': -1
|
||||
}
|
||||
else:
|
||||
tx_pos = tx_num - tx_counts[tx_height - 1]
|
||||
branch, root = self.merkle.branch_and_root(
|
||||
self.total_transactions[tx_counts[tx_height - 1]:tx_counts[tx_height]], tx_pos
|
||||
)
|
||||
merkle = {
|
||||
'block_height': tx_height,
|
||||
'merkle': [
|
||||
hash_to_hex_str(hash)
|
||||
for hash in branch
|
||||
],
|
||||
'pos': tx_pos
|
||||
}
|
||||
if tx_height + 10 < self.db_height:
|
||||
tx_cache[tx_hash] = tx, merkle
|
||||
tx_infos[tx_hash] = (None if not tx else tx.hex(), merkle)
|
||||
return tx_infos
|
||||
|
||||
async def fs_transactions(self, txids):
|
||||
return await asyncio.get_event_loop().run_in_executor(self.executor, self._fs_transactions, txids)
|
||||
|
||||
async def fs_block_hashes(self, height, count):
|
||||
headers_concat, headers_count = await self.read_headers(height, count)
|
||||
if headers_count != count:
|
||||
raise self.DBError(f'only got {headers_count:,d} headers starting at {height:,d}, not {count:,d}')
|
||||
offset = 0
|
||||
headers = []
|
||||
for n in range(count):
|
||||
hlen = self.header_len(height + n)
|
||||
headers.append(headers_concat[offset:offset + hlen])
|
||||
offset += hlen
|
||||
|
||||
return [self.coin.header_hash(header) for header in headers]
|
||||
if height + count > len(self.headers):
|
||||
raise self.DBError(f'only got {len(self.headers) - height:,d} headers starting at {height:,d}, not {count:,d}')
|
||||
return [self.coin.header_hash(header) for header in self.headers[height:height + count]]
|
||||
|
||||
async def limited_history(self, hashX, *, limit=1000):
|
||||
"""Return an unpruned, sorted list of (tx_hash, height) tuples of
|
||||
@@ -441,15 +527,33 @@ class LevelDB:
|
||||
transactions. By default returns at most 1000 entries. Set
|
||||
limit to None to get them all.
|
||||
"""
|
||||
|
||||
def read_history():
|
||||
tx_nums = list(self.history.get_txnums(hashX, limit))
|
||||
fs_tx_hash = self.fs_tx_hash
|
||||
return [fs_tx_hash(tx_num) for tx_num in tx_nums]
|
||||
db_height = self.db_height
|
||||
tx_counts = self.tx_counts
|
||||
|
||||
cnt = 0
|
||||
txs = []
|
||||
|
||||
for hist in self.history.db.iterator(prefix=hashX, include_key=False):
|
||||
a = array.array('I')
|
||||
a.frombytes(hist)
|
||||
for tx_num in a:
|
||||
tx_height = bisect_right(tx_counts, tx_num)
|
||||
if tx_height > db_height:
|
||||
return
|
||||
txs.append((tx_num, tx_height))
|
||||
cnt += 1
|
||||
if limit and cnt >= limit:
|
||||
break
|
||||
if limit and cnt >= limit:
|
||||
break
|
||||
return txs
|
||||
|
||||
while True:
|
||||
history = await asyncio.get_event_loop().run_in_executor(self.executor, read_history)
|
||||
if all(hash is not None for hash, height in history):
|
||||
return history
|
||||
if history is not None:
|
||||
return [(self.total_transactions[tx_num], tx_height) for (tx_num, tx_height) in history]
|
||||
self.logger.warning(f'limited_history: tx hash '
|
||||
f'not found (reorg?), retrying...')
|
||||
await sleep(0.25)
|
||||
@@ -474,16 +578,20 @@ class LevelDB:
|
||||
batch_put(self.undo_key(height), b''.join(undo_info))
|
||||
|
||||
def raw_block_prefix(self):
|
||||
return 'meta/block'
|
||||
return 'block'
|
||||
|
||||
def raw_block_path(self, height):
|
||||
return os.path.join(self.env.db_dir, f'{self.raw_block_prefix()}{height:d}')
|
||||
|
||||
def read_raw_block(self, height):
|
||||
async def read_raw_block(self, height):
|
||||
"""Returns a raw block read from disk. Raises FileNotFoundError
|
||||
if the block isn't on-disk."""
|
||||
with util.open_file(self.raw_block_path(height)) as f:
|
||||
return f.read(-1)
|
||||
|
||||
def read():
|
||||
with util.open_file(self.raw_block_path(height)) as f:
|
||||
return f.read(-1)
|
||||
|
||||
return await asyncio.get_event_loop().run_in_executor(self.executor, read)
|
||||
|
||||
def write_raw_block(self, block, height):
|
||||
"""Write a raw block to disk."""
|
||||
@@ -602,13 +710,14 @@ class LevelDB:
|
||||
utxos = []
|
||||
utxos_append = utxos.append
|
||||
s_unpack = unpack
|
||||
fs_tx_hash = self.fs_tx_hash
|
||||
# Key: b'u' + address_hashX + tx_idx + tx_num
|
||||
# Value: the UTXO value as a 64-bit unsigned integer
|
||||
prefix = b'u' + hashX
|
||||
for db_key, db_value in self.utxo_db.iterator(prefix=prefix):
|
||||
tx_pos, tx_num = s_unpack('<HI', db_key[-6:])
|
||||
value, = unpack('<Q', db_value)
|
||||
tx_hash, height = self.fs_tx_hash(tx_num)
|
||||
tx_hash, height = fs_tx_hash(tx_num)
|
||||
utxos_append(UTXO(tx_num, tx_pos, tx_hash, height, value))
|
||||
return utxos
|
||||
|
||||
|
||||
@@ -6,13 +6,12 @@
|
||||
# and warranty status of this software.
|
||||
|
||||
"""Mempool handling."""
|
||||
import os
|
||||
import asyncio
|
||||
import itertools
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import defaultdict
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
from prometheus_client import Histogram
|
||||
|
||||
import attr
|
||||
|
||||
@@ -73,12 +72,22 @@ class MemPoolAPI(ABC):
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
async def on_mempool(self, touched, height):
|
||||
async def on_mempool(self, touched, new_touched, height):
|
||||
"""Called each time the mempool is synchronized. touched is a set of
|
||||
hashXs touched since the previous call. height is the
|
||||
daemon's height at the time the mempool was obtained."""
|
||||
|
||||
|
||||
NAMESPACE = "wallet_server"
|
||||
HISTOGRAM_BUCKETS = (
|
||||
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
|
||||
)
|
||||
mempool_process_time_metric = Histogram(
|
||||
"processed_mempool", "Time to process mempool and notify touched addresses",
|
||||
namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS
|
||||
)
|
||||
|
||||
|
||||
class MemPool:
|
||||
"""Representation of the daemon's mempool.
|
||||
|
||||
@@ -93,7 +102,7 @@ class MemPool:
|
||||
hashXs: hashX -> set of all hashes of txs touching the hashX
|
||||
"""
|
||||
|
||||
def __init__(self, coin, api, refresh_secs=5.0, log_status_secs=120.0):
|
||||
def __init__(self, coin, api, refresh_secs=1.0, log_status_secs=120.0):
|
||||
assert isinstance(api, MemPoolAPI)
|
||||
self.coin = coin
|
||||
self.api = api
|
||||
@@ -106,7 +115,8 @@ class MemPool:
|
||||
# Prevents mempool refreshes during fee histogram calculation
|
||||
self.lock = asyncio.Lock()
|
||||
self.wakeup = asyncio.Event()
|
||||
self.executor = ThreadPoolExecutor(max(os.cpu_count() - 1, 1))
|
||||
self.mempool_process_time_metric = mempool_process_time_metric
|
||||
self.notified_mempool_txs = set()
|
||||
|
||||
async def _logging(self, synchronized_event):
|
||||
"""Print regular logs of mempool stats."""
|
||||
@@ -123,15 +133,11 @@ class MemPool:
|
||||
await synchronized_event.wait()
|
||||
|
||||
async def _refresh_histogram(self, synchronized_event):
|
||||
try:
|
||||
while True:
|
||||
await synchronized_event.wait()
|
||||
async with self.lock:
|
||||
# Threaded as can be expensive
|
||||
await asyncio.get_event_loop().run_in_executor(self.executor, self._update_histogram, 100_000)
|
||||
await asyncio.sleep(self.coin.MEMPOOL_HISTOGRAM_REFRESH_SECS)
|
||||
finally:
|
||||
self.executor.shutdown(wait=True)
|
||||
while True:
|
||||
await synchronized_event.wait()
|
||||
async with self.lock:
|
||||
self._update_histogram(100_000)
|
||||
await asyncio.sleep(self.coin.MEMPOOL_HISTOGRAM_REFRESH_SECS)
|
||||
|
||||
def _update_histogram(self, bin_size):
|
||||
# Build a histogram by fee rate
|
||||
@@ -207,16 +213,24 @@ class MemPool:
|
||||
async def _refresh_hashes(self, synchronized_event):
|
||||
"""Refresh our view of the daemon's mempool."""
|
||||
while True:
|
||||
start = time.perf_counter()
|
||||
height = self.api.cached_height()
|
||||
hex_hashes = await self.api.mempool_hashes()
|
||||
if height != await self.api.height():
|
||||
continue
|
||||
hashes = {hex_str_to_hash(hh) for hh in hex_hashes}
|
||||
async with self.lock:
|
||||
new_hashes = hashes.difference(self.notified_mempool_txs)
|
||||
touched = await self._process_mempool(hashes)
|
||||
self.notified_mempool_txs.update(new_hashes)
|
||||
new_touched = {
|
||||
touched_hashx for touched_hashx, txs in self.hashXs.items() if txs.intersection(new_hashes)
|
||||
}
|
||||
synchronized_event.set()
|
||||
synchronized_event.clear()
|
||||
await self.api.on_mempool(touched, height)
|
||||
await self.api.on_mempool(touched, new_touched, height)
|
||||
duration = time.perf_counter() - start
|
||||
self.mempool_process_time_metric.observe(duration)
|
||||
try:
|
||||
# we wait up to `refresh_secs` but go early if a broadcast happens (which triggers wakeup event)
|
||||
await asyncio.wait_for(self.wakeup.wait(), timeout=self.refresh_secs)
|
||||
@@ -228,7 +242,8 @@ class MemPool:
|
||||
async def _process_mempool(self, all_hashes):
|
||||
# Re-sync with the new set of hashes
|
||||
txs = self.txs
|
||||
hashXs = self.hashXs
|
||||
|
||||
hashXs = self.hashXs # hashX: [tx_hash, ...]
|
||||
touched = set()
|
||||
|
||||
# First handle txs that have disappeared
|
||||
@@ -259,8 +274,8 @@ class MemPool:
|
||||
# FIXME: this is not particularly efficient
|
||||
while tx_map and len(tx_map) != prior_count:
|
||||
prior_count = len(tx_map)
|
||||
tx_map, utxo_map = self._accept_transactions(tx_map, utxo_map,
|
||||
touched)
|
||||
tx_map, utxo_map = self._accept_transactions(tx_map, utxo_map, touched)
|
||||
|
||||
if tx_map:
|
||||
self.logger.info(f'{len(tx_map)} txs dropped')
|
||||
|
||||
@@ -271,30 +286,25 @@ class MemPool:
|
||||
hex_hashes_iter = (hash_to_hex_str(hash) for hash in hashes)
|
||||
raw_txs = await self.api.raw_transactions(hex_hashes_iter)
|
||||
|
||||
def deserialize_txs(): # This function is pure
|
||||
to_hashX = self.coin.hashX_from_script
|
||||
deserializer = self.coin.DESERIALIZER
|
||||
to_hashX = self.coin.hashX_from_script
|
||||
deserializer = self.coin.DESERIALIZER
|
||||
|
||||
txs = {}
|
||||
for hash, raw_tx in zip(hashes, raw_txs):
|
||||
# The daemon may have evicted the tx from its
|
||||
# mempool or it may have gotten in a block
|
||||
if not raw_tx:
|
||||
continue
|
||||
tx, tx_size = deserializer(raw_tx).read_tx_and_vsize()
|
||||
# Convert the inputs and outputs into (hashX, value) pairs
|
||||
# Drop generation-like inputs from MemPoolTx.prevouts
|
||||
txin_pairs = tuple((txin.prev_hash, txin.prev_idx)
|
||||
for txin in tx.inputs
|
||||
if not txin.is_generation())
|
||||
txout_pairs = tuple((to_hashX(txout.pk_script), txout.value)
|
||||
for txout in tx.outputs)
|
||||
txs[hash] = MemPoolTx(txin_pairs, None, txout_pairs,
|
||||
0, tx_size)
|
||||
return txs
|
||||
|
||||
# Thread this potentially slow operation so as not to block
|
||||
tx_map = await asyncio.get_event_loop().run_in_executor(self.executor, deserialize_txs)
|
||||
tx_map = {}
|
||||
for hash, raw_tx in zip(hashes, raw_txs):
|
||||
# The daemon may have evicted the tx from its
|
||||
# mempool or it may have gotten in a block
|
||||
if not raw_tx:
|
||||
continue
|
||||
tx, tx_size = deserializer(raw_tx).read_tx_and_vsize()
|
||||
# Convert the inputs and outputs into (hashX, value) pairs
|
||||
# Drop generation-like inputs from MemPoolTx.prevouts
|
||||
txin_pairs = tuple((txin.prev_hash, txin.prev_idx)
|
||||
for txin in tx.inputs
|
||||
if not txin.is_generation())
|
||||
txout_pairs = tuple((to_hashX(txout.pk_script), txout.value)
|
||||
for txout in tx.outputs)
|
||||
tx_map[hash] = MemPoolTx(txin_pairs, None, txout_pairs,
|
||||
0, tx_size)
|
||||
|
||||
# Determine all prevouts not in the mempool, and fetch the
|
||||
# UTXO information from the database. Failed prevout lookups
|
||||
@@ -317,7 +327,7 @@ class MemPool:
|
||||
"""Keep the mempool synchronized with the daemon."""
|
||||
await asyncio.wait([
|
||||
self._refresh_hashes(synchronized_event),
|
||||
self._refresh_histogram(synchronized_event),
|
||||
# self._refresh_histogram(synchronized_event),
|
||||
self._logging(synchronized_event)
|
||||
])
|
||||
|
||||
@@ -334,7 +344,7 @@ class MemPool:
|
||||
value += sum(v for h168, v in tx.out_pairs if h168 == hashX)
|
||||
return value
|
||||
|
||||
async def compact_fee_histogram(self):
|
||||
def compact_fee_histogram(self):
|
||||
"""Return a compact fee histogram of the current mempool."""
|
||||
return self.cached_compact_histogram
|
||||
|
||||
@@ -351,7 +361,7 @@ class MemPool:
|
||||
result.update(tx.prevouts)
|
||||
return result
|
||||
|
||||
async def transaction_summaries(self, hashX):
|
||||
def transaction_summaries(self, hashX):
|
||||
"""Return a list of MemPoolTxSummary objects for the hashX."""
|
||||
result = []
|
||||
for tx_hash in self.hashXs.get(hashX, ()):
|
||||
@@ -374,3 +384,17 @@ class MemPool:
|
||||
if hX == hashX:
|
||||
utxos.append(UTXO(-1, pos, tx_hash, 0, value))
|
||||
return utxos
|
||||
|
||||
def get_mempool_height(self, tx_hash):
|
||||
# Height Progression
|
||||
# -2: not broadcast
|
||||
# -1: in mempool but has unconfirmed inputs
|
||||
# 0: in mempool and all inputs confirmed
|
||||
# +num: confirmed in a specific block (height)
|
||||
if tx_hash not in self.txs:
|
||||
return -2
|
||||
tx = self.txs[tx_hash]
|
||||
unspent_inputs = sum(1 if hash in self.txs else 0 for hash, idx in tx.prevouts)
|
||||
if unspent_inputs:
|
||||
return -1
|
||||
return 0
|
||||
@@ -63,7 +63,7 @@ class Merkle:
|
||||
raise TypeError('index must be an integer')
|
||||
# This also asserts hashes is not empty
|
||||
if not 0 <= index < len(hashes):
|
||||
raise ValueError('index out of range')
|
||||
raise ValueError(f"index '{index}/{len(hashes)}' out of range")
|
||||
natural_length = self.branch_length(len(hashes))
|
||||
if length is None:
|
||||
length = natural_length
|
||||
|
||||
@@ -319,7 +319,7 @@ class PeerManager:
|
||||
|
||||
# Check prior header too in case of hard fork.
|
||||
check_height = min(our_height, their_height)
|
||||
raw_header = await self.db.raw_header(check_height)
|
||||
raw_header = self.db.raw_header(check_height)
|
||||
if ptuple >= (1, 4):
|
||||
ours = raw_header.hex()
|
||||
message = 'blockchain.block.header'
|
||||
|
||||
@@ -1,89 +0,0 @@
|
||||
import os
|
||||
from aiohttp import web
|
||||
from prometheus_client import Counter, Info, generate_latest as prom_generate_latest, Histogram, Gauge
|
||||
from lbry import __version__ as version
|
||||
from lbry.build_info import BUILD, COMMIT_HASH, DOCKER_TAG
|
||||
from lbry.wallet.server import util
|
||||
import lbry.wallet.server.version as wallet_server_version
|
||||
|
||||
NAMESPACE = "wallet_server"
|
||||
CPU_COUNT = f"{os.cpu_count()}"
|
||||
VERSION_INFO = Info('build', 'Wallet server build info (e.g. version, commit hash)', namespace=NAMESPACE)
|
||||
VERSION_INFO.info({
|
||||
'build': BUILD,
|
||||
"commit": COMMIT_HASH,
|
||||
"docker_tag": DOCKER_TAG,
|
||||
'version': version,
|
||||
"min_version": util.version_string(wallet_server_version.PROTOCOL_MIN),
|
||||
"cpu_count": CPU_COUNT
|
||||
})
|
||||
SESSIONS_COUNT = Gauge("session_count", "Number of connected client sessions", namespace=NAMESPACE,
|
||||
labelnames=("version", ))
|
||||
REQUESTS_COUNT = Counter("requests_count", "Number of requests received", namespace=NAMESPACE,
|
||||
labelnames=("method", "version"))
|
||||
RESPONSE_TIMES = Histogram("response_time", "Response times", namespace=NAMESPACE, labelnames=("method", "version"))
|
||||
NOTIFICATION_COUNT = Counter("notification", "Number of notifications sent (for subscriptions)",
|
||||
namespace=NAMESPACE, labelnames=("method", "version"))
|
||||
REQUEST_ERRORS_COUNT = Counter("request_error", "Number of requests that returned errors", namespace=NAMESPACE,
|
||||
labelnames=("method", "version"))
|
||||
SQLITE_INTERRUPT_COUNT = Counter("interrupt", "Number of interrupted queries", namespace=NAMESPACE)
|
||||
SQLITE_OPERATIONAL_ERROR_COUNT = Counter(
|
||||
"operational_error", "Number of queries that raised operational errors", namespace=NAMESPACE
|
||||
)
|
||||
SQLITE_INTERNAL_ERROR_COUNT = Counter(
|
||||
"internal_error", "Number of queries raising unexpected errors", namespace=NAMESPACE
|
||||
)
|
||||
SQLITE_EXECUTOR_TIMES = Histogram("executor_time", "SQLite executor times", namespace=NAMESPACE)
|
||||
SQLITE_PENDING_COUNT = Gauge(
|
||||
"pending_queries_count", "Number of pending and running sqlite queries", namespace=NAMESPACE
|
||||
)
|
||||
LBRYCRD_REQUEST_TIMES = Histogram(
|
||||
"lbrycrd_request", "lbrycrd requests count", namespace=NAMESPACE, labelnames=("method",)
|
||||
)
|
||||
LBRYCRD_PENDING_COUNT = Gauge(
|
||||
"lbrycrd_pending_count", "Number of lbrycrd rpcs that are in flight", namespace=NAMESPACE, labelnames=("method",)
|
||||
)
|
||||
CLIENT_VERSIONS = Counter(
|
||||
"clients", "Number of connections received per client version",
|
||||
namespace=NAMESPACE, labelnames=("version",)
|
||||
)
|
||||
BLOCK_COUNT = Gauge(
|
||||
"block_count", "Number of processed blocks", namespace=NAMESPACE
|
||||
)
|
||||
BLOCK_UPDATE_TIMES = Histogram("block_time", "Block update times", namespace=NAMESPACE)
|
||||
REORG_COUNT = Gauge(
|
||||
"reorg_count", "Number of reorgs", namespace=NAMESPACE
|
||||
)
|
||||
RESET_CONNECTIONS = Counter(
|
||||
"reset_clients", "Number of reset connections by client version",
|
||||
namespace=NAMESPACE, labelnames=("version",)
|
||||
)
|
||||
|
||||
|
||||
class PrometheusServer:
|
||||
def __init__(self):
|
||||
self.logger = util.class_logger(__name__, self.__class__.__name__)
|
||||
self.runner = None
|
||||
|
||||
async def start(self, port: int):
|
||||
prom_app = web.Application()
|
||||
prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
|
||||
self.runner = web.AppRunner(prom_app)
|
||||
await self.runner.setup()
|
||||
|
||||
metrics_site = web.TCPSite(self.runner, "0.0.0.0", port, shutdown_timeout=.5)
|
||||
await metrics_site.start()
|
||||
self.logger.info('metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2])
|
||||
|
||||
async def handle_metrics_get_request(self, request: web.Request):
|
||||
try:
|
||||
return web.Response(
|
||||
text=prom_generate_latest().decode(),
|
||||
content_type='text/plain; version=0.0.4'
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception('could not generate prometheus data')
|
||||
raise
|
||||
|
||||
async def stop(self):
|
||||
await self.runner.cleanup()
|
||||
@@ -6,7 +6,7 @@ import typing
|
||||
|
||||
import lbry
|
||||
from lbry.wallet.server.mempool import MemPool, MemPoolAPI
|
||||
from lbry.wallet.server.prometheus import PrometheusServer
|
||||
from lbry.prometheus import PrometheusServer
|
||||
|
||||
|
||||
class Notifications:
|
||||
@@ -25,9 +25,10 @@ class Notifications:
|
||||
def __init__(self):
|
||||
self._touched_mp = {}
|
||||
self._touched_bp = {}
|
||||
self.notified_mempool_txs = set()
|
||||
self._highest_block = -1
|
||||
|
||||
async def _maybe_notify(self):
|
||||
async def _maybe_notify(self, new_touched):
|
||||
tmp, tbp = self._touched_mp, self._touched_bp
|
||||
common = set(tmp).intersection(tbp)
|
||||
if common:
|
||||
@@ -44,24 +45,24 @@ class Notifications:
|
||||
del tmp[old]
|
||||
for old in [h for h in tbp if h <= height]:
|
||||
touched.update(tbp.pop(old))
|
||||
await self.notify(height, touched)
|
||||
await self.notify(height, touched, new_touched)
|
||||
|
||||
async def notify(self, height, touched):
|
||||
async def notify(self, height, touched, new_touched):
|
||||
pass
|
||||
|
||||
async def start(self, height, notify_func):
|
||||
self._highest_block = height
|
||||
self.notify = notify_func
|
||||
await self.notify(height, set())
|
||||
await self.notify(height, set(), set())
|
||||
|
||||
async def on_mempool(self, touched, height):
|
||||
async def on_mempool(self, touched, new_touched, height):
|
||||
self._touched_mp[height] = touched
|
||||
await self._maybe_notify()
|
||||
await self._maybe_notify(new_touched)
|
||||
|
||||
async def on_block(self, touched, height):
|
||||
self._touched_bp[height] = touched
|
||||
self._highest_block = height
|
||||
await self._maybe_notify()
|
||||
await self._maybe_notify(set())
|
||||
|
||||
|
||||
class Server:
|
||||
@@ -84,9 +85,12 @@ class Server:
|
||||
notifications.mempool_hashes = daemon.mempool_hashes
|
||||
notifications.raw_transactions = daemon.getrawtransactions
|
||||
notifications.lookup_utxos = db.lookup_utxos
|
||||
|
||||
MemPoolAPI.register(Notifications)
|
||||
self.mempool = mempool = MemPool(env.coin, notifications)
|
||||
|
||||
notifications.notified_mempool_txs = self.mempool.notified_mempool_txs
|
||||
|
||||
self.session_mgr = env.coin.SESSION_MANAGER(
|
||||
env, db, bp, daemon, mempool, self.shutdown_event
|
||||
)
|
||||
@@ -106,11 +110,14 @@ class Server:
|
||||
self.cancellable_tasks.append(asyncio.ensure_future(run(*args, _flag)))
|
||||
return _flag.wait()
|
||||
|
||||
await self.start_prometheus()
|
||||
await self.bp.status_server.start(0, bytes.fromhex(self.bp.coin.GENESIS_HASH)[::-1]
|
||||
, self.env.host, self.env.tcp_port)
|
||||
await _start_cancellable(self.bp.fetch_and_process_blocks)
|
||||
|
||||
await self.db.populate_header_merkle_cache()
|
||||
await _start_cancellable(self.mempool.keep_synchronized)
|
||||
await _start_cancellable(self.session_mgr.serve, self.notifications)
|
||||
await self.start_prometheus()
|
||||
|
||||
async def stop(self):
|
||||
for task in reversed(self.cancellable_tasks):
|
||||
@@ -143,4 +150,4 @@ class Server:
|
||||
async def start_prometheus(self):
|
||||
if not self.prometheus_server and self.env.prometheus_port:
|
||||
self.prometheus_server = PrometheusServer()
|
||||
await self.prometheus_server.start(self.env.prometheus_port)
|
||||
await self.prometheus_server.start("0.0.0.0", self.env.prometheus_port)
|
||||
+231
-231
@@ -4,7 +4,6 @@ import math
|
||||
import time
|
||||
import json
|
||||
import zlib
|
||||
import pylru
|
||||
import base64
|
||||
import codecs
|
||||
import typing
|
||||
@@ -17,19 +16,18 @@ from asyncio import Event, sleep
|
||||
from collections import defaultdict
|
||||
from functools import partial
|
||||
|
||||
from binascii import hexlify
|
||||
from pylru import lrucache
|
||||
from binascii import hexlify, unhexlify
|
||||
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
|
||||
from prometheus_client import Counter, Info, Histogram, Gauge
|
||||
|
||||
import lbry
|
||||
from lbry.utils import LRUCacheWithMetrics
|
||||
from lbry.build_info import BUILD, COMMIT_HASH, DOCKER_TAG
|
||||
from lbry.wallet.server.block_processor import LBRYBlockProcessor
|
||||
from lbry.wallet.server.db.writer import LBRYLevelDB
|
||||
from lbry.wallet.server.db import reader
|
||||
from lbry.wallet.server.websocket import AdminWebSocket
|
||||
from lbry.wallet.server.metrics import ServerLoadData, APICallMetrics
|
||||
from lbry.wallet.server.prometheus import REQUESTS_COUNT, SQLITE_INTERRUPT_COUNT, SQLITE_INTERNAL_ERROR_COUNT
|
||||
from lbry.wallet.server.prometheus import SQLITE_OPERATIONAL_ERROR_COUNT, SQLITE_EXECUTOR_TIMES, SESSIONS_COUNT
|
||||
from lbry.wallet.server.prometheus import SQLITE_PENDING_COUNT, CLIENT_VERSIONS
|
||||
from lbry.wallet.rpc.framing import NewlineFramer
|
||||
import lbry.wallet.server.version as VERSION
|
||||
|
||||
@@ -119,9 +117,66 @@ class SessionGroup:
|
||||
self.semaphore = asyncio.Semaphore(20)
|
||||
|
||||
|
||||
NAMESPACE = "wallet_server"
|
||||
HISTOGRAM_BUCKETS = (
|
||||
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
|
||||
)
|
||||
|
||||
|
||||
class SessionManager:
|
||||
"""Holds global state about all sessions."""
|
||||
|
||||
version_info_metric = Info(
|
||||
'build', 'Wallet server build info (e.g. version, commit hash)', namespace=NAMESPACE
|
||||
)
|
||||
version_info_metric.info({
|
||||
'build': BUILD,
|
||||
"commit": COMMIT_HASH,
|
||||
"docker_tag": DOCKER_TAG,
|
||||
'version': lbry.__version__,
|
||||
"min_version": util.version_string(VERSION.PROTOCOL_MIN),
|
||||
"cpu_count": str(os.cpu_count())
|
||||
})
|
||||
session_count_metric = Gauge("session_count", "Number of connected client sessions", namespace=NAMESPACE,
|
||||
labelnames=("version",))
|
||||
request_count_metric = Counter("requests_count", "Number of requests received", namespace=NAMESPACE,
|
||||
labelnames=("method", "version"))
|
||||
tx_request_count_metric = Counter("requested_transaction", "Number of transactions requested", namespace=NAMESPACE)
|
||||
tx_replied_count_metric = Counter("replied_transaction", "Number of transactions responded", namespace=NAMESPACE)
|
||||
urls_to_resolve_count_metric = Counter("urls_to_resolve", "Number of urls to resolve", namespace=NAMESPACE)
|
||||
resolved_url_count_metric = Counter("resolved_url", "Number of resolved urls", namespace=NAMESPACE)
|
||||
|
||||
interrupt_count_metric = Counter("interrupt", "Number of interrupted queries", namespace=NAMESPACE)
|
||||
db_operational_error_metric = Counter(
|
||||
"operational_error", "Number of queries that raised operational errors", namespace=NAMESPACE
|
||||
)
|
||||
db_error_metric = Counter(
|
||||
"internal_error", "Number of queries raising unexpected errors", namespace=NAMESPACE
|
||||
)
|
||||
executor_time_metric = Histogram(
|
||||
"executor_time", "SQLite executor times", namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS
|
||||
)
|
||||
pending_query_metric = Gauge(
|
||||
"pending_queries_count", "Number of pending and running sqlite queries", namespace=NAMESPACE
|
||||
)
|
||||
|
||||
client_version_metric = Counter(
|
||||
"clients", "Number of connections received per client version",
|
||||
namespace=NAMESPACE, labelnames=("version",)
|
||||
)
|
||||
address_history_metric = Histogram(
|
||||
"address_history", "Time to fetch an address history",
|
||||
namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS
|
||||
)
|
||||
notifications_in_flight_metric = Gauge(
|
||||
"notifications_in_flight", "Count of notifications in flight",
|
||||
namespace=NAMESPACE
|
||||
)
|
||||
notifications_sent_metric = Histogram(
|
||||
"notifications_sent", "Time to send an address notification",
|
||||
namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS
|
||||
)
|
||||
|
||||
def __init__(self, env: 'Env', db: LBRYLevelDB, bp: LBRYBlockProcessor, daemon: 'Daemon', mempool: 'MemPool',
|
||||
shutdown_event: asyncio.Event):
|
||||
env.max_send = max(350000, env.max_send)
|
||||
@@ -134,11 +189,13 @@ class SessionManager:
|
||||
self.shutdown_event = shutdown_event
|
||||
self.logger = util.class_logger(__name__, self.__class__.__name__)
|
||||
self.servers: typing.Dict[str, asyncio.AbstractServer] = {}
|
||||
self.sessions: typing.Set['SessionBase'] = set()
|
||||
self.sessions: typing.Dict[int, 'SessionBase'] = {}
|
||||
self.hashx_subscriptions_by_session: typing.DefaultDict[str, typing.Set[int]] = defaultdict(set)
|
||||
self.mempool_statuses = {}
|
||||
self.cur_group = SessionGroup(0)
|
||||
self.txs_sent = 0
|
||||
self.start_time = time.time()
|
||||
self.history_cache = pylru.lrucache(256)
|
||||
self.history_cache = self.bp.history_cache
|
||||
self.notified_height: typing.Optional[int] = None
|
||||
# Cache some idea of room to avoid recounting on each subscription
|
||||
self.subs_room = 0
|
||||
@@ -190,11 +247,12 @@ class SessionManager:
|
||||
async def _manage_servers(self):
|
||||
paused = False
|
||||
max_sessions = self.env.max_sessions
|
||||
low_watermark = max_sessions * 19 // 20
|
||||
low_watermark = int(max_sessions * 0.95)
|
||||
while True:
|
||||
await self.session_event.wait()
|
||||
self.session_event.clear()
|
||||
if not paused and len(self.sessions) >= max_sessions:
|
||||
self.bp.status_server.set_unavailable()
|
||||
self.logger.info(f'maximum sessions {max_sessions:,d} '
|
||||
f'reached, stopping new connections until '
|
||||
f'count drops to {low_watermark:,d}')
|
||||
@@ -203,6 +261,7 @@ class SessionManager:
|
||||
# Start listening for incoming connections if paused and
|
||||
# session count has fallen
|
||||
if paused and len(self.sessions) <= low_watermark:
|
||||
self.bp.status_server.set_available()
|
||||
self.logger.info('resuming listening for incoming connections')
|
||||
await self._start_external_servers()
|
||||
paused = False
|
||||
@@ -220,12 +279,12 @@ class SessionManager:
|
||||
|
||||
def _group_map(self):
|
||||
group_map = defaultdict(list)
|
||||
for session in self.sessions:
|
||||
for session in self.sessions.values():
|
||||
group_map[session.group].append(session)
|
||||
return group_map
|
||||
|
||||
def _sub_count(self) -> int:
|
||||
return sum(s.sub_count() for s in self.sessions)
|
||||
return sum(s.sub_count() for s in self.sessions.values())
|
||||
|
||||
def _lookup_session(self, session_id):
|
||||
try:
|
||||
@@ -233,7 +292,7 @@ class SessionManager:
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
for session in self.sessions:
|
||||
for session in self.sessions.values():
|
||||
if session.session_id == session_id:
|
||||
return session
|
||||
return None
|
||||
@@ -257,7 +316,7 @@ class SessionManager:
|
||||
while True:
|
||||
await sleep(session_timeout // 10)
|
||||
stale_cutoff = time.perf_counter() - session_timeout
|
||||
stale_sessions = [session for session in self.sessions
|
||||
stale_sessions = [session for session in self.sessions.values()
|
||||
if session.last_recv < stale_cutoff]
|
||||
if stale_sessions:
|
||||
text = ', '.join(str(session.session_id)
|
||||
@@ -289,7 +348,7 @@ class SessionManager:
|
||||
pending_requests = 0
|
||||
closing = 0
|
||||
|
||||
for s in self.sessions:
|
||||
for s in self.sessions.values():
|
||||
error_count += s.errors
|
||||
if s.log_me:
|
||||
logged += 1
|
||||
@@ -323,7 +382,7 @@ class SessionManager:
|
||||
def _session_data(self, for_log):
|
||||
"""Returned to the RPC 'sessions' call."""
|
||||
now = time.time()
|
||||
sessions = sorted(self.sessions, key=lambda s: s.start_time)
|
||||
sessions = sorted(self.sessions.values(), key=lambda s: s.start_time)
|
||||
return [(session.session_id,
|
||||
session.flags(),
|
||||
session.peer_address_str(for_log=for_log),
|
||||
@@ -515,6 +574,7 @@ class SessionManager:
|
||||
await self.start_other()
|
||||
await self._start_external_servers()
|
||||
server_listening_event.set()
|
||||
self.bp.status_server.set_available()
|
||||
# Peer discovery should start after the external servers
|
||||
# because we connect to ourself
|
||||
await asyncio.wait([
|
||||
@@ -525,9 +585,10 @@ class SessionManager:
|
||||
])
|
||||
finally:
|
||||
await self._close_servers(list(self.servers.keys()))
|
||||
log.warning("disconnect %i sessions", len(self.sessions))
|
||||
if self.sessions:
|
||||
await asyncio.wait([
|
||||
session.close(force_after=1) for session in self.sessions
|
||||
session.close(force_after=1) for session in self.sessions.values()
|
||||
])
|
||||
await self.stop_other()
|
||||
|
||||
@@ -551,7 +612,7 @@ class SessionManager:
|
||||
async def raw_header(self, height):
|
||||
"""Return the binary header at the given height."""
|
||||
try:
|
||||
return await self.db.raw_header(height)
|
||||
return self.db.raw_header(height)
|
||||
except IndexError:
|
||||
raise RPCError(BAD_REQUEST, f'height {height:,d} '
|
||||
'out of range') from None
|
||||
@@ -563,39 +624,56 @@ class SessionManager:
|
||||
|
||||
async def broadcast_transaction(self, raw_tx):
|
||||
hex_hash = await self.daemon.broadcast_transaction(raw_tx)
|
||||
self.mempool.wakeup.set()
|
||||
self.txs_sent += 1
|
||||
return hex_hash
|
||||
|
||||
async def limited_history(self, hashX):
|
||||
"""A caching layer."""
|
||||
hc = self.history_cache
|
||||
if hashX not in hc:
|
||||
if hashX not in self.history_cache:
|
||||
# History DoS limit. Each element of history is about 99
|
||||
# bytes when encoded as JSON. This limits resource usage
|
||||
# on bloated history requests, and uses a smaller divisor
|
||||
# so large requests are logged before refusing them.
|
||||
limit = self.env.max_send // 97
|
||||
hc[hashX] = await self.db.limited_history(hashX, limit=limit)
|
||||
return hc[hashX]
|
||||
self.history_cache[hashX] = await self.db.limited_history(hashX, limit=limit)
|
||||
return self.history_cache[hashX]
|
||||
|
||||
async def _notify_sessions(self, height, touched):
|
||||
async def _notify_sessions(self, height, touched, new_touched):
|
||||
"""Notify sessions about height changes and touched addresses."""
|
||||
height_changed = height != self.notified_height
|
||||
if height_changed:
|
||||
await self._refresh_hsub_results(height)
|
||||
# Invalidate our history cache for touched hashXs
|
||||
hc = self.history_cache
|
||||
for hashX in set(hc).intersection(touched):
|
||||
del hc[hashX]
|
||||
|
||||
if self.sessions:
|
||||
await asyncio.wait([
|
||||
session.notify(touched, height_changed) for session in self.sessions
|
||||
])
|
||||
if not self.sessions:
|
||||
return
|
||||
|
||||
if height_changed:
|
||||
header_tasks = [
|
||||
session.send_notification('blockchain.headers.subscribe', (self.hsub_results[session.subscribe_headers_raw], ))
|
||||
for session in self.sessions.values() if session.subscribe_headers
|
||||
]
|
||||
if header_tasks:
|
||||
self.logger.info(f'notify {len(header_tasks)} sessions of new header')
|
||||
asyncio.create_task(asyncio.wait(header_tasks))
|
||||
for hashX in touched.intersection(self.mempool_statuses.keys()):
|
||||
self.mempool_statuses.pop(hashX, None)
|
||||
|
||||
touched.intersection_update(self.hashx_subscriptions_by_session.keys())
|
||||
|
||||
if touched or (height_changed and self.mempool_statuses):
|
||||
notified_hashxs = 0
|
||||
notified_sessions = 0
|
||||
to_notify = touched if height_changed else new_touched
|
||||
for hashX in to_notify:
|
||||
for session_id in self.hashx_subscriptions_by_session[hashX]:
|
||||
asyncio.create_task(self.sessions[session_id].send_history_notification(hashX))
|
||||
notified_sessions += 1
|
||||
notified_hashxs += 1
|
||||
if notified_sessions:
|
||||
self.logger.info(f'notified {notified_sessions} sessions/{notified_hashxs:,d} touched addresses')
|
||||
|
||||
def add_session(self, session):
|
||||
self.sessions.add(session)
|
||||
self.sessions[id(session)] = session
|
||||
self.session_event.set()
|
||||
gid = int(session.start_time - self.start_time) // 900
|
||||
if self.cur_group.gid != gid:
|
||||
@@ -604,7 +682,13 @@ class SessionManager:
|
||||
|
||||
def remove_session(self, session):
|
||||
"""Remove a session from our sessions list if there."""
|
||||
self.sessions.remove(session)
|
||||
session_id = id(session)
|
||||
for hashX in session.hashX_subs:
|
||||
sessions = self.hashx_subscriptions_by_session[hashX]
|
||||
sessions.remove(session_id)
|
||||
if not sessions:
|
||||
self.hashx_subscriptions_by_session.pop(hashX)
|
||||
self.sessions.pop(session_id)
|
||||
self.session_event.set()
|
||||
|
||||
|
||||
@@ -639,8 +723,6 @@ class SessionBase(RPCSession):
|
||||
self._receive_message_orig = self.connection.receive_message
|
||||
self.connection.receive_message = self.receive_message
|
||||
|
||||
async def notify(self, touched, height_changed):
|
||||
pass
|
||||
|
||||
def default_framer(self):
|
||||
return NewlineFramer(self.env.max_receive)
|
||||
@@ -677,7 +759,7 @@ class SessionBase(RPCSession):
|
||||
context = {'conn_id': f'{self.session_id}'}
|
||||
self.logger = util.ConnectionLogger(self.logger, context)
|
||||
self.group = self.session_mgr.add_session(self)
|
||||
SESSIONS_COUNT.labels(version=self.client_version).inc()
|
||||
self.session_mgr.session_count_metric.labels(version=self.client_version).inc()
|
||||
peer_addr_str = self.peer_address_str()
|
||||
self.logger.info(f'{self.kind} {peer_addr_str}, '
|
||||
f'{self.session_mgr.session_count():,d} total')
|
||||
@@ -686,7 +768,7 @@ class SessionBase(RPCSession):
|
||||
"""Handle client disconnection."""
|
||||
super().connection_lost(exc)
|
||||
self.session_mgr.remove_session(self)
|
||||
SESSIONS_COUNT.labels(version=self.client_version).dec()
|
||||
self.session_mgr.session_count_metric.labels(version=self.client_version).dec()
|
||||
msg = ''
|
||||
if not self._can_send.is_set():
|
||||
msg += ' whilst paused'
|
||||
@@ -710,7 +792,7 @@ class SessionBase(RPCSession):
|
||||
"""Handle an incoming request. ElectrumX doesn't receive
|
||||
notifications from client sessions.
|
||||
"""
|
||||
REQUESTS_COUNT.labels(method=request.method, version=self.client_version).inc()
|
||||
self.session_mgr.request_count_metric.labels(method=request.method, version=self.client_version).inc()
|
||||
if isinstance(request, Request):
|
||||
handler = self.request_handlers.get(request.method)
|
||||
handler = partial(handler, self)
|
||||
@@ -732,8 +814,8 @@ class LBRYSessionManager(SessionManager):
|
||||
if self.env.websocket_host is not None and self.env.websocket_port is not None:
|
||||
self.websocket = AdminWebSocket(self)
|
||||
self.search_cache = self.bp.search_cache
|
||||
self.search_cache['search'] = lrucache(10000)
|
||||
self.search_cache['resolve'] = lrucache(10000)
|
||||
self.search_cache['search'] = LRUCacheWithMetrics(2 ** 14, metric_name='search', namespace=NAMESPACE)
|
||||
self.search_cache['resolve'] = LRUCacheWithMetrics(2 ** 16, metric_name='resolve', namespace=NAMESPACE)
|
||||
|
||||
async def process_metrics(self):
|
||||
while self.running:
|
||||
@@ -786,6 +868,7 @@ class LBRYElectrumX(SessionBase):
|
||||
max_errors = math.inf # don't disconnect people for errors! let them happen...
|
||||
session_mgr: LBRYSessionManager
|
||||
version = lbry.__version__
|
||||
cached_server_features = {}
|
||||
|
||||
@classmethod
|
||||
def initialize_request_handlers(cls):
|
||||
@@ -832,14 +915,15 @@ class LBRYElectrumX(SessionBase):
|
||||
super().__init__(*args, **kwargs)
|
||||
if not LBRYElectrumX.request_handlers:
|
||||
LBRYElectrumX.initialize_request_handlers()
|
||||
if not LBRYElectrumX.cached_server_features:
|
||||
LBRYElectrumX.set_server_features(self.env)
|
||||
self.subscribe_headers = False
|
||||
self.subscribe_headers_raw = False
|
||||
self.connection.max_response_size = self.env.max_send
|
||||
self.hashX_subs = {}
|
||||
self.sv_seen = False
|
||||
self.mempool_statuses = {}
|
||||
self.protocol_tuple = self.PROTOCOL_MIN
|
||||
|
||||
self.protocol_string = None
|
||||
self.daemon = self.session_mgr.daemon
|
||||
self.bp: LBRYBlockProcessor = self.session_mgr.bp
|
||||
self.db: LBRYLevelDB = self.bp.db
|
||||
@@ -850,10 +934,10 @@ class LBRYElectrumX(SessionBase):
|
||||
for ver in (cls.PROTOCOL_MIN, cls.PROTOCOL_MAX)]
|
||||
|
||||
@classmethod
|
||||
def server_features(cls, env):
|
||||
def set_server_features(cls, env):
|
||||
"""Return the server features dictionary."""
|
||||
min_str, max_str = cls.protocol_min_max_strings()
|
||||
return {
|
||||
cls.cached_server_features.update({
|
||||
'hosts': env.hosts_dict(),
|
||||
'pruning': None,
|
||||
'server_version': cls.version,
|
||||
@@ -866,10 +950,10 @@ class LBRYElectrumX(SessionBase):
|
||||
'daily_fee': env.daily_fee,
|
||||
'hash_function': 'sha256',
|
||||
'trending_algorithm': env.trending_algorithms[0]
|
||||
}
|
||||
})
|
||||
|
||||
async def server_features_async(self):
|
||||
return self.server_features(self.env)
|
||||
return self.cached_server_features
|
||||
|
||||
@classmethod
|
||||
def server_version_args(cls):
|
||||
@@ -882,56 +966,22 @@ class LBRYElectrumX(SessionBase):
|
||||
def sub_count(self):
|
||||
return len(self.hashX_subs)
|
||||
|
||||
async def notify(self, touched, height_changed):
|
||||
"""Notify the client about changes to touched addresses (from mempool
|
||||
updates or new blocks) and height.
|
||||
"""
|
||||
if height_changed and self.subscribe_headers:
|
||||
args = (await self.subscribe_headers_result(), )
|
||||
try:
|
||||
await self.send_notification('blockchain.headers.subscribe', args)
|
||||
except asyncio.TimeoutError:
|
||||
self.logger.info("timeout sending headers notification to %s", self.peer_address_str(for_log=True))
|
||||
self.abort()
|
||||
return
|
||||
|
||||
touched = touched.intersection(self.hashX_subs)
|
||||
if touched or (height_changed and self.mempool_statuses):
|
||||
changed = {}
|
||||
|
||||
for hashX in touched:
|
||||
alias = self.hashX_subs[hashX]
|
||||
status = await self.address_status(hashX)
|
||||
changed[alias] = status
|
||||
|
||||
# Check mempool hashXs - the status is a function of the
|
||||
# confirmed state of other transactions. Note: we cannot
|
||||
# iterate over mempool_statuses as it changes size.
|
||||
for hashX in tuple(self.mempool_statuses):
|
||||
# Items can be evicted whilst await-ing status; False
|
||||
# ensures such hashXs are notified
|
||||
old_status = self.mempool_statuses.get(hashX, False)
|
||||
status = await self.address_status(hashX)
|
||||
if status != old_status:
|
||||
alias = self.hashX_subs[hashX]
|
||||
changed[alias] = status
|
||||
|
||||
for alias, status in changed.items():
|
||||
if len(alias) == 64:
|
||||
method = 'blockchain.scripthash.subscribe'
|
||||
else:
|
||||
method = 'blockchain.address.subscribe'
|
||||
|
||||
try:
|
||||
await self.send_notification(method, (alias, status))
|
||||
except asyncio.TimeoutError:
|
||||
self.logger.info("timeout sending address notification to %s", self.peer_address_str(for_log=True))
|
||||
self.abort()
|
||||
return
|
||||
|
||||
if changed:
|
||||
es = '' if len(changed) == 1 else 'es'
|
||||
self.logger.info(f'notified of {len(changed):,d} address{es}')
|
||||
async def send_history_notification(self, hashX):
|
||||
start = time.perf_counter()
|
||||
alias = self.hashX_subs[hashX]
|
||||
if len(alias) == 64:
|
||||
method = 'blockchain.scripthash.subscribe'
|
||||
else:
|
||||
method = 'blockchain.address.subscribe'
|
||||
try:
|
||||
self.session_mgr.notifications_in_flight_metric.inc()
|
||||
status = await self.address_status(hashX)
|
||||
self.session_mgr.address_history_metric.observe(time.perf_counter() - start)
|
||||
start = time.perf_counter()
|
||||
await self.send_notification(method, (alias, status))
|
||||
self.session_mgr.notifications_sent_metric.observe(time.perf_counter() - start)
|
||||
finally:
|
||||
self.session_mgr.notifications_in_flight_metric.dec()
|
||||
|
||||
def get_metrics_or_placeholder_for_api(self, query_name):
|
||||
""" Do not hold on to a reference to the metrics
|
||||
@@ -946,7 +996,7 @@ class LBRYElectrumX(SessionBase):
|
||||
async def run_in_executor(self, query_name, func, kwargs):
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
SQLITE_PENDING_COUNT.inc()
|
||||
self.session_mgr.pending_query_metric.inc()
|
||||
result = await asyncio.get_running_loop().run_in_executor(
|
||||
self.session_mgr.query_executor, func, kwargs
|
||||
)
|
||||
@@ -955,18 +1005,18 @@ class LBRYElectrumX(SessionBase):
|
||||
except reader.SQLiteInterruptedError as error:
|
||||
metrics = self.get_metrics_or_placeholder_for_api(query_name)
|
||||
metrics.query_interrupt(start, error.metrics)
|
||||
SQLITE_INTERRUPT_COUNT.inc()
|
||||
self.session_mgr.interrupt_count_metric.inc()
|
||||
raise RPCError(JSONRPC.QUERY_TIMEOUT, 'sqlite query timed out')
|
||||
except reader.SQLiteOperationalError as error:
|
||||
metrics = self.get_metrics_or_placeholder_for_api(query_name)
|
||||
metrics.query_error(start, error.metrics)
|
||||
SQLITE_OPERATIONAL_ERROR_COUNT.inc()
|
||||
self.session_mgr.db_operational_error_metric.inc()
|
||||
raise RPCError(JSONRPC.INTERNAL_ERROR, 'query failed to execute')
|
||||
except Exception:
|
||||
log.exception("dear devs, please handle this exception better")
|
||||
metrics = self.get_metrics_or_placeholder_for_api(query_name)
|
||||
metrics.query_error(start, {})
|
||||
SQLITE_INTERNAL_ERROR_COUNT.inc()
|
||||
self.session_mgr.db_error_metric.inc()
|
||||
raise RPCError(JSONRPC.INTERNAL_ERROR, 'unknown server error')
|
||||
else:
|
||||
if self.env.track_metrics:
|
||||
@@ -975,8 +1025,8 @@ class LBRYElectrumX(SessionBase):
|
||||
metrics.query_response(start, metrics_data)
|
||||
return base64.b64encode(result).decode()
|
||||
finally:
|
||||
SQLITE_PENDING_COUNT.dec()
|
||||
SQLITE_EXECUTOR_TIMES.observe(time.perf_counter() - start)
|
||||
self.session_mgr.pending_query_metric.dec()
|
||||
self.session_mgr.executor_time_metric.observe(time.perf_counter() - start)
|
||||
|
||||
async def run_and_cache_query(self, query_name, function, kwargs):
|
||||
metrics = self.get_metrics_or_placeholder_for_api(query_name)
|
||||
@@ -1008,7 +1058,12 @@ class LBRYElectrumX(SessionBase):
|
||||
|
||||
async def claimtrie_resolve(self, *urls):
|
||||
if urls:
|
||||
return await self.run_and_cache_query('resolve', reader.resolve_to_bytes, urls)
|
||||
count = len(urls)
|
||||
try:
|
||||
self.session_mgr.urls_to_resolve_count_metric.inc(count)
|
||||
return await self.run_and_cache_query('resolve', reader.resolve_to_bytes, urls)
|
||||
finally:
|
||||
self.session_mgr.resolved_url_count_metric.inc(count)
|
||||
|
||||
async def get_server_height(self):
|
||||
return self.bp.height
|
||||
@@ -1141,8 +1196,9 @@ class LBRYElectrumX(SessionBase):
|
||||
"""
|
||||
# Note history is ordered and mempool unordered in electrum-server
|
||||
# For mempool, height is -1 if it has unconfirmed inputs, otherwise 0
|
||||
|
||||
db_history = await self.session_mgr.limited_history(hashX)
|
||||
mempool = await self.mempool.transaction_summaries(hashX)
|
||||
mempool = self.mempool.transaction_summaries(hashX)
|
||||
|
||||
status = ''.join(f'{hash_to_hex_str(tx_hash)}:'
|
||||
f'{height:d}:'
|
||||
@@ -1156,10 +1212,9 @@ class LBRYElectrumX(SessionBase):
|
||||
status = None
|
||||
|
||||
if mempool:
|
||||
self.mempool_statuses[hashX] = status
|
||||
self.session_mgr.mempool_statuses[hashX] = status
|
||||
else:
|
||||
self.mempool_statuses.pop(hashX, None)
|
||||
|
||||
self.session_mgr.mempool_statuses.pop(hashX, None)
|
||||
return status
|
||||
|
||||
async def hashX_listunspent(self, hashX):
|
||||
@@ -1178,10 +1233,14 @@ class LBRYElectrumX(SessionBase):
|
||||
|
||||
async def hashX_subscribe(self, hashX, alias):
|
||||
self.hashX_subs[hashX] = alias
|
||||
self.session_mgr.hashx_subscriptions_by_session[hashX].add(id(self))
|
||||
return await self.address_status(hashX)
|
||||
|
||||
async def hashX_unsubscribe(self, hashX, alias):
|
||||
del self.hashX_subs[hashX]
|
||||
sessions = self.session_mgr.hashx_subscriptions_by_session[hashX]
|
||||
sessions.remove(id(self))
|
||||
if not sessions:
|
||||
self.hashX_subs.pop(hashX, None)
|
||||
|
||||
def address_to_hashX(self, address):
|
||||
try:
|
||||
@@ -1203,7 +1262,7 @@ class LBRYElectrumX(SessionBase):
|
||||
async def address_get_mempool(self, address):
|
||||
"""Return the mempool transactions touching an address."""
|
||||
hashX = self.address_to_hashX(address)
|
||||
return await self.unconfirmed_history(hashX)
|
||||
return self.unconfirmed_history(hashX)
|
||||
|
||||
async def address_listunspent(self, address):
|
||||
"""Return the list of UTXOs of an address."""
|
||||
@@ -1219,7 +1278,7 @@ class LBRYElectrumX(SessionBase):
|
||||
hashXes = [
|
||||
(self.address_to_hashX(address), address) for address in addresses
|
||||
]
|
||||
return await asyncio.gather(*(self.hashX_subscribe(*args) for args in hashXes))
|
||||
return [await self.hashX_subscribe(*args) for args in hashXes]
|
||||
|
||||
async def address_unsubscribe(self, address):
|
||||
"""Unsubscribe an address.
|
||||
@@ -1239,20 +1298,20 @@ class LBRYElectrumX(SessionBase):
|
||||
hashX = scripthash_to_hashX(scripthash)
|
||||
return await self.get_balance(hashX)
|
||||
|
||||
async def unconfirmed_history(self, hashX):
|
||||
def unconfirmed_history(self, hashX):
|
||||
# Note unconfirmed history is unordered in electrum-server
|
||||
# height is -1 if it has unconfirmed inputs, otherwise 0
|
||||
return [{'tx_hash': hash_to_hex_str(tx.hash),
|
||||
'height': -tx.has_unconfirmed_inputs,
|
||||
'fee': tx.fee}
|
||||
for tx in await self.mempool.transaction_summaries(hashX)]
|
||||
for tx in self.mempool.transaction_summaries(hashX)]
|
||||
|
||||
async def confirmed_and_unconfirmed_history(self, hashX):
|
||||
# Note history is ordered but unconfirmed is unordered in e-s
|
||||
history = await self.session_mgr.limited_history(hashX)
|
||||
conf = [{'tx_hash': hash_to_hex_str(tx_hash), 'height': height}
|
||||
for tx_hash, height in history]
|
||||
return conf + await self.unconfirmed_history(hashX)
|
||||
return conf + self.unconfirmed_history(hashX)
|
||||
|
||||
async def scripthash_get_history(self, scripthash):
|
||||
"""Return the confirmed and unconfirmed history of a scripthash."""
|
||||
@@ -1262,7 +1321,7 @@ class LBRYElectrumX(SessionBase):
|
||||
async def scripthash_get_mempool(self, scripthash):
|
||||
"""Return the mempool transactions touching a scripthash."""
|
||||
hashX = scripthash_to_hashX(scripthash)
|
||||
return await self.unconfirmed_history(hashX)
|
||||
return self.unconfirmed_history(hashX)
|
||||
|
||||
async def scripthash_listunspent(self, scripthash):
|
||||
"""Return the list of UTXOs of a scripthash."""
|
||||
@@ -1283,31 +1342,12 @@ class LBRYElectrumX(SessionBase):
|
||||
f'require header height {height:,d} <= '
|
||||
f'cp_height {cp_height:,d} <= '
|
||||
f'chain height {max_height:,d}')
|
||||
branch, root = await self.db.header_branch_and_root(cp_height + 1,
|
||||
height)
|
||||
branch, root = await self.db.header_branch_and_root(cp_height + 1, height)
|
||||
return {
|
||||
'branch': [hash_to_hex_str(elt) for elt in branch],
|
||||
'root': hash_to_hex_str(root),
|
||||
}
|
||||
|
||||
async def block_header(self, height, cp_height=0):
|
||||
"""Return a raw block header as a hexadecimal string, or as a
|
||||
dictionary with a merkle proof."""
|
||||
height = non_negative_integer(height)
|
||||
cp_height = non_negative_integer(cp_height)
|
||||
raw_header_hex = (await self.session_mgr.raw_header(height)).hex()
|
||||
if cp_height == 0:
|
||||
return raw_header_hex
|
||||
result = {'header': raw_header_hex}
|
||||
result.update(await self._merkle_proof(cp_height, height))
|
||||
return result
|
||||
|
||||
async def block_header_13(self, height):
|
||||
"""Return a raw block header as a hexadecimal string.
|
||||
|
||||
height: the header's height"""
|
||||
return await self.block_header(height)
|
||||
|
||||
async def block_headers(self, start_height, count, cp_height=0, b64=False):
|
||||
"""Return count concatenated block headers as hex for the main chain;
|
||||
starting at start_height.
|
||||
@@ -1321,9 +1361,13 @@ class LBRYElectrumX(SessionBase):
|
||||
|
||||
max_size = self.MAX_CHUNK_SIZE
|
||||
count = min(count, max_size)
|
||||
headers, count = await self.db.read_headers(start_height, count)
|
||||
compressobj = zlib.compressobj(wbits=-15, level=1, memLevel=9)
|
||||
headers = base64.b64encode(compressobj.compress(headers) + compressobj.flush()).decode() if b64 else headers.hex()
|
||||
headers, count = self.db.read_headers(start_height, count)
|
||||
|
||||
if b64:
|
||||
compressobj = zlib.compressobj(wbits=-15, level=1, memLevel=9)
|
||||
headers = base64.b64encode(compressobj.compress(headers) + compressobj.flush()).decode()
|
||||
else:
|
||||
headers = headers.hex()
|
||||
result = {
|
||||
'base64' if b64 else 'hex': headers,
|
||||
'count': count,
|
||||
@@ -1341,7 +1385,7 @@ class LBRYElectrumX(SessionBase):
|
||||
index = non_negative_integer(index)
|
||||
size = self.coin.CHUNK_SIZE
|
||||
start_height = index * size
|
||||
headers, _ = await self.db.read_headers(start_height, size)
|
||||
headers, _ = self.db.read_headers(start_height, size)
|
||||
return headers.hex()
|
||||
|
||||
async def block_get_header(self, height):
|
||||
@@ -1430,7 +1474,8 @@ class LBRYElectrumX(SessionBase):
|
||||
client_name: a string identifying the client
|
||||
protocol_version: the protocol version spoken by the client
|
||||
"""
|
||||
|
||||
if self.protocol_string is not None:
|
||||
return self.version, self.protocol_string
|
||||
if self.sv_seen and self.protocol_tuple >= (1, 4):
|
||||
raise RPCError(BAD_REQUEST, f'server.version already sent')
|
||||
self.sv_seen = True
|
||||
@@ -1440,29 +1485,25 @@ class LBRYElectrumX(SessionBase):
|
||||
if self.env.drop_client is not None and \
|
||||
self.env.drop_client.match(client_name):
|
||||
self.close_after_send = True
|
||||
raise RPCError(BAD_REQUEST,
|
||||
f'unsupported client: {client_name}')
|
||||
raise RPCError(BAD_REQUEST, f'unsupported client: {client_name}')
|
||||
if self.client_version != client_name[:17]:
|
||||
SESSIONS_COUNT.labels(version=self.client_version).dec()
|
||||
self.session_mgr.session_count_metric.labels(version=self.client_version).dec()
|
||||
self.client_version = client_name[:17]
|
||||
SESSIONS_COUNT.labels(version=self.client_version).inc()
|
||||
CLIENT_VERSIONS.labels(version=self.client_version).inc()
|
||||
self.session_mgr.session_count_metric.labels(version=self.client_version).inc()
|
||||
self.session_mgr.client_version_metric.labels(version=self.client_version).inc()
|
||||
|
||||
# Find the highest common protocol version. Disconnect if
|
||||
# that protocol version in unsupported.
|
||||
ptuple, client_min = util.protocol_version(
|
||||
protocol_version, self.PROTOCOL_MIN, self.PROTOCOL_MAX)
|
||||
ptuple, client_min = util.protocol_version(protocol_version, self.PROTOCOL_MIN, self.PROTOCOL_MAX)
|
||||
if ptuple is None:
|
||||
# FIXME: this fills the logs
|
||||
# if client_min > self.PROTOCOL_MIN:
|
||||
# self.logger.info(f'client requested future protocol version '
|
||||
# f'{util.version_string(client_min)} '
|
||||
# f'- is your software out of date?')
|
||||
self.close_after_send = True
|
||||
raise RPCError(BAD_REQUEST,
|
||||
f'unsupported protocol version: {protocol_version}')
|
||||
ptuple, client_min = util.protocol_version(protocol_version, (1, 1, 0), (1, 4, 0))
|
||||
if ptuple is None:
|
||||
self.close_after_send = True
|
||||
raise RPCError(BAD_REQUEST, f'unsupported protocol version: {protocol_version}')
|
||||
|
||||
self.protocol_tuple = ptuple
|
||||
return self.version, self.protocol_version_string()
|
||||
self.protocol_string = util.version_string(ptuple)
|
||||
return self.version, self.protocol_string
|
||||
|
||||
async def transaction_broadcast(self, raw_tx):
|
||||
"""Broadcast a raw transaction to the network.
|
||||
@@ -1472,6 +1513,7 @@ class LBRYElectrumX(SessionBase):
|
||||
try:
|
||||
hex_hash = await self.session_mgr.broadcast_transaction(raw_tx)
|
||||
self.txs_sent += 1
|
||||
self.mempool.wakeup.set()
|
||||
self.logger.info(f'sent tx: {hex_hash}')
|
||||
return hex_hash
|
||||
except DaemonError as e:
|
||||
@@ -1482,51 +1524,46 @@ class LBRYElectrumX(SessionBase):
|
||||
f'network rules.\n\n{message}\n[{raw_tx}]')
|
||||
|
||||
async def transaction_info(self, tx_hash: str):
|
||||
assert_tx_hash(tx_hash)
|
||||
tx_info = await self.daemon_request('getrawtransaction', tx_hash, True)
|
||||
raw_tx = tx_info['hex']
|
||||
block_hash = tx_info.get('blockhash')
|
||||
if not block_hash:
|
||||
return raw_tx, {'block_height': -1}
|
||||
merkle_height = (await self.daemon_request('deserialised_block', block_hash))['height']
|
||||
merkle = await self.transaction_merkle(tx_hash, merkle_height)
|
||||
return raw_tx, merkle
|
||||
return (await self.transaction_get_batch(tx_hash))[tx_hash]
|
||||
|
||||
async def transaction_get_batch(self, *tx_hashes):
|
||||
self.session_mgr.tx_request_count_metric.inc(len(tx_hashes))
|
||||
if len(tx_hashes) > 100:
|
||||
raise RPCError(BAD_REQUEST, f'too many tx hashes in request: {len(tx_hashes)}')
|
||||
for tx_hash in tx_hashes:
|
||||
assert_tx_hash(tx_hash)
|
||||
batch_result = {}
|
||||
height = None
|
||||
block_hash = None
|
||||
block = None
|
||||
batch_result = await self.db.fs_transactions(tx_hashes)
|
||||
needed_merkles = {}
|
||||
|
||||
for tx_hash in tx_hashes:
|
||||
if tx_hash in batch_result and batch_result[tx_hash][0]:
|
||||
continue
|
||||
tx_info = await self.daemon_request('getrawtransaction', tx_hash, True)
|
||||
raw_tx = tx_info['hex']
|
||||
if height is None:
|
||||
if 'blockhash' in tx_info:
|
||||
block_hash = tx_info['blockhash']
|
||||
block = await self.daemon_request('deserialised_block', block_hash)
|
||||
height = block['height']
|
||||
else:
|
||||
height = -1
|
||||
if block_hash != tx_info.get('blockhash'):
|
||||
raise RPCError(BAD_REQUEST, f'request contains a mix of transaction heights')
|
||||
block_hash = tx_info.get('blockhash')
|
||||
if block_hash:
|
||||
block = await self.daemon.deserialised_block(block_hash)
|
||||
height = block['height']
|
||||
try:
|
||||
pos = block['tx'].index(tx_hash)
|
||||
except ValueError:
|
||||
raise RPCError(BAD_REQUEST, f'tx hash {tx_hash} not in '
|
||||
f'block {block_hash} at height {height:,d}')
|
||||
needed_merkles[tx_hash] = raw_tx, block['tx'], pos, height
|
||||
else:
|
||||
if not block_hash:
|
||||
merkle = {'block_height': -1}
|
||||
else:
|
||||
try:
|
||||
pos = block['tx'].index(tx_hash)
|
||||
except ValueError:
|
||||
raise RPCError(BAD_REQUEST, f'tx hash {tx_hash} not in '
|
||||
f'block {block_hash} at height {height:,d}')
|
||||
merkle = {
|
||||
"merkle": self._get_merkle_branch(block['tx'], pos),
|
||||
"pos": pos
|
||||
}
|
||||
batch_result[tx_hash] = [raw_tx, merkle]
|
||||
batch_result[tx_hash] = [raw_tx, {'block_height': -1}]
|
||||
|
||||
def threaded_get_merkle():
|
||||
for tx_hash, (raw_tx, block_txs, pos, block_height) in needed_merkles.items():
|
||||
batch_result[tx_hash] = raw_tx, {
|
||||
'merkle': self._get_merkle_branch(block_txs, pos),
|
||||
'pos': pos,
|
||||
'block_height': block_height
|
||||
}
|
||||
if needed_merkles:
|
||||
await asyncio.get_running_loop().run_in_executor(self.db.executor, threaded_get_merkle)
|
||||
|
||||
self.session_mgr.tx_replied_count_metric.inc(len(tx_hashes))
|
||||
return batch_result
|
||||
|
||||
async def transaction_get(self, tx_hash, verbose=False):
|
||||
@@ -1541,19 +1578,6 @@ class LBRYElectrumX(SessionBase):
|
||||
|
||||
return await self.daemon_request('getrawtransaction', tx_hash, verbose)
|
||||
|
||||
async def _block_hash_and_tx_hashes(self, height):
|
||||
"""Returns a pair (block_hash, tx_hashes) for the main chain block at
|
||||
the given height.
|
||||
|
||||
block_hash is a hexadecimal string, and tx_hashes is an
|
||||
ordered list of hexadecimal strings.
|
||||
"""
|
||||
height = non_negative_integer(height)
|
||||
hex_hashes = await self.daemon_request('block_hex_hashes', height, 1)
|
||||
block_hash = hex_hashes[0]
|
||||
block = await self.daemon_request('deserialised_block', block_hash)
|
||||
return block_hash, block['tx']
|
||||
|
||||
def _get_merkle_branch(self, tx_hashes, tx_pos):
|
||||
"""Return a merkle branch to a transaction.
|
||||
|
||||
@@ -1573,35 +1597,11 @@ class LBRYElectrumX(SessionBase):
|
||||
height: the height of the block it is in
|
||||
"""
|
||||
assert_tx_hash(tx_hash)
|
||||
block_hash, tx_hashes = await self._block_hash_and_tx_hashes(height)
|
||||
try:
|
||||
pos = tx_hashes.index(tx_hash)
|
||||
except ValueError:
|
||||
result = await self.transaction_get_batch(tx_hash)
|
||||
if tx_hash not in result or result[tx_hash][1]['block_height'] <= 0:
|
||||
raise RPCError(BAD_REQUEST, f'tx hash {tx_hash} not in '
|
||||
f'block {block_hash} at height {height:,d}')
|
||||
branch = self._get_merkle_branch(tx_hashes, pos)
|
||||
return {"block_height": height, "merkle": branch, "pos": pos}
|
||||
|
||||
async def transaction_id_from_pos(self, height, tx_pos, merkle=False):
|
||||
"""Return the txid and optionally a merkle proof, given
|
||||
a block height and position in the block.
|
||||
"""
|
||||
tx_pos = non_negative_integer(tx_pos)
|
||||
if merkle not in (True, False):
|
||||
raise RPCError(BAD_REQUEST, f'"merkle" must be a boolean')
|
||||
|
||||
block_hash, tx_hashes = await self._block_hash_and_tx_hashes(height)
|
||||
try:
|
||||
tx_hash = tx_hashes[tx_pos]
|
||||
except IndexError:
|
||||
raise RPCError(BAD_REQUEST, f'no tx at position {tx_pos:,d} in '
|
||||
f'block {block_hash} at height {height:,d}')
|
||||
|
||||
if merkle:
|
||||
branch = self._get_merkle_branch(tx_hashes, tx_pos)
|
||||
return {"tx_hash": tx_hash, "merkle": branch}
|
||||
else:
|
||||
return tx_hash
|
||||
f'block at height {height:,d}')
|
||||
return result[tx_hash][1]
|
||||
|
||||
|
||||
class LocalRPC(SessionBase):
|
||||
|
||||
@@ -77,18 +77,16 @@ class LevelDB(Storage):
|
||||
import plyvel
|
||||
cls.module = plyvel
|
||||
|
||||
def open(self, name, create):
|
||||
mof = 512 if self.for_sync else 128
|
||||
def open(self, name, create, lru_cache_size=None):
|
||||
mof = 10000
|
||||
path = os.path.join(self.db_dir, name)
|
||||
# Use snappy compression (the default)
|
||||
self.db = self.module.DB(path, create_if_missing=create,
|
||||
max_open_files=mof)
|
||||
self.db = self.module.DB(path, create_if_missing=create, max_open_files=mof)
|
||||
self.close = self.db.close
|
||||
self.get = self.db.get
|
||||
self.put = self.db.put
|
||||
self.iterator = self.db.iterator
|
||||
self.write_batch = partial(self.db.write_batch, transaction=True,
|
||||
sync=True)
|
||||
self.write_batch = partial(self.db.write_batch, transaction=True, sync=True)
|
||||
|
||||
|
||||
class RocksDB(Storage):
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
import asyncio
|
||||
import struct
|
||||
from time import perf_counter
|
||||
import logging
|
||||
from typing import Optional, Tuple, NamedTuple
|
||||
from lbry.utils import LRUCache
|
||||
# from prometheus_client import Counter
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
_MAGIC = 1446058291 # genesis blocktime (which is actually wrong)
|
||||
# ping_count_metric = Counter("ping_count", "Number of pings received", namespace='wallet_server_status')
|
||||
_PAD_BYTES = b'\x00' * 64
|
||||
|
||||
|
||||
class SPVPing(NamedTuple):
|
||||
magic: int
|
||||
protocol_version: int
|
||||
pad_bytes: bytes
|
||||
|
||||
def encode(self):
|
||||
return struct.pack(b'!lB64s', *self)
|
||||
|
||||
@staticmethod
|
||||
def make(protocol_version=1) -> bytes:
|
||||
return SPVPing(_MAGIC, protocol_version, _PAD_BYTES).encode()
|
||||
|
||||
@classmethod
|
||||
def decode(cls, packet: bytes):
|
||||
decoded = cls(*struct.unpack(b'!lB64s', packet[:69]))
|
||||
if decoded.magic != _MAGIC:
|
||||
raise ValueError("invalid magic bytes")
|
||||
return decoded
|
||||
|
||||
|
||||
class SPVPong(NamedTuple):
|
||||
protocol_version: int
|
||||
flags: int
|
||||
height: int
|
||||
tip: bytes
|
||||
source_address_raw: bytes
|
||||
|
||||
def encode(self):
|
||||
return struct.pack(b'!BBl32s4s', *self)
|
||||
|
||||
@staticmethod
|
||||
def make(height: int, tip: bytes, flags: int, protocol_version: int = 1) -> bytes:
|
||||
# note: drops the last 4 bytes so the result can be cached and have addresses added to it as needed
|
||||
return SPVPong(protocol_version, flags, height, tip, b'\x00\x00\x00\x00').encode()[:38]
|
||||
|
||||
@classmethod
|
||||
def decode(cls, packet: bytes):
|
||||
return cls(*struct.unpack(b'!BBl32s4s', packet[:42]))
|
||||
|
||||
@property
|
||||
def available(self) -> bool:
|
||||
return (self.flags & 0b00000001) > 0
|
||||
|
||||
@property
|
||||
def ip_address(self) -> str:
|
||||
return ".".join(map(str, self.source_address_raw))
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"SPVPong(external_ip={self.ip_address}, version={self.protocol_version}, " \
|
||||
f"available={'True' if self.flags & 1 > 0 else 'False'}," \
|
||||
f" height={self.height}, tip={self.tip[::-1].hex()})"
|
||||
|
||||
|
||||
class SPVServerStatusProtocol(asyncio.DatagramProtocol):
|
||||
PROTOCOL_VERSION = 1
|
||||
|
||||
def __init__(self, height: int, tip: bytes, throttle_cache_size: int = 1024, throttle_reqs_per_sec: int = 10):
|
||||
super().__init__()
|
||||
self.transport: Optional[asyncio.transports.DatagramTransport] = None
|
||||
self._height = height
|
||||
self._tip = tip
|
||||
self._flags = 0
|
||||
self._cached_response = None
|
||||
self.update_cached_response()
|
||||
self._throttle = LRUCache(throttle_cache_size)
|
||||
self._should_log = LRUCache(throttle_cache_size)
|
||||
self._min_delay = 1 / throttle_reqs_per_sec
|
||||
|
||||
def update_cached_response(self):
|
||||
self._cached_response = SPVPong.make(self._height, self._tip, self._flags, self.PROTOCOL_VERSION)
|
||||
|
||||
def set_unavailable(self):
|
||||
self._flags &= 0b11111110
|
||||
self.update_cached_response()
|
||||
|
||||
def set_available(self):
|
||||
self._flags |= 0b00000001
|
||||
self.update_cached_response()
|
||||
|
||||
def set_height(self, height: int, tip: bytes):
|
||||
self._height, self._tip = height, tip
|
||||
self.update_cached_response()
|
||||
|
||||
def should_throttle(self, host: str):
|
||||
now = perf_counter()
|
||||
last_requested = self._throttle.get(host, default=0)
|
||||
self._throttle[host] = now
|
||||
if now - last_requested < self._min_delay:
|
||||
log_cnt = self._should_log.get(host, default=0) + 1
|
||||
if log_cnt % 100 == 0:
|
||||
log.warning("throttle spv status to %s", host)
|
||||
self._should_log[host] = log_cnt
|
||||
return True
|
||||
return False
|
||||
|
||||
def make_pong(self, host):
|
||||
return self._cached_response + bytes(int(b) for b in host.split("."))
|
||||
|
||||
def datagram_received(self, data: bytes, addr: Tuple[str, int]):
|
||||
if self.should_throttle(addr[0]):
|
||||
return
|
||||
try:
|
||||
SPVPing.decode(data)
|
||||
except (ValueError, struct.error, AttributeError, TypeError):
|
||||
# log.exception("derp")
|
||||
return
|
||||
self.transport.sendto(self.make_pong(addr[0]), addr)
|
||||
# ping_count_metric.inc()
|
||||
|
||||
def connection_made(self, transport) -> None:
|
||||
self.transport = transport
|
||||
|
||||
def connection_lost(self, exc: Optional[Exception]) -> None:
|
||||
self.transport = None
|
||||
|
||||
def close(self):
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
|
||||
|
||||
class StatusServer:
|
||||
def __init__(self):
|
||||
self._protocol: Optional[SPVServerStatusProtocol] = None
|
||||
|
||||
async def start(self, height: int, tip: bytes, interface: str, port: int):
|
||||
if self._protocol:
|
||||
return
|
||||
loop = asyncio.get_event_loop()
|
||||
self._protocol = SPVServerStatusProtocol(height, tip)
|
||||
interface = interface if interface.lower() != 'localhost' else '127.0.0.1'
|
||||
await loop.create_datagram_endpoint(lambda: self._protocol, (interface, port))
|
||||
log.info("started udp status server on %s:%i", interface, port)
|
||||
|
||||
def stop(self):
|
||||
if self._protocol:
|
||||
self._protocol.close()
|
||||
self._protocol = None
|
||||
|
||||
def set_unavailable(self):
|
||||
self._protocol.set_unavailable()
|
||||
|
||||
def set_available(self):
|
||||
self._protocol.set_available()
|
||||
|
||||
def set_height(self, height: int, tip: bytes):
|
||||
self._protocol.set_height(height, tip)
|
||||
|
||||
|
||||
class SPVStatusClientProtocol(asyncio.DatagramProtocol):
|
||||
PROTOCOL_VERSION = 1
|
||||
|
||||
def __init__(self, responses: asyncio.Queue):
|
||||
super().__init__()
|
||||
self.transport: Optional[asyncio.transports.DatagramTransport] = None
|
||||
self.responses = responses
|
||||
self._ping_packet = SPVPing.make(self.PROTOCOL_VERSION)
|
||||
|
||||
def datagram_received(self, data: bytes, addr: Tuple[str, int]):
|
||||
try:
|
||||
self.responses.put_nowait(((addr, perf_counter()), SPVPong.decode(data)))
|
||||
except (ValueError, struct.error, AttributeError, TypeError, RuntimeError):
|
||||
return
|
||||
|
||||
def connection_made(self, transport) -> None:
|
||||
self.transport = transport
|
||||
|
||||
def connection_lost(self, exc: Optional[Exception]) -> None:
|
||||
self.transport = None
|
||||
log.info("closed udp spv server selection client")
|
||||
|
||||
def ping(self, server: Tuple[str, int]):
|
||||
self.transport.sendto(self._ping_packet, server)
|
||||
|
||||
def close(self):
|
||||
# log.info("close udp client")
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
@@ -333,11 +333,13 @@ unpack_le_uint64_from = struct_le_Q.unpack_from
|
||||
unpack_be_uint16_from = struct_be_H.unpack_from
|
||||
unpack_be_uint32_from = struct_be_I.unpack_from
|
||||
|
||||
unpack_be_uint64 = lambda x: int.from_bytes(x, byteorder='big')
|
||||
|
||||
pack_le_int32 = struct_le_i.pack
|
||||
pack_le_int64 = struct_le_q.pack
|
||||
pack_le_uint16 = struct_le_H.pack
|
||||
pack_le_uint32 = struct_le_I.pack
|
||||
pack_le_uint64 = struct_le_Q.pack
|
||||
pack_be_uint64 = lambda x: x.to_bytes(8, byteorder='big')
|
||||
pack_be_uint16 = struct_be_H.pack
|
||||
pack_be_uint32 = struct_be_I.pack
|
||||
pack_byte = structB.pack
|
||||
|
||||
+82
-26
@@ -2,6 +2,7 @@ import struct
|
||||
import hashlib
|
||||
import logging
|
||||
import typing
|
||||
import asyncio
|
||||
from binascii import hexlify, unhexlify
|
||||
from typing import List, Iterable, Optional, Tuple
|
||||
|
||||
@@ -18,7 +19,9 @@ from lbry.crypto.hash import hash160, sha256
|
||||
from lbry.crypto.base58 import Base58
|
||||
from lbry.schema.url import normalize_name
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.schema.base import Signable
|
||||
from lbry.schema.purchase import Purchase
|
||||
from lbry.schema.support import Support
|
||||
|
||||
from .script import InputScript, OutputScript
|
||||
from .constants import COIN, NULL_HASH32
|
||||
@@ -210,7 +213,7 @@ class Output(InputOutput):
|
||||
'amount', 'script', 'is_internal_transfer', 'is_spent', 'is_my_output', 'is_my_input',
|
||||
'channel', 'private_key', 'meta', 'sent_supports', 'sent_tips', 'received_tips',
|
||||
'purchase', 'purchased_claim', 'purchase_receipt',
|
||||
'reposted_claim', 'claims',
|
||||
'reposted_claim', 'claims', '_signable'
|
||||
)
|
||||
|
||||
def __init__(self, amount: int, script: OutputScript,
|
||||
@@ -238,6 +241,7 @@ class Output(InputOutput):
|
||||
self.purchase_receipt: 'Output' = None # txo representing purchase receipt for this claim
|
||||
self.reposted_claim: 'Output' = None # txo representing claim being reposted
|
||||
self.claims: List['Output'] = None # resolved claims for collection
|
||||
self._signable: Optional[Signable] = None
|
||||
self.meta = {}
|
||||
|
||||
def update_annotations(self, annotated: 'Output'):
|
||||
@@ -311,6 +315,10 @@ class Output(InputOutput):
|
||||
def is_support(self) -> bool:
|
||||
return self.script.is_support_claim
|
||||
|
||||
@property
|
||||
def is_support_data(self) -> bool:
|
||||
return self.script.is_support_claim_data
|
||||
|
||||
@property
|
||||
def claim_hash(self) -> bytes:
|
||||
if self.script.is_claim_name:
|
||||
@@ -346,9 +354,33 @@ class Output(InputOutput):
|
||||
def can_decode_claim(self):
|
||||
try:
|
||||
return self.claim
|
||||
except: # pylint: disable=bare-except
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
@property
|
||||
def support(self) -> Support:
|
||||
if self.is_support_data:
|
||||
if not isinstance(self.script.values['support'], Support):
|
||||
self.script.values['support'] = Support.from_bytes(self.script.values['support'])
|
||||
return self.script.values['support']
|
||||
raise ValueError('Only supports with data can be represented as Supports.')
|
||||
|
||||
@property
|
||||
def can_decode_support(self):
|
||||
try:
|
||||
return self.support
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
@property
|
||||
def signable(self) -> Signable:
|
||||
if self._signable is None:
|
||||
if self.is_claim:
|
||||
self._signable = self.claim
|
||||
elif self.is_support_data:
|
||||
self._signable = self.support
|
||||
return self._signable
|
||||
|
||||
@property
|
||||
def permanent_url(self) -> str:
|
||||
if self.script.is_claim_involved:
|
||||
@@ -360,22 +392,22 @@ class Output(InputOutput):
|
||||
return self.private_key is not None
|
||||
|
||||
def get_signature_digest(self, ledger):
|
||||
if self.claim.unsigned_payload:
|
||||
if self.signable.unsigned_payload:
|
||||
pieces = [
|
||||
Base58.decode(self.get_address(ledger)),
|
||||
self.claim.unsigned_payload,
|
||||
self.claim.signing_channel_hash[::-1]
|
||||
self.signable.unsigned_payload,
|
||||
self.signable.signing_channel_hash[::-1]
|
||||
]
|
||||
else:
|
||||
pieces = [
|
||||
self.tx_ref.tx.inputs[0].txo_ref.hash,
|
||||
self.claim.signing_channel_hash,
|
||||
self.claim.to_message_bytes()
|
||||
self.signable.signing_channel_hash,
|
||||
self.signable.to_message_bytes()
|
||||
]
|
||||
return sha256(b''.join(pieces))
|
||||
|
||||
def get_encoded_signature(self):
|
||||
signature = hexlify(self.claim.signature)
|
||||
signature = hexlify(self.signable.signature)
|
||||
r = int(signature[:int(len(signature)/2)], 16)
|
||||
s = int(signature[int(len(signature)/2):], 16)
|
||||
return ecdsa.util.sigencode_der(r, s, len(signature)*4)
|
||||
@@ -399,21 +431,23 @@ class Output(InputOutput):
|
||||
|
||||
def sign(self, channel: 'Output', first_input_id=None):
|
||||
self.channel = channel
|
||||
self.claim.signing_channel_hash = channel.claim_hash
|
||||
self.signable.signing_channel_hash = channel.claim_hash
|
||||
digest = sha256(b''.join([
|
||||
first_input_id or self.tx_ref.tx.inputs[0].txo_ref.hash,
|
||||
self.claim.signing_channel_hash,
|
||||
self.claim.to_message_bytes()
|
||||
self.signable.signing_channel_hash,
|
||||
self.signable.to_message_bytes()
|
||||
]))
|
||||
self.claim.signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
|
||||
self.signable.signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
|
||||
self.script.generate()
|
||||
|
||||
def clear_signature(self):
|
||||
self.channel = None
|
||||
self.claim.clear_signature()
|
||||
self.signable.clear_signature()
|
||||
|
||||
def generate_channel_private_key(self):
|
||||
self.private_key = ecdsa.SigningKey.generate(curve=ecdsa.SECP256k1, hashfunc=hashlib.sha256)
|
||||
async def generate_channel_private_key(self):
|
||||
self.private_key = await asyncio.get_event_loop().run_in_executor(
|
||||
None, ecdsa.SigningKey.generate, ecdsa.SECP256k1, None, hashlib.sha256
|
||||
)
|
||||
self.claim.channel.public_key_bytes = self.private_key.get_verifying_key().to_der()
|
||||
self.script.generate()
|
||||
return self.private_key
|
||||
@@ -443,6 +477,14 @@ class Output(InputOutput):
|
||||
)
|
||||
return cls(amount, script)
|
||||
|
||||
@classmethod
|
||||
def pay_support_data_pubkey_hash(
|
||||
cls, amount: int, claim_name: str, claim_id: str, support: Support, pubkey_hash: bytes) -> 'Output':
|
||||
script = OutputScript.pay_support_data_pubkey_hash(
|
||||
claim_name.encode(), unhexlify(claim_id)[::-1], support, pubkey_hash
|
||||
)
|
||||
return cls(amount, script)
|
||||
|
||||
@classmethod
|
||||
def add_purchase_data(cls, purchase: Purchase) -> 'Output':
|
||||
script = OutputScript.return_data(purchase)
|
||||
@@ -497,6 +539,7 @@ class Transaction:
|
||||
height: int = -2, position: int = -1, julian_day: int = None) -> None:
|
||||
self._raw = raw
|
||||
self._raw_sans_segwit = None
|
||||
self._raw_outputs = None
|
||||
self.is_segwit_flag = 0
|
||||
self.witnesses: List[bytes] = []
|
||||
self.ref = TXRefMutable(self)
|
||||
@@ -558,6 +601,7 @@ class Transaction:
|
||||
def _reset(self):
|
||||
self._raw = None
|
||||
self._raw_sans_segwit = None
|
||||
self._raw_outputs = None
|
||||
self.ref.reset()
|
||||
|
||||
@property
|
||||
@@ -651,9 +695,7 @@ class Transaction:
|
||||
stream.write_compact_size(len(self._inputs))
|
||||
for txin in self._inputs:
|
||||
txin.serialize_to(stream)
|
||||
stream.write_compact_size(len(self._outputs))
|
||||
for txout in self._outputs:
|
||||
txout.serialize_to(stream)
|
||||
self._serialize_outputs(stream)
|
||||
stream.write_uint32(self.locktime)
|
||||
return stream.get_bytes()
|
||||
|
||||
@@ -667,13 +709,19 @@ class Transaction:
|
||||
txin.serialize_to(stream, txin.txo_ref.txo.script.source)
|
||||
else:
|
||||
txin.serialize_to(stream, b'')
|
||||
stream.write_compact_size(len(self._outputs))
|
||||
for txout in self._outputs:
|
||||
txout.serialize_to(stream)
|
||||
self._serialize_outputs(stream)
|
||||
stream.write_uint32(self.locktime)
|
||||
stream.write_uint32(self.signature_hash_type(1)) # signature hash type: SIGHASH_ALL
|
||||
return stream.get_bytes()
|
||||
|
||||
def _serialize_outputs(self, stream):
|
||||
if self._raw_outputs is None:
|
||||
self._raw_outputs = BCDataStream()
|
||||
self._raw_outputs.write_compact_size(len(self._outputs))
|
||||
for txout in self._outputs:
|
||||
txout.serialize_to(self._raw_outputs)
|
||||
stream.write(self._raw_outputs.get_bytes())
|
||||
|
||||
def _deserialize(self):
|
||||
if self._raw is not None:
|
||||
stream = BCDataStream(self._raw)
|
||||
@@ -857,12 +905,20 @@ class Transaction:
|
||||
|
||||
@classmethod
|
||||
def support(cls, claim_name: str, claim_id: str, amount: int, holding_address: str,
|
||||
funding_accounts: List['Account'], change_account: 'Account'):
|
||||
funding_accounts: List['Account'], change_account: 'Account', signing_channel: Output = None):
|
||||
ledger, _ = cls.ensure_all_have_same_ledger_and_wallet(funding_accounts, change_account)
|
||||
support_output = Output.pay_support_pubkey_hash(
|
||||
amount, claim_name, claim_id, ledger.address_to_hash160(holding_address)
|
||||
)
|
||||
return cls.create([], [support_output], funding_accounts, change_account)
|
||||
if signing_channel is not None:
|
||||
support = Support()
|
||||
support.emoji = '👍'
|
||||
support_output = Output.pay_support_data_pubkey_hash(
|
||||
amount, claim_name, claim_id, support, ledger.address_to_hash160(holding_address)
|
||||
)
|
||||
support_output.sign(signing_channel, b'placeholder txid:nout')
|
||||
else:
|
||||
support_output = Output.pay_support_pubkey_hash(
|
||||
amount, claim_name, claim_id, ledger.address_to_hash160(holding_address)
|
||||
)
|
||||
return cls.create([], [support_output], funding_accounts, change_account, sign=False)
|
||||
|
||||
@classmethod
|
||||
def purchase(cls, claim_id: str, amount: int, merchant_address: bytes,
|
||||
|
||||
@@ -35,7 +35,7 @@ class TimestampedPreferences(UserDict):
|
||||
def __setitem__(self, key, value):
|
||||
self.data[key] = {
|
||||
'value': value,
|
||||
'ts': time.time()
|
||||
'ts': int(time.time())
|
||||
}
|
||||
|
||||
def __repr__(self):
|
||||
@@ -145,14 +145,15 @@ class Wallet:
|
||||
elif not self.is_locked:
|
||||
log.warning(
|
||||
"Disk encryption requested but no password available for encryption. "
|
||||
"Saving wallet in an unencrypted state."
|
||||
"Resetting encryption preferences and saving wallet in an unencrypted state."
|
||||
)
|
||||
self.preferences[ENCRYPT_ON_DISK] = False
|
||||
return self.storage.write(self.to_dict())
|
||||
|
||||
@property
|
||||
def hash(self) -> bytes:
|
||||
h = sha256()
|
||||
if self.preferences.get(ENCRYPT_ON_DISK, False):
|
||||
if self.is_encrypted:
|
||||
assert self.encryption_password is not None, \
|
||||
"Encryption is enabled but no password is available, cannot generate hash."
|
||||
h.update(self.encryption_password.encode())
|
||||
@@ -219,7 +220,11 @@ class Wallet:
|
||||
|
||||
@property
|
||||
def is_encrypted(self) -> bool:
|
||||
return self.is_locked or self.preferences.get(ENCRYPT_ON_DISK, False)
|
||||
# either its locked or it was unlocked using a password.
|
||||
# if its set to encrypt on preferences but isnt encrypted and no password was given so far,
|
||||
# then its not encrypted
|
||||
return self.is_locked or (
|
||||
self.preferences.get(ENCRYPT_ON_DISK, False) and self.encryption_password is not None)
|
||||
|
||||
def decrypt(self):
|
||||
assert not self.is_locked, "Cannot decrypt a locked wallet, unlock first."
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import curses
|
||||
import time
|
||||
import asyncio
|
||||
import lbry.wallet
|
||||
from lbry.conf import Config
|
||||
from lbry.extras.daemon.client import daemon_rpc
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ import os
|
||||
import asyncio
|
||||
import socket
|
||||
import ipaddress
|
||||
import lbry.wallet
|
||||
from lbry.conf import Config
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from lbry.blob.blob_manager import BlobManager
|
||||
|
||||
@@ -13,6 +13,7 @@ from lbry.extras.cli import set_kwargs, get_argument_parser
|
||||
from lbry.extras.daemon.daemon import (
|
||||
Daemon, jsonrpc_dumps_pretty, encode_pagination_doc
|
||||
)
|
||||
from tests.integration.other.test_comment_commands import MockedCommentServer
|
||||
from lbry.extras.daemon.json_response_encoder import (
|
||||
encode_tx_doc, encode_txo_doc, encode_account_doc, encode_file_doc,
|
||||
encode_wallet_doc
|
||||
@@ -25,7 +26,8 @@ RETURN_DOCS = {
|
||||
'File': encode_file_doc(),
|
||||
'Transaction': encode_tx_doc(),
|
||||
'Output': encode_txo_doc(),
|
||||
'Address': 'an address in base58'
|
||||
'Address': 'an address in base58',
|
||||
'Dict': 'glorious data in dictionary',
|
||||
}
|
||||
|
||||
|
||||
@@ -65,6 +67,10 @@ class Examples(CommandTestCase):
|
||||
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
self.daemon.conf.comment_server = 'http://localhost:2903/api'
|
||||
self.comment_server = MockedCommentServer(2903)
|
||||
await self.comment_server.start()
|
||||
self.addCleanup(self.comment_server.stop)
|
||||
self.recorder = ExampleRecorder(self)
|
||||
|
||||
async def play(self):
|
||||
@@ -336,7 +342,8 @@ class Examples(CommandTestCase):
|
||||
'Use the parent_id param to make replies',
|
||||
'comment', 'create',
|
||||
'--comment="I have photographic evidence confirming Sasquatch exists"',
|
||||
f'--channel_name=@channel', f'--parent_id={comment["comment_id"]}'
|
||||
f'--channel_name=@channel', f'--parent_id={comment["comment_id"]}',
|
||||
f'--claim_id={stream_id}'
|
||||
)
|
||||
|
||||
await r(
|
||||
|
||||
@@ -6,7 +6,7 @@ source =
|
||||
lbry
|
||||
.tox/*/lib/python*/site-packages/lbry
|
||||
|
||||
[cryptography.*,coincurve.*,pbkdf2]
|
||||
[cryptography.*,coincurve.*,pbkdf2, libtorrent]
|
||||
ignore_missing_imports = True
|
||||
|
||||
[pylint]
|
||||
@@ -18,6 +18,7 @@ max-line-length=120
|
||||
good-names=T,t,n,i,j,k,x,y,s,f,d,h,c,e,op,db,tx,io,cachedproperty,log,id,r,iv,ts,l
|
||||
valid-metaclass-classmethod-first-arg=mcs
|
||||
disable=
|
||||
c-extension-no-member,
|
||||
fixme,
|
||||
broad-except,
|
||||
no-else-return,
|
||||
@@ -34,6 +35,7 @@ disable=
|
||||
too-many-statements,
|
||||
too-many-nested-blocks,
|
||||
too-many-public-methods,
|
||||
too-many-return-statements,
|
||||
too-many-instance-attributes,
|
||||
protected-access,
|
||||
unused-argument
|
||||
@@ -34,7 +34,7 @@ setup(
|
||||
},
|
||||
install_requires=[
|
||||
'aiohttp==3.5.4',
|
||||
'aioupnp==0.0.17',
|
||||
'aioupnp==0.0.18',
|
||||
'appdirs==1.4.3',
|
||||
'certifi>=2018.11.29',
|
||||
'colorama==0.3.7',
|
||||
@@ -46,7 +46,7 @@ setup(
|
||||
'msgpack==0.6.1',
|
||||
'prometheus_client==0.7.1',
|
||||
'ecdsa==0.13.3',
|
||||
'pyyaml==4.2b1',
|
||||
'pyyaml==5.3.1',
|
||||
'docopt==0.6.2',
|
||||
'hachoir',
|
||||
'multidict==4.6.1',
|
||||
|
||||
@@ -2,7 +2,6 @@ import logging
|
||||
import asyncio
|
||||
from binascii import hexlify
|
||||
from lbry.testcase import CommandTestCase
|
||||
from lbry.wallet.server.prometheus import REORG_COUNT
|
||||
|
||||
|
||||
class BlockchainReorganizationTests(CommandTestCase):
|
||||
@@ -10,23 +9,39 @@ class BlockchainReorganizationTests(CommandTestCase):
|
||||
VERBOSITY = logging.WARN
|
||||
|
||||
async def assertBlockHash(self, height):
|
||||
self.assertEqual(
|
||||
(await self.ledger.headers.hash(height)).decode(),
|
||||
await self.blockchain.get_block_hash(height)
|
||||
)
|
||||
bp = self.conductor.spv_node.server.bp
|
||||
|
||||
def get_txids():
|
||||
return [
|
||||
bp.db.fs_tx_hash(tx_num)[0][::-1].hex()
|
||||
for tx_num in range(bp.db.tx_counts[height - 1], bp.db.tx_counts[height])
|
||||
]
|
||||
|
||||
block_hash = await self.blockchain.get_block_hash(height)
|
||||
|
||||
self.assertEqual(block_hash, (await self.ledger.headers.hash(height)).decode())
|
||||
self.assertEqual(block_hash, (await bp.db.fs_block_hashes(height, 1))[0][::-1].hex())
|
||||
|
||||
txids = await asyncio.get_event_loop().run_in_executor(bp.db.executor, get_txids)
|
||||
txs = await bp.db.fs_transactions(txids)
|
||||
block_txs = (await bp.daemon.deserialised_block(block_hash))['tx']
|
||||
self.assertSetEqual(set(block_txs), set(txs.keys()), msg='leveldb/lbrycrd is missing transactions')
|
||||
self.assertListEqual(block_txs, list(txs.keys()), msg='leveldb/lbrycrd transactions are of order')
|
||||
|
||||
async def test_reorg(self):
|
||||
REORG_COUNT.set(0)
|
||||
bp = self.conductor.spv_node.server.bp
|
||||
bp.reorg_count_metric.set(0)
|
||||
# invalidate current block, move forward 2
|
||||
self.assertEqual(self.ledger.headers.height, 206)
|
||||
await self.assertBlockHash(206)
|
||||
height = 206
|
||||
self.assertEqual(self.ledger.headers.height, height)
|
||||
await self.assertBlockHash(height)
|
||||
await self.blockchain.invalidate_block((await self.ledger.headers.hash(206)).decode())
|
||||
await self.blockchain.generate(2)
|
||||
await self.ledger.on_header.where(lambda e: e.height == 207)
|
||||
self.assertEqual(self.ledger.headers.height, 207)
|
||||
await self.assertBlockHash(206)
|
||||
await self.assertBlockHash(207)
|
||||
self.assertEqual(1, REORG_COUNT._samples()[0][2])
|
||||
self.assertEqual(1, bp.reorg_count_metric._samples()[0][2])
|
||||
|
||||
# invalidate current block, move forward 3
|
||||
await self.blockchain.invalidate_block((await self.ledger.headers.hash(206)).decode())
|
||||
@@ -36,7 +51,12 @@ class BlockchainReorganizationTests(CommandTestCase):
|
||||
await self.assertBlockHash(206)
|
||||
await self.assertBlockHash(207)
|
||||
await self.assertBlockHash(208)
|
||||
self.assertEqual(2, REORG_COUNT._samples()[0][2])
|
||||
self.assertEqual(2, bp.reorg_count_metric._samples()[0][2])
|
||||
await self.blockchain.generate(3)
|
||||
await self.ledger.on_header.where(lambda e: e.height == 211)
|
||||
await self.assertBlockHash(209)
|
||||
await self.assertBlockHash(210)
|
||||
await self.assertBlockHash(211)
|
||||
|
||||
async def test_reorg_change_claim_height(self):
|
||||
# sanity check
|
||||
@@ -51,6 +71,8 @@ class BlockchainReorganizationTests(CommandTestCase):
|
||||
|
||||
# create a claim and verify it's returned by claim_search
|
||||
self.assertEqual(self.ledger.headers.height, 207)
|
||||
await self.assertBlockHash(207)
|
||||
|
||||
broadcast_tx = await self.daemon.jsonrpc_stream_create(
|
||||
'hovercraft', '1.0', file_path=self.create_upload_file(data=b'hi!')
|
||||
)
|
||||
@@ -58,6 +80,8 @@ class BlockchainReorganizationTests(CommandTestCase):
|
||||
await self.generate(1)
|
||||
await self.ledger.wait(broadcast_tx, self.blockchain.block_expected)
|
||||
self.assertEqual(self.ledger.headers.height, 208)
|
||||
await self.assertBlockHash(208)
|
||||
|
||||
txos, _, _, _ = await self.ledger.claim_search([], name='hovercraft')
|
||||
self.assertEqual(1, len(txos))
|
||||
txo = txos[0]
|
||||
@@ -75,17 +99,18 @@ class BlockchainReorganizationTests(CommandTestCase):
|
||||
await self.blockchain.clear_mempool()
|
||||
await self.blockchain.generate(2)
|
||||
|
||||
# wait for the client to catch up and verify the reorg
|
||||
await asyncio.wait_for(self.on_header(209), 3.0)
|
||||
await self.assertBlockHash(207)
|
||||
await self.assertBlockHash(208)
|
||||
await self.assertBlockHash(209)
|
||||
|
||||
# verify the claim was dropped from block 208 as returned by lbrycrdd
|
||||
reorg_block_hash = await self.blockchain.get_block_hash(208)
|
||||
self.assertNotEqual(invalidated_block_hash, reorg_block_hash)
|
||||
block_207 = await self.blockchain.get_block(reorg_block_hash)
|
||||
self.assertNotIn(txo.tx_ref.id, block_207['tx'])
|
||||
|
||||
# wait for the client to catch up and verify the reorg
|
||||
await asyncio.wait_for(self.on_header(209), 3.0)
|
||||
await self.assertBlockHash(207)
|
||||
await self.assertBlockHash(208)
|
||||
await self.assertBlockHash(209)
|
||||
client_reorg_block_hash = (await self.ledger.headers.hash(208)).decode()
|
||||
self.assertEqual(client_reorg_block_hash, reorg_block_hash)
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ from binascii import unhexlify
|
||||
from urllib.request import urlopen
|
||||
|
||||
from lbry.error import InsufficientFundsError
|
||||
from lbry.extras.daemon.comment_client import verify
|
||||
|
||||
from lbry.extras.daemon.daemon import DEFAULT_PAGE_SIZE
|
||||
from lbry.testcase import CommandTestCase
|
||||
@@ -89,7 +90,7 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
# 23829 claim ids makes the request just large enough
|
||||
claim_ids = [
|
||||
'0000000000000000000000000000000000000000',
|
||||
] * 23829
|
||||
] * 33829
|
||||
with self.assertRaises(ConnectionResetError):
|
||||
await self.claim_search(claim_ids=claim_ids)
|
||||
|
||||
@@ -262,6 +263,19 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
await self.assertFindsClaims([claim4, claim3, claim2], fee_amount='<1.0', fee_currency='lbc')
|
||||
await self.assertFindsClaims([claim3], fee_amount='0.5', fee_currency='lbc')
|
||||
await self.assertFindsClaims([claim5], fee_currency='usd')
|
||||
|
||||
async def test_search_by_language(self):
|
||||
claim1 = await self.stream_create('claim1', fee_amount='1.0', fee_currency='lbc')
|
||||
claim2 = await self.stream_create('claim2', fee_amount='0.9', fee_currency='lbc')
|
||||
claim3 = await self.stream_create('claim3', fee_amount='0.5', fee_currency='lbc', languages='en')
|
||||
claim4 = await self.stream_create('claim4', fee_amount='0.1', fee_currency='lbc', languages='en')
|
||||
claim5 = await self.stream_create('claim5', fee_amount='1.0', fee_currency='usd', languages='es')
|
||||
|
||||
await self.assertFindsClaims([claim4, claim3], any_languages=['en'])
|
||||
await self.assertFindsClaims([claim2, claim1], any_languages=['none'])
|
||||
await self.assertFindsClaims([claim4, claim3, claim2, claim1], any_languages=['none', 'en'])
|
||||
await self.assertFindsClaims([claim5], any_languages=['es'])
|
||||
await self.assertFindsClaims([claim5, claim4, claim3], any_languages=['en', 'es'])
|
||||
await self.assertFindsClaims([], fee_currency='foo')
|
||||
|
||||
async def test_search_by_channel(self):
|
||||
@@ -319,6 +333,25 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
not_channel_ids=[chan2_id], has_channel_signature=True, valid_channel_signature=True)
|
||||
await match([], not_channel_ids=[chan1_id, chan2_id], has_channel_signature=True, valid_channel_signature=True)
|
||||
|
||||
async def test_limit_claims_per_channel(self):
|
||||
match = self.assertFindsClaims
|
||||
chan1_id = self.get_claim_id(await self.channel_create('@chan1'))
|
||||
chan2_id = self.get_claim_id(await self.channel_create('@chan2'))
|
||||
claim1 = await self.stream_create('claim1')
|
||||
claim2 = await self.stream_create('claim2', channel_id=chan1_id)
|
||||
claim3 = await self.stream_create('claim3', channel_id=chan1_id)
|
||||
claim4 = await self.stream_create('claim4', channel_id=chan1_id)
|
||||
claim5 = await self.stream_create('claim5', channel_id=chan2_id)
|
||||
claim6 = await self.stream_create('claim6', channel_id=chan2_id)
|
||||
await match(
|
||||
[claim6, claim5, claim4, claim3, claim1],
|
||||
limit_claims_per_channel=2, claim_type='stream'
|
||||
)
|
||||
await match(
|
||||
[claim6, claim5, claim4, claim3, claim2, claim1],
|
||||
limit_claims_per_channel=3, claim_type='stream'
|
||||
)
|
||||
|
||||
async def test_claim_type_and_media_type_search(self):
|
||||
# create an invalid/unknown claim
|
||||
address = await self.account.receiving.get_or_create_usable_address()
|
||||
@@ -332,6 +365,7 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
video = await self.stream_create('chrome', file_path=self.video_file_name)
|
||||
image = await self.stream_create('blank-image', data=self.image_data, suffix='.png')
|
||||
repost = await self.stream_repost(self.get_claim_id(image))
|
||||
collection = await self.collection_create('a-collection', claims=[self.get_claim_id(video)])
|
||||
channel = await self.channel_create()
|
||||
unknown = self.sout(tx)
|
||||
|
||||
@@ -339,6 +373,7 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
await self.assertFindsClaims([image, video, octet, unknown], claim_type='stream')
|
||||
await self.assertFindsClaims([channel], claim_type='channel')
|
||||
await self.assertFindsClaims([repost], claim_type='repost')
|
||||
await self.assertFindsClaims([collection], claim_type='collection')
|
||||
|
||||
# stream_type
|
||||
await self.assertFindsClaims([octet, unknown], stream_types=['binary'])
|
||||
@@ -428,6 +463,47 @@ class TransactionCommands(ClaimTestCase):
|
||||
|
||||
class TransactionOutputCommands(ClaimTestCase):
|
||||
|
||||
async def test_txo_list_resolve_supports(self):
|
||||
channel = self.get_claim_id(await self.channel_create('@identity'))
|
||||
stream = self.get_claim_id(await self.stream_create())
|
||||
support = await self.support_create(stream, channel_id=channel)
|
||||
r, = await self.txo_list(type='support')
|
||||
self.assertEqual(r['txid'], support['txid'])
|
||||
self.assertNotIn('name', r['signing_channel'])
|
||||
r, = await self.txo_list(type='support', resolve=True)
|
||||
self.assertIn('name', r['signing_channel'])
|
||||
self.assertEqual(r['signing_channel']['name'], '@identity')
|
||||
|
||||
async def test_txo_list_by_channel_filtering(self):
|
||||
channel_foo = self.get_claim_id(await self.channel_create('@foo'))
|
||||
channel_bar = self.get_claim_id(await self.channel_create('@bar'))
|
||||
stream_a = self.get_claim_id(await self.stream_create('a', channel_id=channel_foo))
|
||||
stream_b = self.get_claim_id(await self.stream_create('b', channel_id=channel_foo))
|
||||
stream_c = self.get_claim_id(await self.stream_create('c', channel_id=channel_bar))
|
||||
stream_d = self.get_claim_id(await self.stream_create('d'))
|
||||
support_c = await self.support_create(stream_c, '0.3', channel_id=channel_foo)
|
||||
support_d = await self.support_create(stream_d, '0.3', channel_id=channel_bar)
|
||||
|
||||
r = await self.txo_list(type='stream')
|
||||
self.assertEqual({stream_a, stream_b, stream_c, stream_d}, {c['claim_id'] for c in r})
|
||||
|
||||
r = await self.txo_list(type='stream', channel_id=channel_foo)
|
||||
self.assertEqual({stream_a, stream_b}, {c['claim_id'] for c in r})
|
||||
|
||||
r = await self.txo_list(type='support', channel_id=channel_foo)
|
||||
self.assertEqual({support_c['txid']}, {s['txid'] for s in r})
|
||||
r = await self.txo_list(type='support', channel_id=channel_bar)
|
||||
self.assertEqual({support_d['txid']}, {s['txid'] for s in r})
|
||||
|
||||
r = await self.txo_list(type='stream', channel_id=[channel_foo, channel_bar])
|
||||
self.assertEqual({stream_a, stream_b, stream_c}, {c['claim_id'] for c in r})
|
||||
|
||||
r = await self.txo_list(type='stream', not_channel_id=channel_foo)
|
||||
self.assertEqual({stream_c, stream_d}, {c['claim_id'] for c in r})
|
||||
|
||||
r = await self.txo_list(type='stream', not_channel_id=[channel_foo, channel_bar])
|
||||
self.assertEqual({stream_d}, {c['claim_id'] for c in r})
|
||||
|
||||
async def test_txo_list_and_sum_filtering(self):
|
||||
channel_id = self.get_claim_id(await self.channel_create())
|
||||
self.assertEqual('1.0', lbc(await self.txo_sum(type='channel', is_not_spent=True)))
|
||||
@@ -661,12 +737,15 @@ class ClaimCommands(ClaimTestCase):
|
||||
channel_id = self.get_claim_id(await self.channel_create())
|
||||
stream_id = self.get_claim_id(await self.stream_create())
|
||||
|
||||
await self.stream_update(stream_id, title='foo')
|
||||
|
||||
# type filtering
|
||||
r = await self.claim_list(claim_type='channel')
|
||||
self.assertEqual(1, len(r))
|
||||
self.assertEqual('channel', r[0]['value_type'])
|
||||
|
||||
r = await self.claim_list(claim_type='stream')
|
||||
# catch a bug where cli sends is_spent=False by default
|
||||
r = await self.claim_list(claim_type='stream', is_spent=False)
|
||||
self.assertEqual(1, len(r))
|
||||
self.assertEqual('stream', r[0]['value_type'])
|
||||
|
||||
@@ -969,6 +1048,18 @@ class ChannelCommands(CommandTestCase):
|
||||
self.assertItemCount(await self.daemon.jsonrpc_channel_list(account_id=self.account.id), 2)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_channel_list(account_id=account2_id), 1)
|
||||
|
||||
async def test_sign_hex_encoded_data(self):
|
||||
data_to_sign = "CAFEBABE"
|
||||
# claim new name
|
||||
await self.channel_create('@someotherchan')
|
||||
channel_tx = await self.daemon.jsonrpc_channel_create('@signer', '0.1')
|
||||
await self.confirm_tx(channel_tx.id)
|
||||
channel = channel_tx.outputs[0]
|
||||
signature1 = await self.out(self.daemon.jsonrpc_channel_sign(channel_name='@signer', hexdata=data_to_sign))
|
||||
signature2 = await self.out(self.daemon.jsonrpc_channel_sign(channel_id=channel.claim_id, hexdata=data_to_sign))
|
||||
self.assertTrue(verify(channel, unhexlify(data_to_sign), signature1))
|
||||
self.assertTrue(verify(channel, unhexlify(data_to_sign), signature2))
|
||||
|
||||
async def test_channel_export_import_before_sending_channel(self):
|
||||
# export
|
||||
tx = await self.channel_create('@foo', '1.0')
|
||||
@@ -1825,7 +1916,8 @@ class SupportCommands(CommandTestCase):
|
||||
# send a tip to the claim using account2
|
||||
tip = await self.out(
|
||||
self.daemon.jsonrpc_support_create(
|
||||
claim_id, '1.0', True, account2.id, 'wallet2', funding_account_ids=[account2.id])
|
||||
claim_id, '1.0', True, account_id=account2.id, wallet_id='wallet2',
|
||||
funding_account_ids=[account2.id])
|
||||
)
|
||||
await self.confirm_tx(tip['txid'])
|
||||
|
||||
@@ -1856,7 +1948,8 @@ class SupportCommands(CommandTestCase):
|
||||
# send a support to the claim using account2
|
||||
support = await self.out(
|
||||
self.daemon.jsonrpc_support_create(
|
||||
claim_id, '2.0', False, account2.id, 'wallet2', funding_account_ids=[account2.id])
|
||||
claim_id, '2.0', False, account_id=account2.id, wallet_id='wallet2',
|
||||
funding_account_ids=[account2.id])
|
||||
)
|
||||
await self.confirm_tx(support['txid'])
|
||||
|
||||
@@ -1883,6 +1976,12 @@ class SupportCommands(CommandTestCase):
|
||||
self.assertTrue(txs[1]['support_info'][0]['is_tip'])
|
||||
self.assertTrue(txs[1]['support_info'][0]['is_spent'])
|
||||
|
||||
async def test_signed_supports(self):
|
||||
channel_id = self.get_claim_id(await self.channel_create())
|
||||
stream_id = self.get_claim_id(await self.stream_create())
|
||||
tx = await self.support_create(stream_id, '0.3', channel_id=channel_id)
|
||||
self.assertTrue(tx['outputs'][0]['is_channel_signature_valid'])
|
||||
|
||||
|
||||
class CollectionCommands(CommandTestCase):
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ class BasicTransactionTest(IntegrationTestCase):
|
||||
channel_txo = Output.pay_claim_name_pubkey_hash(
|
||||
l2d('1.0'), '@bar', channel, self.account.ledger.address_to_hash160(address1)
|
||||
)
|
||||
channel_txo.generate_channel_private_key()
|
||||
await channel_txo.generate_channel_private_key()
|
||||
channel_txo.script.generate()
|
||||
channel_tx = await Transaction.create([], [channel_txo], [self.account], self.account)
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ from unittest.mock import Mock
|
||||
from lbry.wallet.network import Network
|
||||
from lbry.wallet.orchstr8.node import SPVNode
|
||||
from lbry.wallet.rpc import RPCSession
|
||||
from lbry.wallet.server.udp import StatusServer
|
||||
from lbry.testcase import IntegrationTestCase, AsyncioTestCase
|
||||
|
||||
|
||||
@@ -32,18 +33,17 @@ class NetworkTests(IntegrationTestCase):
|
||||
'server_version': lbry.__version__,
|
||||
'trending_algorithm': 'zscore',
|
||||
}, await self.ledger.network.get_server_features())
|
||||
await self.conductor.spv_node.stop()
|
||||
# await self.conductor.spv_node.stop()
|
||||
payment_address, donation_address = await self.account.get_addresses(limit=2)
|
||||
await self.conductor.spv_node.start(
|
||||
self.conductor.blockchain_node,
|
||||
extraconf={
|
||||
'DESCRIPTION': 'Fastest server in the west.',
|
||||
'PAYMENT_ADDRESS': payment_address,
|
||||
'DONATION_ADDRESS': donation_address,
|
||||
'DAILY_FEE': '42'
|
||||
}
|
||||
)
|
||||
await self.ledger.network.on_connected.first
|
||||
self.conductor.spv_node.server.env.payment_address = payment_address
|
||||
self.conductor.spv_node.server.env.donation_address = donation_address
|
||||
self.conductor.spv_node.server.env.description = 'Fastest server in the west.'
|
||||
self.conductor.spv_node.server.env.daily_fee = '42'
|
||||
|
||||
from lbry.wallet.server.session import LBRYElectrumX
|
||||
LBRYElectrumX.set_server_features(self.conductor.spv_node.server.env)
|
||||
|
||||
# await self.ledger.network.on_connected.first
|
||||
self.assertDictEqual({
|
||||
'genesis_hash': self.conductor.spv_node.coin_class.GENESIS_HASH,
|
||||
'hash_function': 'sha256',
|
||||
@@ -65,22 +65,21 @@ class ReconnectTests(IntegrationTestCase):
|
||||
async def test_multiple_servers(self):
|
||||
# we have a secondary node that connects later, so
|
||||
node2 = SPVNode(self.conductor.spv_module, node_number=2)
|
||||
self.ledger.network.config['default_servers'].append((node2.hostname, node2.port))
|
||||
await asyncio.wait_for(self.ledger.stop(), timeout=1)
|
||||
await asyncio.wait_for(self.ledger.start(), timeout=1)
|
||||
self.ledger.network.session_pool.new_connection_event.clear()
|
||||
await node2.start(self.blockchain)
|
||||
# this is only to speed up the test as retrying would take 4+ seconds
|
||||
for session in self.ledger.network.session_pool.sessions:
|
||||
session.trigger_urgent_reconnect.set()
|
||||
await asyncio.wait_for(self.ledger.network.session_pool.new_connection_event.wait(), timeout=1)
|
||||
self.assertEqual(2, len(list(self.ledger.network.session_pool.available_sessions)))
|
||||
|
||||
self.ledger.network.config['default_servers'].append((node2.hostname, node2.port))
|
||||
self.ledger.network.config['default_servers'].reverse()
|
||||
self.assertEqual(50002, self.ledger.network.client.server[1])
|
||||
await self.ledger.stop()
|
||||
await self.ledger.start()
|
||||
|
||||
self.assertTrue(self.ledger.network.is_connected)
|
||||
switch_event = self.ledger.network.on_connected.first
|
||||
self.assertEqual(50003, self.ledger.network.client.server[1])
|
||||
await node2.stop(True)
|
||||
self.assertFalse(self.ledger.network.is_connected)
|
||||
await self.ledger.resolve([], ['derp'])
|
||||
self.assertEqual(50002, self.ledger.network.client.server[1])
|
||||
await node2.stop(True)
|
||||
# secondary down, but primary is ok, do not switch! (switches trigger new on_connected events)
|
||||
with self.assertRaises(asyncio.TimeoutError):
|
||||
await asyncio.wait_for(switch_event, timeout=1)
|
||||
|
||||
async def test_direct_sync(self):
|
||||
await self.ledger.stop()
|
||||
@@ -98,10 +97,13 @@ class ReconnectTests(IntegrationTestCase):
|
||||
async def test_connection_drop_still_receives_events_after_reconnected(self):
|
||||
address1 = await self.account.receiving.get_or_create_usable_address()
|
||||
# disconnect and send a new tx, should reconnect and get it
|
||||
self.ledger.network.client.connection_lost(Exception())
|
||||
self.ledger.network.client.transport.close()
|
||||
self.assertFalse(self.ledger.network.is_connected)
|
||||
await self.ledger.resolve([], 'derp')
|
||||
sendtxid = await self.blockchain.send_to_address(address1, 1.1337)
|
||||
await asyncio.wait_for(self.on_transaction_id(sendtxid), 1.0) # mempool
|
||||
# await self.ledger.resolve([], 'derp')
|
||||
# self.assertTrue(self.ledger.network.is_connected)
|
||||
await asyncio.wait_for(self.on_transaction_id(sendtxid), 10.0) # mempool
|
||||
await self.blockchain.generate(1)
|
||||
await self.on_transaction_id(sendtxid) # confirmed
|
||||
self.assertLess(self.ledger.network.client.response_time, 1) # response time properly set lower, we are fine
|
||||
@@ -123,7 +125,7 @@ class ReconnectTests(IntegrationTestCase):
|
||||
await self.blockchain.generate(1)
|
||||
# (this is just so the test doesn't hang forever if it doesn't reconnect)
|
||||
if not self.ledger.network.is_connected:
|
||||
await asyncio.wait_for(self.ledger.network.on_connected.first, timeout=1.0)
|
||||
await asyncio.wait_for(self.ledger.network.on_connected.first, timeout=10.0)
|
||||
# omg, the burned cable still works! torba is fire proof!
|
||||
await self.ledger.network.get_transaction(sendtxid)
|
||||
|
||||
@@ -136,15 +138,19 @@ class ReconnectTests(IntegrationTestCase):
|
||||
await self.ledger.network.on_connected.first
|
||||
self.assertTrue(self.ledger.network.is_connected)
|
||||
|
||||
async def test_online_but_still_unavailable(self):
|
||||
# Edge case. See issue #2445 for context
|
||||
self.assertIsNotNone(self.ledger.network.session_pool.fastest_session)
|
||||
for session in self.ledger.network.session_pool.sessions:
|
||||
session.response_time = None
|
||||
self.assertIsNone(self.ledger.network.session_pool.fastest_session)
|
||||
# async def test_online_but_still_unavailable(self):
|
||||
# # Edge case. See issue #2445 for context
|
||||
# self.assertIsNotNone(self.ledger.network.session_pool.fastest_session)
|
||||
# for session in self.ledger.network.session_pool.sessions:
|
||||
# session.response_time = None
|
||||
# self.assertIsNone(self.ledger.network.session_pool.fastest_session)
|
||||
|
||||
|
||||
class ServerPickingTestCase(AsyncioTestCase):
|
||||
async def _make_udp_server(self, port):
|
||||
s = StatusServer()
|
||||
await s.start(0, b'\x00' * 32, '127.0.0.1', port)
|
||||
self.addCleanup(s.stop)
|
||||
|
||||
async def _make_fake_server(self, latency=1.0, port=1):
|
||||
# local fake server with artificial latency
|
||||
@@ -156,6 +162,7 @@ class ServerPickingTestCase(AsyncioTestCase):
|
||||
return {'height': 1}
|
||||
server = await self.loop.create_server(lambda: FakeSession(), host='127.0.0.1', port=port)
|
||||
self.addCleanup(server.close)
|
||||
await self._make_udp_server(port)
|
||||
return '127.0.0.1', port
|
||||
|
||||
async def _make_bad_server(self, port=42420):
|
||||
@@ -164,9 +171,10 @@ class ServerPickingTestCase(AsyncioTestCase):
|
||||
writer.write(await reader.read())
|
||||
server = await asyncio.start_server(echo, host='127.0.0.1', port=port)
|
||||
self.addCleanup(server.close)
|
||||
await self._make_udp_server(port)
|
||||
return '127.0.0.1', port
|
||||
|
||||
async def test_pick_fastest(self):
|
||||
async def _test_pick_fastest(self):
|
||||
ledger = Mock(config={
|
||||
'default_servers': [
|
||||
# fast but unhealthy, should be discarded
|
||||
@@ -182,8 +190,8 @@ class ServerPickingTestCase(AsyncioTestCase):
|
||||
|
||||
network = Network(ledger)
|
||||
self.addCleanup(network.stop)
|
||||
asyncio.ensure_future(network.start())
|
||||
await asyncio.wait_for(network.on_connected.first, timeout=1)
|
||||
await network.start()
|
||||
await asyncio.wait_for(network.on_connected.first, timeout=10)
|
||||
self.assertTrue(network.is_connected)
|
||||
self.assertTupleEqual(network.client.server, ('127.0.0.1', 1337))
|
||||
self.assertTrue(all([not session.is_closing() for session in network.session_pool.available_sessions]))
|
||||
|
||||
@@ -11,7 +11,9 @@ class PurchaseCommandTests(CommandTestCase):
|
||||
await super().asyncSetUp()
|
||||
self.merchant_address = await self.blockchain.get_raw_change_address()
|
||||
|
||||
async def priced_stream(self, name='stream', price: Optional[str] = '2.0', currency='LBC') -> Transaction:
|
||||
async def priced_stream(
|
||||
self, name='stream', price: Optional[str] = '0.2', currency='LBC', mine=False
|
||||
) -> Transaction:
|
||||
kwargs = {}
|
||||
if price and currency:
|
||||
kwargs = {
|
||||
@@ -19,6 +21,8 @@ class PurchaseCommandTests(CommandTestCase):
|
||||
'fee_currency': currency,
|
||||
'fee_address': self.merchant_address
|
||||
}
|
||||
if not mine:
|
||||
kwargs['claim_address'] = self.merchant_address
|
||||
file_path = self.create_upload_file(data=b'high value content')
|
||||
tx = await self.daemon.jsonrpc_stream_create(
|
||||
name, '0.01', file_path=file_path, **kwargs
|
||||
@@ -117,6 +121,13 @@ class PurchaseCommandTests(CommandTestCase):
|
||||
abc_stream = await self.priced_stream('abc')
|
||||
await self.assertStreamPurchased(abc_stream, lambda: self.daemon.jsonrpc_purchase_create(url='lbry://abc'))
|
||||
|
||||
# purchase without valid exchange rate fails
|
||||
erm = self.daemon.component_manager.get_component('exchange_rate_manager')
|
||||
for feed in erm.market_feeds:
|
||||
feed.last_check -= 10_000
|
||||
with self.assertRaisesRegex(Exception, "Unable to convert 50 from USD to LBC"):
|
||||
await self.daemon.jsonrpc_purchase_create(claim_id, allow_duplicate_purchase=True)
|
||||
|
||||
async def test_purchase_and_transaction_list(self):
|
||||
self.assertItemCount(await self.daemon.jsonrpc_purchase_list(), 0)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_transaction_list(), 1)
|
||||
@@ -182,3 +193,12 @@ class PurchaseCommandTests(CommandTestCase):
|
||||
await self.ledger.wait(spend)
|
||||
await self.assertBalance(self.account, '0.487695')
|
||||
self.assertItemCount(await self.daemon.jsonrpc_utxo_list(), 1)
|
||||
|
||||
async def test_owner_not_required_purchase_own_content(self):
|
||||
await self.priced_stream(mine=True)
|
||||
# check that `get` doesn't purchase own claim
|
||||
balance = await self.account.get_balance()
|
||||
response = await self.daemon.jsonrpc_get('lbry://stream')
|
||||
self.assertIsNone(response.content_fee)
|
||||
self.assertEqual(await self.account.get_balance(), balance)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_purchase_list(), 0)
|
||||
@@ -117,11 +117,6 @@ class ResolveCommand(BaseResolveTestCase):
|
||||
await self.stream_create('foo', '0.9', allow_duplicate_name=True))
|
||||
# plain winning claim
|
||||
await self.assertResolvesToClaimId('foo', claim_id3)
|
||||
# sequence resolution
|
||||
await self.assertResolvesToClaimId('foo:1', claim_id1)
|
||||
await self.assertResolvesToClaimId('foo:2', claim_id2)
|
||||
await self.assertResolvesToClaimId('foo:3', claim_id3)
|
||||
await self.assertResolvesToClaimId('foo:4', None)
|
||||
# amount order resolution
|
||||
await self.assertResolvesToClaimId('foo$1', claim_id3)
|
||||
await self.assertResolvesToClaimId('foo$2', claim_id2)
|
||||
|
||||
@@ -5,6 +5,7 @@ from itertools import chain
|
||||
from lbry.wallet.transaction import Transaction, Output, Input
|
||||
from lbry.testcase import IntegrationTestCase
|
||||
from lbry.wallet.util import satoshis_to_coins, coins_to_satoshis
|
||||
from lbry.wallet.manager import WalletManager
|
||||
|
||||
|
||||
class BasicTransactionTests(IntegrationTestCase):
|
||||
@@ -137,8 +138,8 @@ class BasicTransactionTests(IntegrationTestCase):
|
||||
# evil trick: mempool is unsorted on real life, but same order between python instances. reproduce it
|
||||
original_summary = self.conductor.spv_node.server.mempool.transaction_summaries
|
||||
|
||||
async def random_summary(*args, **kwargs):
|
||||
summary = await original_summary(*args, **kwargs)
|
||||
def random_summary(*args, **kwargs):
|
||||
summary = original_summary(*args, **kwargs)
|
||||
if summary and len(summary) > 2:
|
||||
ordered = summary.copy()
|
||||
while summary == ordered:
|
||||
@@ -173,3 +174,130 @@ class BasicTransactionTests(IntegrationTestCase):
|
||||
self.assertTrue(await self.ledger.update_history(address, remote_status))
|
||||
self.assertEqual(21, len((await self.ledger.get_local_status_and_history(address))[1]))
|
||||
self.assertEqual(0, len(self.ledger._known_addresses_out_of_sync))
|
||||
|
||||
def wait_for_txid(self, txid, address):
|
||||
return self.ledger.on_transaction.where(
|
||||
lambda e: e.tx.id == txid and e.address == address
|
||||
)
|
||||
|
||||
async def _test_transaction(self, send_amount, address, inputs, change):
|
||||
tx = await Transaction.create(
|
||||
[], [Output.pay_pubkey_hash(send_amount, self.ledger.address_to_hash160(address))], [self.account],
|
||||
self.account
|
||||
)
|
||||
await self.ledger.broadcast(tx)
|
||||
input_amounts = [txi.amount for txi in tx.inputs]
|
||||
self.assertListEqual(inputs, input_amounts)
|
||||
self.assertEqual(len(inputs), len(tx.inputs))
|
||||
self.assertEqual(2, len(tx.outputs))
|
||||
self.assertEqual(send_amount, tx.outputs[0].amount)
|
||||
self.assertEqual(change, tx.outputs[1].amount)
|
||||
return tx
|
||||
|
||||
async def assertSpendable(self, amounts):
|
||||
spendable = await self.ledger.db.get_spendable_utxos(
|
||||
self.ledger, 2000000000000, [self.account], set_reserved=False, return_insufficient_funds=True
|
||||
)
|
||||
got_amounts = [estimator.effective_amount for estimator in spendable]
|
||||
self.assertListEqual(amounts, got_amounts)
|
||||
|
||||
async def test_sqlite_coin_chooser(self):
|
||||
wallet_manager = WalletManager([self.wallet], {self.ledger.get_id(): self.ledger})
|
||||
await self.blockchain.generate(300)
|
||||
await self.assertBalance(self.account, '0.0')
|
||||
address = await self.account.receiving.get_or_create_usable_address()
|
||||
other_account = self.wallet.generate_account(self.ledger)
|
||||
other_address = await other_account.receiving.get_or_create_usable_address()
|
||||
self.ledger.coin_selection_strategy = 'sqlite'
|
||||
await self.ledger.subscribe_account(self.account)
|
||||
|
||||
txids = []
|
||||
txids.append(await self.blockchain.send_to_address(address, 1.0))
|
||||
txids.append(await self.blockchain.send_to_address(address, 1.0))
|
||||
txids.append(await self.blockchain.send_to_address(address, 3.0))
|
||||
txids.append(await self.blockchain.send_to_address(address, 5.0))
|
||||
txids.append(await self.blockchain.send_to_address(address, 10.0))
|
||||
|
||||
await asyncio.wait([self.wait_for_txid(txid, address) for txid in txids], timeout=1)
|
||||
await self.assertBalance(self.account, '20.0')
|
||||
await self.assertSpendable([99992600, 99992600, 299992600, 499992600, 999992600])
|
||||
|
||||
# send 1.5 lbc
|
||||
|
||||
first_tx = await Transaction.create(
|
||||
[], [Output.pay_pubkey_hash(150000000, self.ledger.address_to_hash160(other_address))], [self.account],
|
||||
self.account
|
||||
)
|
||||
|
||||
self.assertEqual(2, len(first_tx.inputs))
|
||||
self.assertEqual(2, len(first_tx.outputs))
|
||||
self.assertEqual(100000000, first_tx.inputs[0].amount)
|
||||
self.assertEqual(100000000, first_tx.inputs[1].amount)
|
||||
self.assertEqual(150000000, first_tx.outputs[0].amount)
|
||||
self.assertEqual(49980200, first_tx.outputs[1].amount)
|
||||
|
||||
await self.assertBalance(self.account, '18.0')
|
||||
await self.assertSpendable([299992600, 499992600, 999992600])
|
||||
|
||||
await wallet_manager.broadcast_or_release(first_tx, blocking=True)
|
||||
await self.assertSpendable([49972800, 299992600, 499992600, 999992600])
|
||||
# 0.499, 3.0, 5.0, 10.0
|
||||
await self.assertBalance(self.account, '18.499802')
|
||||
|
||||
# send 1.5lbc again
|
||||
|
||||
second_tx = await self._test_transaction(150000000, other_address, [49980200, 300000000], 199960400)
|
||||
await self.assertSpendable([499992600, 999992600])
|
||||
|
||||
# replicate cancelling the api call after the tx broadcast while ledger.wait'ing it
|
||||
e = asyncio.Event()
|
||||
|
||||
real_broadcast = self.ledger.broadcast
|
||||
|
||||
async def broadcast(tx):
|
||||
try:
|
||||
return await real_broadcast(tx)
|
||||
finally:
|
||||
e.set()
|
||||
|
||||
self.ledger.broadcast = broadcast
|
||||
|
||||
broadcast_task = asyncio.create_task(wallet_manager.broadcast_or_release(second_tx, blocking=True))
|
||||
# wait for the broadcast to finish
|
||||
await e.wait()
|
||||
# cancel the api call
|
||||
broadcast_task.cancel()
|
||||
with self.assertRaises(asyncio.CancelledError):
|
||||
await broadcast_task
|
||||
|
||||
# test if sending another 1.5 lbc will try to double spend the inputs from the cancelled tx
|
||||
tx1 = await self._test_transaction(150000000, other_address, [500000000], 349987600)
|
||||
await self.ledger.wait(tx1, timeout=1)
|
||||
# wait for the cancelled transaction too, so that it's in the database
|
||||
# needed to keep everything deterministic
|
||||
await self.ledger.wait(second_tx, timeout=1)
|
||||
await self.assertSpendable([199953000, 349980200, 999992600])
|
||||
|
||||
# spend deep into the mempool and see what else breaks
|
||||
tx2 = await self._test_transaction(150000000, other_address, [199960400], 49948000)
|
||||
await self.assertSpendable([349980200, 999992600])
|
||||
await self.ledger.wait(tx2, timeout=1)
|
||||
await self.assertSpendable([49940600, 349980200, 999992600])
|
||||
|
||||
tx3 = await self._test_transaction(150000000, other_address, [49948000, 349987600], 249915800)
|
||||
await self.assertSpendable([999992600])
|
||||
await self.ledger.wait(tx3, timeout=1)
|
||||
await self.assertSpendable([249908400, 999992600])
|
||||
|
||||
tx4 = await self._test_transaction(150000000, other_address, [249915800], 99903400)
|
||||
await self.assertSpendable([999992600])
|
||||
await self.ledger.wait(tx4, timeout=1)
|
||||
await self.assertBalance(self.account, '10.999034')
|
||||
await self.assertSpendable([99896000, 999992600])
|
||||
|
||||
# spend more
|
||||
tx5 = await self._test_transaction(100000000, other_address, [99903400, 1000000000], 999883600)
|
||||
await self.assertSpendable([])
|
||||
await self.ledger.wait(tx5, timeout=1)
|
||||
await self.assertSpendable([999876200])
|
||||
await self.assertBalance(self.account, '9.998836')
|
||||
@@ -10,7 +10,7 @@ from lbry.wallet.dewies import dict_values_to_lbc
|
||||
class WalletCommands(CommandTestCase):
|
||||
|
||||
async def test_wallet_create_and_add_subscribe(self):
|
||||
session = next(iter(self.conductor.spv_node.server.session_mgr.sessions))
|
||||
session = next(iter(self.conductor.spv_node.server.session_mgr.sessions.values()))
|
||||
self.assertEqual(len(session.hashX_subs), 27)
|
||||
wallet = await self.daemon.jsonrpc_wallet_create('foo', create_account=True, single_key=True)
|
||||
self.assertEqual(len(session.hashX_subs), 28)
|
||||
@@ -43,12 +43,14 @@ class WalletCommands(CommandTestCase):
|
||||
)
|
||||
|
||||
async def test_wallet_reconnect(self):
|
||||
status = await self.daemon.jsonrpc_status()
|
||||
self.assertEqual(len(status['wallet']['servers']), 1)
|
||||
self.assertEqual(status['wallet']['servers'][0]['port'], 50002)
|
||||
await self.conductor.spv_node.stop(True)
|
||||
self.conductor.spv_node.port = 54320
|
||||
await self.conductor.spv_node.start(self.conductor.blockchain_node)
|
||||
status = await self.daemon.jsonrpc_status()
|
||||
self.assertEqual(len(status['wallet']['servers']), 1)
|
||||
self.assertEqual(status['wallet']['servers'][0]['port'], 50002)
|
||||
self.assertEqual(len(status['wallet']['servers']), 0)
|
||||
self.daemon.jsonrpc_settings_set('lbryum_servers', ['localhost:54320'])
|
||||
await self.daemon.jsonrpc_wallet_reconnect()
|
||||
status = await self.daemon.jsonrpc_status()
|
||||
|
||||
@@ -4,6 +4,7 @@ import lbry
|
||||
import lbry.wallet
|
||||
from lbry.error import ServerPaymentFeeAboveMaxAllowedError
|
||||
from lbry.wallet.network import ClientSession
|
||||
from lbry.wallet.server.session import LBRYElectrumX
|
||||
from lbry.testcase import IntegrationTestCase, CommandTestCase
|
||||
from lbry.wallet.orchstr8.node import SPVNode
|
||||
|
||||
@@ -46,7 +47,7 @@ class TestSessions(IntegrationTestCase):
|
||||
|
||||
|
||||
class TestUsagePayment(CommandTestCase):
|
||||
async def test_single_server_payment(self):
|
||||
async def _test_single_server_payment(self):
|
||||
wallet_pay_service = self.daemon.component_manager.get_component('wallet_server_payments')
|
||||
wallet_pay_service.payment_period = 1
|
||||
# only starts with a positive max key fee
|
||||
@@ -68,21 +69,21 @@ class TestUsagePayment(CommandTestCase):
|
||||
self.addCleanup(node.stop)
|
||||
self.daemon.jsonrpc_settings_set('lbryum_servers', [f"{node.hostname}:{node.port}"])
|
||||
await self.daemon.jsonrpc_wallet_reconnect()
|
||||
|
||||
LBRYElectrumX.set_server_features(node.server.env)
|
||||
features = await self.ledger.network.get_server_features()
|
||||
self.assertEqual(features["payment_address"], address)
|
||||
self.assertEqual(features["daily_fee"], "1.1")
|
||||
with self.assertRaises(ServerPaymentFeeAboveMaxAllowedError):
|
||||
await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=8)
|
||||
|
||||
await node.stop(False)
|
||||
await node.start(self.blockchain, extraconf={"PAYMENT_ADDRESS": address, "DAILY_FEE": "1.0"})
|
||||
self.daemon.jsonrpc_settings_set('lbryum_servers', [f"{node.hostname}:{node.port}"])
|
||||
await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=30)
|
||||
node.server.env.daily_fee = "1.0"
|
||||
node.server.env.payment_address = address
|
||||
LBRYElectrumX.set_server_features(node.server.env)
|
||||
# self.daemon.jsonrpc_settings_set('lbryum_servers', [f"{node.hostname}:{node.port}"])
|
||||
await self.daemon.jsonrpc_wallet_reconnect()
|
||||
features = await self.ledger.network.get_server_features()
|
||||
self.assertEqual(features["payment_address"], address)
|
||||
self.assertEqual(features["daily_fee"], "1.0")
|
||||
tx = await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=8)
|
||||
tx = await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=30)
|
||||
self.assertIsNotNone(await self.blockchain.get_raw_transaction(tx.id)) # verify its broadcasted
|
||||
self.assertEqual(tx.outputs[0].amount, 100000000)
|
||||
self.assertEqual(tx.outputs[0].get_address(self.ledger), address)
|
||||
@@ -2,10 +2,64 @@ import asyncio
|
||||
import os
|
||||
from binascii import hexlify
|
||||
|
||||
from lbry.schema import Claim
|
||||
from lbry.testcase import CommandTestCase
|
||||
from lbry.torrent.session import TorrentSession
|
||||
from lbry.wallet import Transaction
|
||||
|
||||
|
||||
class FileCommands(CommandTestCase):
|
||||
def __init__(self, *a, **kw):
|
||||
super().__init__(*a, **kw)
|
||||
self.skip_libtorrent = False
|
||||
|
||||
async def initialize_torrent(self, tx_to_update=None):
|
||||
if not hasattr(self, 'seeder_session'):
|
||||
self.seeder_session = TorrentSession(self.loop, None)
|
||||
self.addCleanup(self.seeder_session.stop)
|
||||
await self.seeder_session.bind(port=4040)
|
||||
btih = await self.seeder_session.add_fake_torrent()
|
||||
address = await self.account.receiving.get_or_create_usable_address()
|
||||
if not tx_to_update:
|
||||
claim = Claim()
|
||||
claim.stream.update(bt_infohash=btih)
|
||||
tx = await Transaction.claim_create(
|
||||
'torrent', claim, 1, address, [self.account], self.account
|
||||
)
|
||||
else:
|
||||
claim = tx_to_update.outputs[0].claim
|
||||
claim.stream.update(bt_infohash=btih)
|
||||
tx = await Transaction.claim_update(
|
||||
tx_to_update.outputs[0], claim, 1, address, [self.account], self.account
|
||||
)
|
||||
await tx.sign([self.account])
|
||||
await self.broadcast(tx)
|
||||
await self.confirm_tx(tx.id)
|
||||
self.client_session = self.daemon.file_manager.source_managers['torrent'].torrent_session
|
||||
self.client_session._session.add_dht_node(('localhost', 4040))
|
||||
self.client_session.wait_start = False # fixme: this is super slow on tests
|
||||
return tx, btih
|
||||
|
||||
async def test_download_torrent(self):
|
||||
tx, btih = await self.initialize_torrent()
|
||||
self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent')))
|
||||
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1)
|
||||
# second call, see its there and move on
|
||||
self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent')))
|
||||
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1)
|
||||
self.assertEqual((await self.daemon.jsonrpc_file_list())['items'][0].identifier, btih)
|
||||
self.assertIn(btih, self.client_session._handles)
|
||||
tx, new_btih = await self.initialize_torrent(tx)
|
||||
self.assertNotEqual(btih, new_btih)
|
||||
# claim now points to another torrent, update to it
|
||||
self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent')))
|
||||
self.assertEqual((await self.daemon.jsonrpc_file_list())['items'][0].identifier, new_btih)
|
||||
self.assertIn(new_btih, self.client_session._handles)
|
||||
self.assertNotIn(btih, self.client_session._handles)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1)
|
||||
await self.daemon.jsonrpc_file_delete(delete_all=True)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 0)
|
||||
self.assertNotIn(new_btih, self.client_session._handles)
|
||||
|
||||
async def create_streams_in_range(self, *args, **kwargs):
|
||||
self.stream_claim_ids = []
|
||||
@@ -228,11 +282,11 @@ class FileCommands(CommandTestCase):
|
||||
await self.daemon.jsonrpc_get('lbry://foo')
|
||||
with open(original_path, 'wb') as handle:
|
||||
handle.write(b'some other stuff was there instead')
|
||||
self.daemon.stream_manager.stop()
|
||||
await self.daemon.stream_manager.start()
|
||||
self.daemon.file_manager.stop()
|
||||
await self.daemon.file_manager.start()
|
||||
await asyncio.wait_for(self.wait_files_to_complete(), timeout=5) # if this hangs, file didn't get set completed
|
||||
# check that internal state got through up to the file list API
|
||||
stream = self.daemon.stream_manager.get_stream_by_stream_hash(file_info['stream_hash'])
|
||||
stream = self.daemon.file_manager.get_filtered(stream_hash=file_info['stream_hash'])[0]
|
||||
file_info = (await self.file_list())[0]
|
||||
self.assertEqual(stream.file_name, file_info['file_name'])
|
||||
# checks if what the API shows is what he have at the very internal level.
|
||||
@@ -255,7 +309,7 @@ class FileCommands(CommandTestCase):
|
||||
resp = await self.out(self.daemon.jsonrpc_get('lbry://foo', timeout=2))
|
||||
self.assertNotIn('error', resp)
|
||||
self.assertTrue(os.path.isfile(path))
|
||||
self.daemon.stream_manager.stop()
|
||||
self.daemon.file_manager.stop()
|
||||
await asyncio.sleep(0.01, loop=self.loop) # FIXME: this sleep should not be needed
|
||||
self.assertFalse(os.path.isfile(path))
|
||||
|
||||
@@ -304,7 +358,8 @@ class FileCommands(CommandTestCase):
|
||||
# FAIL: beyond available balance
|
||||
await self.stream_create(
|
||||
'expensive', '0.01', data=b'pay me if you can',
|
||||
fee_currency='LBC', fee_amount='11.0', fee_address=target_address
|
||||
fee_currency='LBC', fee_amount='11.0',
|
||||
fee_address=target_address, claim_address=target_address
|
||||
)
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='expensive')
|
||||
response = await self.out(self.daemon.jsonrpc_get('lbry://expensive'))
|
||||
@@ -314,7 +369,8 @@ class FileCommands(CommandTestCase):
|
||||
# FAIL: beyond maximum key fee
|
||||
await self.stream_create(
|
||||
'maxkey', '0.01', data=b'no pay me, no',
|
||||
fee_currency='LBC', fee_amount='111.0', fee_address=target_address
|
||||
fee_currency='LBC', fee_amount='111.0',
|
||||
fee_address=target_address, claim_address=target_address
|
||||
)
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='maxkey')
|
||||
response = await self.out(self.daemon.jsonrpc_get('lbry://maxkey'))
|
||||
@@ -326,7 +382,8 @@ class FileCommands(CommandTestCase):
|
||||
# PASS: purchase is successful
|
||||
await self.stream_create(
|
||||
'icanpay', '0.01', data=b'I got the power!',
|
||||
fee_currency='LBC', fee_amount='1.0', fee_address=target_address
|
||||
fee_currency='LBC', fee_amount='1.0',
|
||||
fee_address=target_address, claim_address=target_address
|
||||
)
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='icanpay')
|
||||
await self.assertBalance(self.account, '9.925679')
|
||||
@@ -348,8 +405,8 @@ class FileCommands(CommandTestCase):
|
||||
|
||||
# restart the daemon and make sure the fee is still there
|
||||
|
||||
self.daemon.stream_manager.stop()
|
||||
await self.daemon.stream_manager.start()
|
||||
self.daemon.file_manager.stop()
|
||||
await self.daemon.file_manager.start()
|
||||
self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1)
|
||||
self.assertEqual((await self.daemon.jsonrpc_file_list())['items'][0].content_fee.raw, raw_content_fee)
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='icanpay')
|
||||
@@ -377,7 +434,8 @@ class FileCommands(CommandTestCase):
|
||||
|
||||
await self.stream_create(
|
||||
'somename', '0.5', data=b'Yes, please',
|
||||
fee_currency='LBC', fee_amount='1.0', fee_address=target_address
|
||||
fee_currency='LBC', fee_amount='1.0',
|
||||
fee_address=target_address, claim_address=target_address
|
||||
)
|
||||
self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='somename'))
|
||||
# Assert the fee and bid are subtracted
|
||||
|
||||
@@ -21,8 +21,8 @@ def get_random_bytes(n: int) -> bytes:
|
||||
|
||||
class RangeRequests(CommandTestCase):
|
||||
async def _restart_stream_manager(self):
|
||||
self.daemon.stream_manager.stop()
|
||||
await self.daemon.stream_manager.start()
|
||||
self.daemon.file_manager.stop()
|
||||
await self.daemon.file_manager.start()
|
||||
return
|
||||
|
||||
async def _setup_stream(self, data: bytes, save_blobs: bool = True, save_files: bool = False, file_size=0):
|
||||
|
||||
@@ -4,6 +4,7 @@ from lbry.testcase import CommandTestCase
|
||||
class EpicAdventuresOfChris45(CommandTestCase):
|
||||
|
||||
async def test_no_this_is_not_a_test_its_an_adventure(self):
|
||||
|
||||
# Chris45 is an avid user of LBRY and this is his story. It's fact and fiction
|
||||
# and everything in between; it's also the setting of some record setting
|
||||
# integration tests.
|
||||
@@ -194,3 +195,9 @@ class EpicAdventuresOfChris45(CommandTestCase):
|
||||
}},
|
||||
await self.resolve(uri)
|
||||
)
|
||||
|
||||
# He closes and opens the wallet server databases to see how horribly they break
|
||||
db = self.conductor.spv_node.server.db
|
||||
db.close()
|
||||
await db.open_for_serving()
|
||||
# They didn't! (error would be AssertionError: 276 vs 266 (264 counts) on startup)
|
||||
@@ -6,8 +6,9 @@ from lbry.conf import Config
|
||||
from lbry.extras import cli
|
||||
from lbry.extras.daemon.components import (
|
||||
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
|
||||
HASH_ANNOUNCER_COMPONENT, STREAM_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT
|
||||
HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT,
|
||||
LIBTORRENT_COMPONENT
|
||||
)
|
||||
from lbry.extras.daemon.daemon import Daemon
|
||||
|
||||
@@ -21,8 +22,9 @@ class CLIIntegrationTest(AsyncioTestCase):
|
||||
conf.api = 'localhost:5299'
|
||||
conf.components_to_skip = (
|
||||
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
|
||||
HASH_ANNOUNCER_COMPONENT, STREAM_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT
|
||||
HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT,
|
||||
LIBTORRENT_COMPONENT
|
||||
)
|
||||
Daemon.component_attributes = {}
|
||||
self.daemon = Daemon(conf)
|
||||
@@ -34,4 +36,4 @@ class CLIIntegrationTest(AsyncioTestCase):
|
||||
with contextlib.redirect_stdout(actual_output):
|
||||
cli.main(["--api", "localhost:5299", "status"])
|
||||
actual_output = actual_output.getvalue()
|
||||
self.assertIn("connection_status", actual_output)
|
||||
self.assertIn("connection_status", actual_output)
|
||||
@@ -1,8 +1,8 @@
|
||||
import re
|
||||
|
||||
import time
|
||||
import typing
|
||||
from math import ceil
|
||||
import secrets
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -30,6 +30,17 @@ class MockedCommentServer:
|
||||
'timestamp': None,
|
||||
'channel_url': None,
|
||||
'is_hidden': False,
|
||||
'is_pinned': False,
|
||||
}
|
||||
|
||||
REACT_SCHEMA = {
|
||||
'react_id': None,
|
||||
'comment_id': None,
|
||||
'reaction_type': None,
|
||||
'timestamp': None,
|
||||
'signature': None,
|
||||
'channel_id': None,
|
||||
'channel_name': None
|
||||
}
|
||||
|
||||
def __init__(self, port=2903):
|
||||
@@ -39,7 +50,9 @@ class MockedCommentServer:
|
||||
self.runner = None
|
||||
self.server = None
|
||||
self.comments = []
|
||||
self.comment_id = 0
|
||||
self.reacts = {}
|
||||
self.index = 0
|
||||
self.react_id = 0
|
||||
|
||||
@classmethod
|
||||
def _create_comment(cls, **kwargs):
|
||||
@@ -47,15 +60,22 @@ class MockedCommentServer:
|
||||
schema.update(**kwargs)
|
||||
return schema
|
||||
|
||||
@classmethod
|
||||
def _react(cls, **kwargs):
|
||||
schema = cls.REACT_SCHEMA.copy()
|
||||
schema.update(**kwargs)
|
||||
return schema
|
||||
|
||||
def create_comment(self, claim_id=None, parent_id=None, channel_name=None, channel_id=None, **kwargs):
|
||||
comment_id = self.comment_id
|
||||
comment_id = secrets.token_hex(64)
|
||||
channel_url = 'lbry://' + channel_name + '#' + channel_id if channel_id else None
|
||||
|
||||
if parent_id:
|
||||
claim_id = self.comments[self.get_comment_id(parent_id)]['claim_id']
|
||||
parent_comment = list(filter(lambda c: c['comment_id'] == parent_id, self.comments))[0]
|
||||
claim_id = parent_comment['claim_id']
|
||||
|
||||
comment = self._create_comment(
|
||||
comment_id=str(comment_id),
|
||||
comment_id=comment_id,
|
||||
channel_name=channel_name,
|
||||
channel_id=channel_id,
|
||||
channel_url=channel_url,
|
||||
@@ -65,15 +85,14 @@ class MockedCommentServer:
|
||||
**kwargs
|
||||
)
|
||||
self.comments.append(comment)
|
||||
self.comment_id += 1
|
||||
return self.clean(comment)
|
||||
|
||||
def abandon_comment(self, comment_id: int, channel_id: str, **kwargs):
|
||||
def abandon_comment(self, comment_id: str, channel_id: str, **kwargs):
|
||||
deleted = False
|
||||
comment_id = self.get_comment_id(comment_id)
|
||||
index = self.get_index_for_comment_id(comment_id)
|
||||
try:
|
||||
if self.comments[comment_id]['channel_id'] == channel_id:
|
||||
self.comments.pop(comment_id)
|
||||
if index >= 0:
|
||||
self.comments.pop(index)
|
||||
deleted = True
|
||||
finally:
|
||||
return {
|
||||
@@ -82,29 +101,43 @@ class MockedCommentServer:
|
||||
}
|
||||
}
|
||||
|
||||
def edit_comment(self, comment_id: typing.Union[str, int], comment: str, channel_id: str,
|
||||
def edit_comment(self, comment_id: str, comment: str, channel_id: str,
|
||||
channel_name: str, signature: str, signing_ts: str) -> dict:
|
||||
edited = False
|
||||
if self.credentials_are_valid(channel_id, channel_name, signature, signing_ts) \
|
||||
and self.is_valid_body(comment):
|
||||
comment_id = self.get_comment_id(comment_id)
|
||||
if self.comments[comment_id]['channel_id'] == channel_id:
|
||||
self.comments[comment_id].update({
|
||||
index = self.get_index_for_comment_id(comment_id)
|
||||
if self.comments[index]['channel_id'] == channel_id:
|
||||
self.comments[index].update({
|
||||
'comment': comment,
|
||||
'signature': signature,
|
||||
'signing_ts': signing_ts
|
||||
})
|
||||
edited = True
|
||||
|
||||
return self.comments[comment_id] if edited else None
|
||||
return self.comments[index] if edited else None
|
||||
|
||||
def hide_comment(self, comment_id: typing.Union[int, str], signing_ts: str, signature: str):
|
||||
comment_id = self.get_comment_id(comment_id)
|
||||
def hide_comment(self, comment_id: str, signing_ts: str, signature: str):
|
||||
if self.is_signable(signature, signing_ts):
|
||||
self.comments[comment_id]['is_hidden'] = True
|
||||
self.comments[self.get_index_for_comment_id(comment_id)]['is_hidden'] = True
|
||||
return True
|
||||
return False
|
||||
|
||||
def pin_comment(
|
||||
self,
|
||||
comment_id: str,
|
||||
channel_name: str, channel_id: str, remove: bool,
|
||||
signing_ts: str, signature: str
|
||||
):
|
||||
index = self.get_index_for_comment_id(comment_id)
|
||||
if self.is_signable(signature, signing_ts):
|
||||
if remove:
|
||||
self.comments[index]['is_pinned'] = False
|
||||
else:
|
||||
self.comments[index]['is_pinned'] = True
|
||||
return self.comments[index]
|
||||
return False
|
||||
|
||||
def hide_comments(self, pieces: list):
|
||||
hidden = []
|
||||
for p in pieces:
|
||||
@@ -117,38 +150,29 @@ class MockedCommentServer:
|
||||
'visible': list(comment_ids - set(hidden))
|
||||
}
|
||||
|
||||
def get_claim_comments(self, claim_id, page=1, page_size=50,**kwargs):
|
||||
def get_claim_comments(self, claim_id, hidden=False, visible=False, page=1, page_size=50,**kwargs):
|
||||
comments = list(filter(lambda c: c['claim_id'] == claim_id, self.comments))
|
||||
if hidden ^ visible:
|
||||
comments = list(filter(lambda c: c['is_hidden'] == hidden, comments))
|
||||
|
||||
return {
|
||||
'page': page,
|
||||
'page_size': page_size,
|
||||
'total_pages': ceil(len(comments)/page_size),
|
||||
'total_items': len(comments),
|
||||
'items': [self.clean(c) for c in (comments[::-1])[(page - 1) * page_size: page * page_size]],
|
||||
'has_hidden_comments': bool(list(filter(lambda x: x['is_hidden'], comments)))
|
||||
'has_hidden_comments': bool(list(filter(lambda x: x['is_hidden'], self.comments)))
|
||||
}
|
||||
|
||||
def get_claim_hidden_comments(self, claim_id, hidden=True, page=1, page_size=50):
|
||||
comments = list(filter(lambda c: c['claim_id'] == claim_id, self.comments))
|
||||
select_comments = list(filter(lambda c: c['is_hidden'] == hidden, comments))
|
||||
return {
|
||||
'page': page,
|
||||
'page_size': page_size,
|
||||
'total_pages': ceil(len(select_comments) / page_size),
|
||||
'total_items': len(select_comments),
|
||||
'items': [self.clean(c) for c in (select_comments[::-1])[(page - 1) * page_size: page * page_size]],
|
||||
'has_hidden_comments': bool(list(filter(lambda c: c['is_hidden'], comments)))
|
||||
}
|
||||
|
||||
def get_comment_channel_by_id(self, comment_id: int, **kwargs):
|
||||
comment = self.comments[self.get_comment_id(comment_id)]
|
||||
def get_comment_channel_by_id(self, comment_id: str, **kwargs):
|
||||
comment = self.comments[self.get_index_for_comment_id(comment_id)]
|
||||
return {
|
||||
'channel_id': comment['channel_id'],
|
||||
'channel_name': comment['channel_name'],
|
||||
}
|
||||
|
||||
def get_comments_by_id(self, comment_ids: list):
|
||||
comments = [self.comments[self.get_comment_id(cid)] for cid in comment_ids]
|
||||
comments = [self.comments[self.get_index_for_comment_id(cid)] for cid in comment_ids]
|
||||
return {
|
||||
'page': 1,
|
||||
'page_size': len(comment_ids),
|
||||
@@ -157,15 +181,99 @@ class MockedCommentServer:
|
||||
'has_hidden_comments': bool({c for c in comments if c['is_hidden']})
|
||||
}
|
||||
|
||||
def react(
|
||||
self,
|
||||
comment_ids=None,
|
||||
channel_name=None,
|
||||
channel_id=None,
|
||||
remove=None,
|
||||
clear_types=None,
|
||||
type=None,
|
||||
signing_ts=None,
|
||||
**kwargs
|
||||
):
|
||||
comment_batch = comment_ids.split(',')
|
||||
for item in comment_batch:
|
||||
react_id = self.react_id
|
||||
c_id = item.strip()
|
||||
reacts_for_comment_id = [r for r in list(self.reacts.values()) if r['comment_id'] == c_id]
|
||||
channels_reacts_for_comment_id = [r for r in reacts_for_comment_id if r['channel_id'] == channel_id]
|
||||
if remove:
|
||||
matching_react = None
|
||||
for reaction in channels_reacts_for_comment_id:
|
||||
if reaction['reaction_type'] == type:
|
||||
matching_react = reaction
|
||||
break
|
||||
if matching_react:
|
||||
self.reacts.pop(int(matching_react['react_id']), None)
|
||||
else:
|
||||
if clear_types:
|
||||
for r_type in clear_types.split(','):
|
||||
for reaction in channels_reacts_for_comment_id:
|
||||
if reaction['reaction_type'] == r_type:
|
||||
x = self.reacts.pop(int(reaction['react_id']), None)
|
||||
react = self._react(
|
||||
react_id=str(react_id),
|
||||
comment_id=str(c_id),
|
||||
channel_name=channel_name,
|
||||
channel_id=channel_id,
|
||||
reaction_type=type,
|
||||
timestamp=str(int(time.time())),
|
||||
**kwargs
|
||||
)
|
||||
self.reacts[self.react_id] = react
|
||||
self.react_id += 1
|
||||
self.clean(react)
|
||||
return True
|
||||
|
||||
def list_reacts(self, comment_ids, channel_id, channel_name, types=None, **kwargs):
|
||||
all_types = list(set([r['reaction_type'] for r in list(self.reacts.values())]))
|
||||
comment_id_list = comment_ids.split(',')
|
||||
# _reacts: {'a1': {'like': 0, 'dislike': 0}, 'a2': {'like': 0, 'dislike': 0}}
|
||||
own_reacts = {}
|
||||
other_reacts = {}
|
||||
|
||||
# for each comment_id
|
||||
# add comment_id: {} to own_reacts and other_reacts
|
||||
# for each react in own_reacts
|
||||
# for each react in other_reacts
|
||||
for cid in comment_id_list:
|
||||
own_reacts[cid] = {}
|
||||
other_reacts[cid] = {}
|
||||
|
||||
for r_type in all_types:
|
||||
own_reacts[cid][r_type] = 0
|
||||
other_reacts[cid][r_type] = 0
|
||||
# process that comment ids reactions for own and other categories
|
||||
reacts_for_comment = list(filter(lambda c: c['comment_id'] == cid, list(self.reacts.values())))
|
||||
if types:
|
||||
reacts_for_comment = list(filter(lambda c: c['reaction_type'] in types.split(','), reacts_for_comment))
|
||||
own_reacts_for_comment = list(filter(lambda c: c['channel_id'] == channel_id, reacts_for_comment))
|
||||
other_reacts_for_comment = list(filter(lambda c: c['channel_id'] != channel_id, reacts_for_comment))
|
||||
|
||||
if own_reacts_for_comment:
|
||||
for react in own_reacts_for_comment:
|
||||
own_reacts[cid][react['reaction_type']] += 1
|
||||
if other_reacts_for_comment:
|
||||
for react in other_reacts_for_comment:
|
||||
other_reacts[cid][react['reaction_type']] += 1
|
||||
|
||||
return {
|
||||
'my_reactions': own_reacts,
|
||||
'others_reactions': other_reacts,
|
||||
}
|
||||
|
||||
methods = {
|
||||
'get_claim_comments': get_claim_comments,
|
||||
'comment.List': get_claim_comments,
|
||||
'get_comments_by_id': get_comments_by_id,
|
||||
'create_comment': create_comment,
|
||||
'abandon_comment': abandon_comment,
|
||||
'get_channel_from_comment_id': get_comment_channel_by_id,
|
||||
'get_claim_hidden_comments': get_claim_hidden_comments,
|
||||
'hide_comments': hide_comments,
|
||||
'edit_comment': edit_comment,
|
||||
'comment.Create': create_comment,
|
||||
'comment.Abandon': abandon_comment,
|
||||
'comment.GetChannelFromCommentID': get_comment_channel_by_id,
|
||||
'comment.Hide': hide_comments,
|
||||
'comment.Pin': pin_comment,
|
||||
'comment.Edit': edit_comment,
|
||||
'reaction.React': react,
|
||||
'reaction.List': list_reacts,
|
||||
}
|
||||
|
||||
def process_json(self, body) -> dict:
|
||||
@@ -223,17 +331,18 @@ class MockedCommentServer:
|
||||
return 0 < len(comment) <= 2000
|
||||
|
||||
def is_valid_comment_id(self, comment_id: typing.Union[int, str]) -> bool:
|
||||
if isinstance(comment_id, str) and comment_id.isalnum():
|
||||
comment_id = int(comment_id)
|
||||
|
||||
if isinstance(comment_id, int):
|
||||
return 0 <= comment_id < len(self.comments)
|
||||
if isinstance(comment_id, str):
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_comment_id(self, cid: typing.Union[int, str, any]) -> int:
|
||||
if not self.is_valid_comment_id(cid):
|
||||
raise ValueError('Comment ID is Invalid')
|
||||
return cid if isinstance(cid, int) else int(cid)
|
||||
def get_comment_for_id(self, cid: str) -> dict:
|
||||
return list(filter(lambda c: c['comment_id'] == cid, self.comments))[0]
|
||||
|
||||
def get_index_for_comment_id(self, value: str):
|
||||
for i, dic in enumerate(self.comments):
|
||||
if dic['comment_id'] == value:
|
||||
return i
|
||||
return -1
|
||||
|
||||
@staticmethod
|
||||
def claim_id_is_valid(claim_id: str) -> bool:
|
||||
@@ -276,7 +385,6 @@ class MockedCommentServer:
|
||||
(parent_id is None or self.is_valid_comment_id(parent_id))
|
||||
|
||||
|
||||
|
||||
class CommentCommands(CommandTestCase):
|
||||
|
||||
async def asyncSetUp(self):
|
||||
@@ -304,7 +412,7 @@ class CommentCommands(CommandTestCase):
|
||||
self.assertEqual(stream['claim_id'], comments[0]['claim_id'])
|
||||
|
||||
channel2 = (await self.channel_create('@BuffettJimmy'))['outputs'][0]
|
||||
await self.daemon.jsonrpc_comment_create(
|
||||
comment2 = await self.daemon.jsonrpc_comment_create(
|
||||
claim_id=stream['claim_id'],
|
||||
channel_name=channel2['name'],
|
||||
comment='Let\'s all go to Margaritaville',
|
||||
@@ -427,6 +535,56 @@ class CommentCommands(CommandTestCase):
|
||||
for item in items_visible + items_hidden:
|
||||
self.assertIn(item, comments['items'])
|
||||
|
||||
async def test05_comment_pin(self):
|
||||
# wherein a bee makes a sick burn on moth's channel and moth pins it
|
||||
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
|
||||
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
|
||||
moth_id = moth['claim_id']
|
||||
moth_stream = await self.stream_create('Cool_Lamps_to_Sit_On', channel_id=moth_id)
|
||||
moth_claim_id = moth_stream['outputs'][0]['claim_id']
|
||||
|
||||
comment1 = await self.daemon.jsonrpc_comment_create(
|
||||
comment='Who on earth would want to sit around on a lamp all day',
|
||||
claim_id=moth_claim_id,
|
||||
channel_id=bee['claim_id']
|
||||
)
|
||||
self.assertFalse(comment1['is_pinned'])
|
||||
|
||||
comment2 = await self.daemon.jsonrpc_comment_create(
|
||||
comment='sick burn, brah',
|
||||
claim_id=moth_claim_id,
|
||||
channel_id=moth_id,
|
||||
)
|
||||
self.assertFalse(comment2['is_pinned'])
|
||||
|
||||
comments = await self.daemon.jsonrpc_comment_list(moth_claim_id)
|
||||
comments_items = comments['items']
|
||||
self.assertIn('is_pinned', comments_items[0])
|
||||
self.assertFalse(comments_items[0]['is_pinned'])
|
||||
|
||||
pinned = await self.daemon.jsonrpc_comment_pin(
|
||||
comment_id=comment1['comment_id'],
|
||||
channel_id=moth['claim_id'],
|
||||
channel_name=moth['name']
|
||||
)
|
||||
self.assertTrue(pinned['is_pinned'])
|
||||
|
||||
comments_after_pinning = await self.daemon.jsonrpc_comment_list(moth_claim_id)
|
||||
items_after_pin = comments_after_pinning['items']
|
||||
self.assertTrue(items_after_pin[1]['is_pinned'])
|
||||
|
||||
unpinned = await self.daemon.jsonrpc_comment_pin(
|
||||
comment_id=comment1['comment_id'],
|
||||
channel_id=moth['claim_id'],
|
||||
channel_name=moth['name'],
|
||||
remove=True
|
||||
)
|
||||
self.assertFalse(unpinned['is_pinned'])
|
||||
|
||||
comments_after_unpinning = await self.daemon.jsonrpc_comment_list(moth_claim_id)
|
||||
items_after_unpin = comments_after_unpinning['items']
|
||||
self.assertFalse(items_after_unpin[1]['is_pinned'])
|
||||
|
||||
async def test06_comment_list(self):
|
||||
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
|
||||
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
|
||||
@@ -450,7 +608,6 @@ class CommentCommands(CommandTestCase):
|
||||
channel_id=bee['claim_id']
|
||||
)
|
||||
all_comments = [other_comment, owner_comment, hidden_comment]
|
||||
|
||||
normal_list = await self.daemon.jsonrpc_comment_list(claim_id)
|
||||
self.assertEqual(
|
||||
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
|
||||
@@ -532,3 +689,124 @@ class CommentCommands(CommandTestCase):
|
||||
comment='If you see it and you mean then you know you have to go',
|
||||
comment_id=original_cid
|
||||
)
|
||||
|
||||
async def test06_reactions(self):
|
||||
# wherein a bee insulted by a rude moth accidentally
|
||||
# 1) likes an insult
|
||||
# 2) clicks dislike instead, testing "clear_types"
|
||||
# 3) then clicks dislike again, testing "remove"
|
||||
# calls from the point of view of the moth
|
||||
|
||||
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
|
||||
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
|
||||
stream = await self.stream_create('Cool_Lamps_to_Sit_On', channel_id=moth['claim_id'])
|
||||
claim_id = stream['outputs'][0]['claim_id']
|
||||
|
||||
first_comment = await self.daemon.jsonrpc_comment_create(
|
||||
comment='Go away you yellow freak',
|
||||
claim_id=claim_id,
|
||||
channel_id=moth['claim_id'],
|
||||
)
|
||||
second_comment = await self.daemon.jsonrpc_comment_create(
|
||||
comment='I got my swim trunks and my flippy-floppies',
|
||||
claim_id=claim_id,
|
||||
channel_id=bee['claim_id']
|
||||
)
|
||||
first_comment_id = first_comment['comment_id']
|
||||
second_comment_id = second_comment['comment_id']
|
||||
comment_list = await self.daemon.jsonrpc_comment_list(claim_id)
|
||||
self.assertEqual(
|
||||
{'items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages'},
|
||||
set(comment_list)
|
||||
)
|
||||
self.assertEqual(comment_list['total_items'], 2)
|
||||
|
||||
bee_like_reaction = await self.daemon.jsonrpc_comment_react(
|
||||
comment_ids=first_comment_id,
|
||||
channel_id=bee['claim_id'],
|
||||
channel_name=bee['name'],
|
||||
react_type='like',
|
||||
)
|
||||
|
||||
moth_like_reaction = await self.daemon.jsonrpc_comment_react(
|
||||
comment_ids=first_comment_id,
|
||||
channel_id=moth['claim_id'],
|
||||
channel_name=moth['name'],
|
||||
react_type='like',
|
||||
)
|
||||
reactions = await self.daemon.jsonrpc_comment_react_list(
|
||||
comment_ids=first_comment_id,
|
||||
channel_id=moth['claim_id'],
|
||||
channel_name=moth['name'],
|
||||
)
|
||||
# {'my_reactions': {'0': {'like': 1}}, 'others_reactions': {'0': {'like': 1}}}
|
||||
self.assertEqual(reactions['my_reactions'][first_comment_id]['like'], 1)
|
||||
self.assertEqual(reactions['others_reactions'][first_comment_id]['like'], 1)
|
||||
|
||||
bee_dislike_reaction = await self.daemon.jsonrpc_comment_react(
|
||||
comment_ids=first_comment_id,
|
||||
channel_id=bee['claim_id'],
|
||||
channel_name=bee['name'],
|
||||
react_type='dislike',
|
||||
clear_types='like',
|
||||
)
|
||||
|
||||
reactions_after_bee_dislikes = await self.daemon.jsonrpc_comment_react_list(
|
||||
comment_ids=first_comment_id,
|
||||
channel_id=moth['claim_id'],
|
||||
channel_name=moth['name'],
|
||||
)
|
||||
# {'my_reactions': {'0': {'like': 1, 'dislike', 0}}, 'others_reactions': {'0': {'like': 0, 'dislike': 1}}}
|
||||
self.assertEqual(reactions_after_bee_dislikes['my_reactions'][first_comment_id]['like'], 1)
|
||||
self.assertEqual(reactions_after_bee_dislikes['my_reactions'][first_comment_id]['dislike'], 0)
|
||||
self.assertEqual(reactions_after_bee_dislikes['others_reactions'][first_comment_id]['dislike'], 1)
|
||||
self.assertEqual(reactions_after_bee_dislikes['others_reactions'][first_comment_id]['like'], 0)
|
||||
|
||||
only_likes_after_bee_dislikes = await self.daemon.jsonrpc_comment_react_list(
|
||||
comment_ids=first_comment_id,
|
||||
channel_id=moth['claim_id'],
|
||||
channel_name=moth['name'],
|
||||
react_types='like',
|
||||
)
|
||||
|
||||
self.assertEqual(only_likes_after_bee_dislikes['my_reactions'][first_comment_id]['like'], 1)
|
||||
self.assertEqual(only_likes_after_bee_dislikes['my_reactions'][first_comment_id]['dislike'], 0)
|
||||
self.assertEqual(only_likes_after_bee_dislikes['others_reactions'][first_comment_id]['dislike'], 0)
|
||||
self.assertEqual(only_likes_after_bee_dislikes['others_reactions'][first_comment_id]['like'], 0)
|
||||
|
||||
bee_un_dislike_reaction = await self.daemon.jsonrpc_comment_react(
|
||||
comment_ids=first_comment_id,
|
||||
channel_id=bee['claim_id'],
|
||||
channel_name=bee['name'],
|
||||
remove=True,
|
||||
react_type='dislike',
|
||||
)
|
||||
|
||||
reactions_after_bee_absconds = await self.daemon.jsonrpc_comment_react_list(
|
||||
comment_ids=first_comment_id,
|
||||
channel_id=moth['claim_id'],
|
||||
channel_name=moth['name'],
|
||||
)
|
||||
|
||||
self.assertEqual(reactions_after_bee_absconds['my_reactions'][first_comment_id]['like'], 1)
|
||||
self.assertNotIn('dislike', reactions_after_bee_absconds['my_reactions'][first_comment_id])
|
||||
self.assertEqual(reactions_after_bee_absconds['others_reactions'][first_comment_id]['like'], 0)
|
||||
self.assertNotIn('dislike', reactions_after_bee_absconds['others_reactions'][first_comment_id])
|
||||
|
||||
bee_reacts_to_both_comments = await self.daemon.jsonrpc_comment_react(
|
||||
comment_ids=first_comment_id + ',' + second_comment_id,
|
||||
channel_id=bee['claim_id'],
|
||||
channel_name=bee['name'],
|
||||
react_type='frozen_tom',
|
||||
)
|
||||
|
||||
reactions_after_double_frozen_tom = await self.daemon.jsonrpc_comment_react_list(
|
||||
comment_ids=first_comment_id + ',' + second_comment_id,
|
||||
channel_id=moth['claim_id'],
|
||||
channel_name=moth['name'],
|
||||
)
|
||||
|
||||
self.assertEqual(reactions_after_double_frozen_tom['my_reactions'][first_comment_id]['like'], 1)
|
||||
self.assertNotIn('dislike', reactions_after_double_frozen_tom['my_reactions'][first_comment_id])
|
||||
self.assertEqual(reactions_after_double_frozen_tom['others_reactions'][first_comment_id]['frozen_tom'], 1)
|
||||
self.assertEqual(reactions_after_double_frozen_tom['others_reactions'][second_comment_id]['frozen_tom'], 1)
|
||||
Loaded 100 of 124 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user