Compare commits
355
Commits
android
...
v0.34.0rc2
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3b0ba3e534 | ||
|
|
8106eb1fa5 | ||
|
|
c9b2829227 | ||
|
|
4f2f8b16f3 | ||
|
|
2ab2b846b1 | ||
|
|
2df7284c29 | ||
|
|
ab5d71ff61 | ||
|
|
99b289f135 | ||
|
|
6db02e4362 | ||
|
|
98d8b7d463 | ||
|
|
96f1883fc4 | ||
|
|
a914de155a | ||
|
|
2dee0ff0bf | ||
|
|
993cb43e5b | ||
|
|
5d212a0f82 | ||
|
|
0cc602ca53 | ||
|
|
f390ff144d | ||
|
|
b266cde511 | ||
|
|
d0a1541d0a | ||
|
|
4cbde1b08f | ||
|
|
c6740d8d70 | ||
|
|
46358142ed | ||
|
|
f0f651fc77 | ||
|
|
5d428138ed | ||
|
|
8d9da61c7d | ||
|
|
c326a9382c | ||
|
|
5bdd87c904 | ||
|
|
ad134fe8bb | ||
|
|
2f3c616acd | ||
|
|
3caa2a5039 | ||
|
|
e38a0f7a95 | ||
|
|
9de3f3ff09 | ||
|
|
df8934d472 | ||
|
|
be4a96c1b3 | ||
|
|
d97d978961 | ||
|
|
f52e44b56a | ||
|
|
10d73fe480 | ||
|
|
4fd2d994f2 | ||
|
|
d95903d219 | ||
|
|
591e66bfbb | ||
|
|
e047748961 | ||
|
|
9f071bbe36 | ||
|
|
2d8ebe25ed | ||
|
|
7a96e742f2 | ||
|
|
13c7026b20 | ||
|
|
876a3c68dd | ||
|
|
573617d6b6 | ||
|
|
e8c9abb944 | ||
|
|
8cee627dfa | ||
|
|
ab4a5cee01 | ||
|
|
5bb0a07fae | ||
|
|
5ca594b9fd | ||
|
|
3910964df1 | ||
|
|
faa1935203 | ||
|
|
ba518e2a0e | ||
|
|
2a82ff7cc5 | ||
|
|
673b958020 | ||
|
|
3f4ed0a896 | ||
|
|
f49578cdbc | ||
|
|
b834f29634 | ||
|
|
61132312dd | ||
|
|
fcdd9295b4 | ||
|
|
edb5083a5e | ||
|
|
4e7d88311f | ||
|
|
aa40337ded | ||
|
|
690e2e7e0c | ||
|
|
0c67469321 | ||
|
|
fc64d8d1a8 | ||
|
|
7fe9513857 | ||
|
|
e76107c318 | ||
|
|
3868dea819 | ||
|
|
c52541197f | ||
|
|
03cd6b7200 | ||
|
|
6915a7a553 | ||
|
|
7ad29b4bfb | ||
|
|
4294bf742d | ||
|
|
57ad9f1c52 | ||
|
|
e260de59f1 | ||
|
|
154702e6dd | ||
|
|
d06d998c81 | ||
|
|
8aff860bfe | ||
|
|
d7ba90bbea | ||
|
|
59325ad8c1 | ||
|
|
1f72e27272 | ||
|
|
aa41b8b558 | ||
|
|
2496e7ed44 | ||
|
|
ee12985135 | ||
|
|
4c7f357441 | ||
|
|
8944a36291 | ||
|
|
cb5d428d12 | ||
|
|
f175e8826c | ||
|
|
38ed212c01 | ||
|
|
d035c13883 | ||
|
|
82ce2ccee6 | ||
|
|
2cf9b2dd9e | ||
|
|
fcbdaa336f | ||
|
|
3ae6137edb | ||
|
|
0fdd2ae178 | ||
|
|
8c5007fe07 | ||
|
|
d2150cf142 | ||
|
|
ce2bb22929 | ||
|
|
1ce3adb6a0 | ||
|
|
c2dc1f3c1d | ||
|
|
b36c22e2f4 | ||
|
|
71b66ab337 | ||
|
|
461c3c8408 | ||
|
|
03396c1993 | ||
|
|
fa7e9923f4 | ||
|
|
3f7e466c97 | ||
|
|
4483380352 | ||
|
|
4a0a6b0243 | ||
|
|
a003e794ba | ||
|
|
051fabc180 | ||
|
|
ca686041e7 | ||
|
|
fc450eb164 | ||
|
|
652d7cbd24 | ||
|
|
43b372c0fe | ||
|
|
eaacf7e034 | ||
|
|
576da1bc30 | ||
|
|
5c217f1544 | ||
|
|
cfbf0dbe0f | ||
|
|
fd081496c0 | ||
|
|
32b4405a4c | ||
|
|
0d558bd552 | ||
|
|
ca835f3f80 | ||
|
|
dbb6ba6241 | ||
|
|
a228d20137 | ||
|
|
f1b60e3ef2 | ||
|
|
39737c790f | ||
|
|
dba4a09d38 | ||
|
|
918122e7cf | ||
|
|
a49f479a5f | ||
|
|
320aabbd70 | ||
|
|
19fdcad5af | ||
|
|
1734e08297 | ||
|
|
9d89f9b834 | ||
|
|
56c41b2fea | ||
|
|
9c8593d88e | ||
|
|
85f41887fd | ||
|
|
8eede6dfc7 | ||
|
|
0d8c37a63b | ||
|
|
d2ba3fdcfb | ||
|
|
be5ab0f307 | ||
|
|
04f70d1f31 | ||
|
|
60ba07cc17 | ||
|
|
dbc8cfe5ae | ||
|
|
d91e8f5996 | ||
|
|
b6df2b9d28 | ||
|
|
5c1ab42277 | ||
|
|
bbdc96e762 | ||
|
|
fc0c949187 | ||
|
|
3fce8e4023 | ||
|
|
37bb765a2e | ||
|
|
21ca4f60f7 | ||
|
|
119d498bce | ||
|
|
8108aa39c1 | ||
|
|
0a068011d8 | ||
|
|
1193d2cea4 | ||
|
|
76ecb5d50d | ||
|
|
dbc1164562 | ||
|
|
e2170ec83d | ||
|
|
266e777655 | ||
|
|
68905d82b9 | ||
|
|
396c49f7c6 | ||
|
|
5e509c9fd6 | ||
|
|
e6b6a3f55e | ||
|
|
019af119d9 | ||
|
|
58da2f16c2 | ||
|
|
8aa50aa14f | ||
|
|
65c6c83b51 | ||
|
|
f497989871 | ||
|
|
5366779b1b | ||
|
|
a3f59f041a | ||
|
|
d560b0c1cd | ||
|
|
e94229121a | ||
|
|
097291a498 | ||
|
|
afdc44ca13 | ||
|
|
6732b302bb | ||
|
|
c9d54bb049 | ||
|
|
f65e70df2f | ||
|
|
4541e754dd | ||
|
|
9f89c62e8a | ||
|
|
6386815a29 | ||
|
|
0c93090d2c | ||
|
|
45404b6ad4 | ||
|
|
aeb2891baa | ||
|
|
8abfdc82b2 | ||
|
|
da9b4e317a | ||
|
|
e414cc5c48 | ||
|
|
939edb2f18 | ||
|
|
868110a6f2 | ||
|
|
869a8b712b | ||
|
|
9ecec52412 | ||
|
|
8b25acff6e | ||
|
|
e2b06677b5 | ||
|
|
af1619ebfb | ||
|
|
3352e0e4f4 | ||
|
|
141d68a2cd | ||
|
|
c06ec6cd69 | ||
|
|
1be5dce30e | ||
|
|
73da223ae1 | ||
|
|
2d7eb83518 | ||
|
|
ad03f91d24 | ||
|
|
5586a226c2 | ||
|
|
a616582733 | ||
|
|
43ac928f0b | ||
|
|
a121110743 | ||
|
|
8bc5bb4e14 | ||
|
|
3a15ff4bcd | ||
|
|
3508da4993 | ||
|
|
dfdc8eda00 | ||
|
|
58f6cb71c6 | ||
|
|
ae11c5bb4b | ||
|
|
f22b5da170 | ||
|
|
c1c6d5bc99 | ||
|
|
0e972ec2ae | ||
|
|
f8dc0f298e | ||
|
|
8b5803c945 | ||
|
|
1314f75219 | ||
|
|
be90ec51e4 | ||
|
|
e06c903731 | ||
|
|
76c8447a56 | ||
|
|
82b53e108c | ||
|
|
7c8ad0fe30 | ||
|
|
073ffdd049 | ||
|
|
7fe211d134 | ||
|
|
43d424978d | ||
|
|
a4ced4dbd2 | ||
|
|
c6b910a5f3 | ||
|
|
63d1ec3c28 | ||
|
|
c31e61586d | ||
|
|
7b8f42e6d0 | ||
|
|
2c275efa48 | ||
|
|
30846f932b | ||
|
|
6ead932ccb | ||
|
|
e9b58577dd | ||
|
|
f766dbefe0 | ||
|
|
f8a7d2ebe2 | ||
|
|
af43b21ef6 | ||
|
|
0685bc5689 | ||
|
|
8cda3143f2 | ||
|
|
5af351a234 | ||
|
|
b9934a7eb7 | ||
|
|
b82d2c3120 | ||
|
|
8e607fcc2a | ||
|
|
2bce7cecfa | ||
|
|
e141cc9e22 | ||
|
|
8883587cc7 | ||
|
|
c85991704d | ||
|
|
377442d9f0 | ||
|
|
ad30b7df6d | ||
|
|
f0b49fc153 | ||
|
|
6033399f05 | ||
|
|
ae1162f5b0 | ||
|
|
7d7d4c35c3 | ||
|
|
ad71a9a89d | ||
|
|
13b3e4ba8e | ||
|
|
43799168ba | ||
|
|
736a949cb5 | ||
|
|
11bb1cc7ee | ||
|
|
dbce49e567 | ||
|
|
dcd78bd86e | ||
|
|
e5af041627 | ||
|
|
73f4e954e5 | ||
|
|
0a027cf2de | ||
|
|
3f98b16800 | ||
|
|
bbadece75d | ||
|
|
8f5a8ab8ba | ||
|
|
91e581422c | ||
|
|
b54d7185a1 | ||
|
|
f2968aab22 | ||
|
|
4e27eaf8e6 | ||
|
|
ccbe8aa0eb | ||
|
|
478f79353a | ||
|
|
c3b0151943 | ||
|
|
ba818354af | ||
|
|
e96b75a0d0 | ||
|
|
efe4afd09e | ||
|
|
c7cb6822cc | ||
|
|
c75665d3f0 | ||
|
|
744375b2c0 | ||
|
|
c1b4a012ec | ||
|
|
40da5b6cb6 | ||
|
|
b36cf3e36c | ||
|
|
4e3b529aec | ||
|
|
0a08f445a4 | ||
|
|
3eac99c509 | ||
|
|
f5bf1463af | ||
|
|
ca4a55ef28 | ||
|
|
832be0758b | ||
|
|
25050fdeeb | ||
|
|
df0635103e | ||
|
|
dabd9cbe1e | ||
|
|
42d7fbdbb3 | ||
|
|
2336015f9a | ||
|
|
5a28128b69 | ||
|
|
31445c7797 | ||
|
|
16d0ff8376 | ||
|
|
87f5d1bd4a | ||
|
|
be6e8744b0 | ||
|
|
2e9acd1baa | ||
|
|
607ccefe90 | ||
|
|
df86937f4e | ||
|
|
957d0290df | ||
|
|
40b90ce79b | ||
|
|
aa7f0845cc | ||
|
|
6e69d306de | ||
|
|
d85d3dfaa4 | ||
|
|
2b035009ef | ||
|
|
f9fd62c214 | ||
|
|
e8a1953815 | ||
|
|
34776e7eb9 | ||
|
|
a4419f562f | ||
|
|
c0160ab4e9 | ||
|
|
e60e5b3919 | ||
|
|
3589cc9977 | ||
|
|
2e978c00b2 | ||
|
|
2335243de9 | ||
|
|
6ad68eabd9 | ||
|
|
ca5c638124 | ||
|
|
262b9a624b | ||
|
|
05915c540a | ||
|
|
6aef6a80b7 | ||
|
|
16efe9ba95 | ||
|
|
fcda9500fe | ||
|
|
64dec821b7 | ||
|
|
d5f6e58aad | ||
|
|
087b65522e | ||
|
|
5ee4b2173a | ||
|
|
3e1c3d1e4b | ||
|
|
21f2b2dcd7 | ||
|
|
ffc7fc0793 | ||
|
|
15dbe5e54e | ||
|
|
2e633c0ee6 | ||
|
|
933d58d49b | ||
|
|
8f7cf0b38f | ||
|
|
adf378b6e9 | ||
|
|
1a4f083e76 | ||
|
|
7bde09dcf8 | ||
|
|
4630fd38aa | ||
|
|
418def56ac | ||
|
|
cefe3eb520 | ||
|
|
64dffa306f | ||
|
|
7d33b4f1f3 | ||
|
|
f507d95198 | ||
|
|
2a7d80f8b4 | ||
|
|
305034ffb2 | ||
|
|
bfaebf0085 | ||
|
|
a94cf04e87 | ||
|
|
71f9f8ae9c | ||
|
|
330862e487 | ||
|
|
778d3826ab | ||
|
|
4a11cf007f | ||
|
|
f6e7700c4c | ||
|
|
573285a5bb |
No files matched your search
@@ -3,6 +3,7 @@
|
||||
/.tox
|
||||
/.idea
|
||||
/.coverage
|
||||
/lbry-venv
|
||||
|
||||
lbrynet.egg-info
|
||||
__pycache__
|
||||
|
||||
@@ -33,9 +33,7 @@ unsafe-load-any-extension=no
|
||||
# A comma-separated list of package or module names from where C extensions may
|
||||
# be loaded. Extensions are loading into the active Python interpreter and may
|
||||
# run arbitrary code
|
||||
extension-pkg-whitelist=
|
||||
miniupnpc,
|
||||
unqlite
|
||||
# extension-pkg-whitelist=
|
||||
|
||||
# Allow optimization of some AST trees. This will activate a peephole AST
|
||||
# optimizer, which will apply various small optimizations. For instance, it can
|
||||
|
||||
+1
-13
@@ -9,9 +9,7 @@ jobs:
|
||||
- stage: code quality
|
||||
name: "pylint lbrynet"
|
||||
install:
|
||||
- pip install astroid==2.0.4 aiohttp==3.4.4
|
||||
# newer astroid and aiohttp fails in pylint so we pre-install older version
|
||||
- pip install pylint
|
||||
- pip install astroid pylint
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install -e .
|
||||
script: pylint lbrynet
|
||||
@@ -106,16 +104,6 @@ jobs:
|
||||
before_install:
|
||||
- brew upgrade python || true
|
||||
- brew upgrade python || true
|
||||
install:
|
||||
- python3 --version
|
||||
- pip3 --version
|
||||
- pip3 install pyinstaller
|
||||
- git clone https://github.com/lbryio/torba.git --depth 1
|
||||
- sed -i -e "s/'plyvel',//" torba/setup.py
|
||||
- cd torba && pip3 install -e . && cd ..
|
||||
- python3 scripts/set_build.py
|
||||
- pip3 install -e .
|
||||
|
||||
- if: tag IS present
|
||||
stage: build
|
||||
name: "Wallet Server Docker Image - Tagged Release"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2015-2018 LBRY Inc
|
||||
Copyright (c) 2015-2019 LBRY Inc
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://travis-ci.org/lbryio/lbry) [](https://codecov.io/gh/lbryio/lbry)
|
||||
|
||||
LBRY is a decentralized peer-to-peer network providing distribution, discovery, and purchase of digital content (data). It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
|
||||
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
|
||||
|
||||
* Built on Python 3.7+ and `asyncio`.
|
||||
* DHT (Distributed Hash Table) implementation for finding peers ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
|
||||
* Blob exchange protocol for downloading content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
|
||||
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
|
||||
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
|
||||
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbrynet.schema](https://github.com/lbryio/lbry/tree/master/lbrynet/schema)).
|
||||
* Wallet implementation for the LBRY blockchain ([lbrynet.extras.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/wallet)).
|
||||
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbrynet.extras.daemon](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/daemon)).
|
||||
|
||||
+46
-262
@@ -76,7 +76,7 @@
|
||||
},
|
||||
{
|
||||
"name": "account_decrypt",
|
||||
"description": "Decrypt an encrypted account, this will remove the wallet password",
|
||||
"description": "Decrypt an encrypted account, this will remove the wallet password. The account must be unlocked to decrypt it",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "account_id",
|
||||
@@ -376,31 +376,6 @@
|
||||
],
|
||||
"returns": "(bool) true if successful"
|
||||
},
|
||||
{
|
||||
"name": "blob_availability",
|
||||
"description": "Get blob availability",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "blob_hash",
|
||||
"type": "str",
|
||||
"description": "check availability for this blob hash",
|
||||
"is_required": true
|
||||
},
|
||||
{
|
||||
"name": "search_timeout",
|
||||
"type": "int",
|
||||
"description": "how long to search for peers for the blob in the dht",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "blob_timeout",
|
||||
"type": "int",
|
||||
"description": "how long to try downloading from a peer",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(dict) {\n \"is_available\": <bool, true if blob is available from a peer from peer list>\n \"reachable_peers\": [\"<ip>:<port>\"],\n \"unreachable_peers\": [\"<ip>:<port>\"]\n }"
|
||||
},
|
||||
{
|
||||
"name": "blob_delete",
|
||||
"description": "Delete a blob",
|
||||
@@ -429,18 +404,6 @@
|
||||
"type": "int",
|
||||
"description": "timeout in number of seconds",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "encoding",
|
||||
"type": "str",
|
||||
"description": "by default no attempt at decoding is made, can be set to one of the following decoders: 'json'",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "payment_rate_manager",
|
||||
"type": "str",
|
||||
"description": "if not given the default payment rate manager will be used. supported alternative rate managers: 'only-free'",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(str) Success/Fail message or (dict) decoded data"
|
||||
@@ -820,12 +783,6 @@
|
||||
],
|
||||
"returns": "(dict) Dictionary containing the transaction information\n {\n \"hex\": (str) raw transaction,\n \"inputs\": (list) inputs(dict) used for the transaction,\n \"outputs\": (list) outputs(dict) for the transaction,\n \"total_fee\": (int) fee in dewies,\n \"total_input\": (int) total of inputs in dewies,\n \"total_output\": (int) total of outputs in dewies(input - fees),\n \"txid\": (str) txid of the transaction,\n }"
|
||||
},
|
||||
{
|
||||
"name": "commands",
|
||||
"description": "Return a list of available commands",
|
||||
"arguments": [],
|
||||
"returns": "(list) list of available commands"
|
||||
},
|
||||
{
|
||||
"name": "file_delete",
|
||||
"description": "Delete a LBRY file",
|
||||
@@ -975,14 +932,32 @@
|
||||
"description": "get file with matching claim name",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "blobs_in_stream<blobs_in_stream>",
|
||||
"type": "int",
|
||||
"description": "get file with matching blobs in stream",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "blobs_remaining",
|
||||
"type": "int",
|
||||
"description": "amount of remaining blobs to download",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "sort",
|
||||
"type": "str",
|
||||
"description": "sort by any property, like 'file_name' or 'metadata.author'; to specify direction append ',asc' or ',desc'",
|
||||
"description": "field to sort by (one of the above filter fields)",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "comparison",
|
||||
"type": "str",
|
||||
"description": "logical comparision, (eq | ne | g | ge | l | le)",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(list) List of files\n\n [\n {\n 'completed': (bool) true if download is completed,\n 'file_name': (str) name of file,\n 'download_directory': (str) download directory,\n 'points_paid': (float) credit paid to download file,\n 'stopped': (bool) true if download is stopped,\n 'stream_hash': (str) stream hash of file,\n 'stream_name': (str) stream name ,\n 'suggested_file_name': (str) suggested file name,\n 'sd_hash': (str) sd hash of file,\n 'download_path': (str) download path of file,\n 'mime_type': (str) mime type of file,\n 'key': (str) key attached to file,\n 'total_bytes': (int) file size in bytes,\n 'written_bytes': (int) written size in bytes,\n 'blobs_completed': (int) number of fully downloaded blobs,\n 'blobs_in_stream': (int) total blobs on stream,\n 'status': (str) downloader status\n 'claim_id': (str) None if claim is not found else the claim id,\n 'outpoint': (str) None if claim is not found else the tx and output,\n 'txid': (str) None if claim is not found else the transaction id,\n 'nout': (int) None if claim is not found else the transaction output index,\n 'metadata': (dict) None if claim is not found else the claim metadata,\n 'channel_claim_id': (str) None if claim is not found or not signed,\n 'channel_name': (str) None if claim is not found or not signed,\n 'claim_name': (str) None if claim is not found else the claim name\n },\n ]"
|
||||
"returns": "(list) List of files\n\n [\n {\n 'completed': (bool) true if download is completed,\n 'file_name': (str) name of file,\n 'download_directory': (str) download directory,\n 'points_paid': (float) credit paid to download file,\n 'stopped': (bool) true if download is stopped,\n 'stream_hash': (str) stream hash of file,\n 'stream_name': (str) stream name ,\n 'suggested_file_name': (str) suggested file name,\n 'sd_hash': (str) sd hash of file,\n 'download_path': (str) download path of file,\n 'mime_type': (str) mime type of file,\n 'key': (str) key attached to file,\n 'total_bytes_lower_bound': (int) lower bound file size in bytes,\n 'total_bytes': (int) file upper bound size in bytes,\n 'written_bytes': (int) written size in bytes,\n 'blobs_completed': (int) number of fully downloaded blobs,\n 'blobs_in_stream': (int) total blobs on stream,\n 'blobs_remaining': (int) total blobs remaining to download,\n 'status': (str) downloader status\n 'claim_id': (str) None if claim is not found else the claim id,\n 'txid': (str) None if claim is not found else the transaction id,\n 'nout': (int) None if claim is not found else the transaction output index,\n 'outpoint': (str) None if claim is not found else the tx and output,\n 'metadata': (dict) None if claim is not found else the claim metadata,\n 'channel_claim_id': (str) None if claim is not found or not signed,\n 'channel_name': (str) None if claim is not found or not signed,\n 'claim_name': (str) None if claim is not found else the claim name\n },\n ]\n}"
|
||||
},
|
||||
{
|
||||
"name": "file_reflect",
|
||||
@@ -1083,19 +1058,6 @@
|
||||
],
|
||||
"returns": "(dict) Dictionary containing information about the stream\n {\n 'completed': (bool) true if download is completed,\n 'file_name': (str) name of file,\n 'download_directory': (str) download directory,\n 'points_paid': (float) credit paid to download file,\n 'stopped': (bool) true if download is stopped,\n 'stream_hash': (str) stream hash of file,\n 'stream_name': (str) stream name ,\n 'suggested_file_name': (str) suggested file name,\n 'sd_hash': (str) sd hash of file,\n 'download_path': (str) download path of file,\n 'mime_type': (str) mime type of file,\n 'key': (str) key attached to file,\n 'total_bytes': (int) file size in bytes,\n 'written_bytes': (int) written size in bytes,\n 'blobs_completed': (int) number of fully downloaded blobs,\n 'blobs_in_stream': (int) total blobs on stream,\n 'status': (str) downloader status,\n 'claim_id': (str) claim id,\n 'outpoint': (str) claim outpoint string,\n 'txid': (str) claim txid,\n 'nout': (int) claim nout,\n 'metadata': (dict) claim metadata,\n 'channel_claim_id': (str) None if claim is not signed\n 'channel_name': (str) None if claim is not signed\n 'claim_name': (str) claim name\n }"
|
||||
},
|
||||
{
|
||||
"name": "help",
|
||||
"description": "Return a useful message for an API command",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "command",
|
||||
"type": "str",
|
||||
"description": "command to retrieve documentation for",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(str) Help message"
|
||||
},
|
||||
{
|
||||
"name": "peer_list",
|
||||
"description": "Get peers for blob hash",
|
||||
@@ -1107,31 +1069,18 @@
|
||||
"is_required": true
|
||||
},
|
||||
{
|
||||
"name": "timeout",
|
||||
"name": "search_bottom_out_limit",
|
||||
"type": "int",
|
||||
"description": "peer search timeout in seconds",
|
||||
"description": "the number of search probes in a row that don't find any new peers before giving up and returning",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(list) List of contact dictionaries {'host': <peer ip>, 'port': <peer port>, 'node_id': <peer node id>}"
|
||||
"returns": "(list) List of contact dictionaries {'address': <peer ip>, 'udp_port': <dht port>, 'tcp_port': <peer port>,\n 'node_id': <peer node id>}"
|
||||
},
|
||||
{
|
||||
"name": "peer_ping",
|
||||
"description": "Send a kademlia ping to the specified peer. If address and port are provided the peer is directly pinged,\nif not provided the peer is located first.",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "address",
|
||||
"type": "str",
|
||||
"description": "ip address of the peer",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "port",
|
||||
"type": "int",
|
||||
"description": "udp port of the peer",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"arguments": [],
|
||||
"returns": "(str) pong, or {'error': <error message>} if an error is encountered"
|
||||
},
|
||||
{
|
||||
@@ -1222,12 +1171,6 @@
|
||||
"description": "whether the content is nsfw",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "sources",
|
||||
"type": "str",
|
||||
"description": "{'lbry_sd_hash': sd_hash} specifies sd hash of file",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "channel_name",
|
||||
"type": "str",
|
||||
@@ -1276,53 +1219,22 @@
|
||||
},
|
||||
{
|
||||
"name": "resolve",
|
||||
"description": "Resolve given LBRY URIs",
|
||||
"description": "Get the claim that a URL refers to.",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "force",
|
||||
"type": "bool",
|
||||
"description": "force refresh and ignore cache",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "uri",
|
||||
"type": "str",
|
||||
"description": "uri to resolve",
|
||||
"is_required": true
|
||||
},
|
||||
{
|
||||
"name": "uris",
|
||||
"type": "list",
|
||||
"description": "uris to resolve",
|
||||
"name": "urls",
|
||||
"type": "str, list",
|
||||
"description": "one or more urls to resolve",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "Dictionary of results, keyed by uri\n '<uri>': {\n If a resolution error occurs:\n 'error': Error message\n\n If the uri resolves to a channel or a claim in a channel:\n 'certificate': {\n 'address': (str) claim address,\n 'amount': (float) claim amount,\n 'effective_amount': (float) claim amount including supports,\n 'claim_id': (str) claim id,\n 'claim_sequence': (int) claim sequence number,\n 'decoded_claim': (bool) whether or not the claim value was decoded,\n 'height': (int) claim height,\n 'depth': (int) claim depth,\n 'has_signature': (bool) included if decoded_claim\n 'name': (str) claim name,\n 'permanent_url': (str) permanent url of the certificate claim,\n 'supports: (list) list of supports [{'txid': (str) txid,\n 'nout': (int) nout,\n 'amount': (float) amount}],\n 'txid': (str) claim txid,\n 'nout': (str) claim nout,\n 'signature_is_valid': (bool), included if has_signature,\n 'value': ClaimDict if decoded, otherwise hex string\n }\n\n If the uri resolves to a channel:\n 'claims_in_channel': (int) number of claims in the channel,\n\n If the uri resolves to a claim:\n 'claim': {\n 'address': (str) claim address,\n 'amount': (float) claim amount,\n 'effective_amount': (float) claim amount including supports,\n 'claim_id': (str) claim id,\n 'claim_sequence': (int) claim sequence number,\n 'decoded_claim': (bool) whether or not the claim value was decoded,\n 'height': (int) claim height,\n 'depth': (int) claim depth,\n 'has_signature': (bool) included if decoded_claim\n 'name': (str) claim name,\n 'permanent_url': (str) permanent url of the claim,\n 'channel_name': (str) channel name if claim is in a channel\n 'supports: (list) list of supports [{'txid': (str) txid,\n 'nout': (int) nout,\n 'amount': (float) amount}]\n 'txid': (str) claim txid,\n 'nout': (str) claim nout,\n 'signature_is_valid': (bool), included if has_signature,\n 'value': ClaimDict if decoded, otherwise hex string\n }\n }"
|
||||
},
|
||||
{
|
||||
"name": "resolve_name",
|
||||
"description": "Resolve stream info from a LBRY name",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "name",
|
||||
"type": "str",
|
||||
"description": "the name to resolve",
|
||||
"is_required": true
|
||||
},
|
||||
{
|
||||
"name": "force",
|
||||
"type": "bool",
|
||||
"description": "force refresh and do not check cache",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(dict) Metadata dictionary from name claim, None if the name is not\n resolvable"
|
||||
"returns": "Dictionary of results, keyed by url\n '<url>': {\n If a resolution error occurs:\n 'error': Error message\n\n If the url resolves to a channel or a claim in a channel:\n 'certificate': {\n 'address': (str) claim address,\n 'amount': (float) claim amount,\n 'effective_amount': (float) claim amount including supports,\n 'claim_id': (str) claim id,\n 'claim_sequence': (int) claim sequence number,\n 'decoded_claim': (bool) whether or not the claim value was decoded,\n 'height': (int) claim height,\n 'depth': (int) claim depth,\n 'has_signature': (bool) included if decoded_claim\n 'name': (str) claim name,\n 'permanent_url': (str) permanent url of the certificate claim,\n 'supports: (list) list of supports [{'txid': (str) txid,\n 'nout': (int) nout,\n 'amount': (float) amount}],\n 'txid': (str) claim txid,\n 'nout': (str) claim nout,\n 'signature_is_valid': (bool), included if has_signature,\n 'value': ClaimDict if decoded, otherwise hex string\n }\n\n If the url resolves to a channel:\n 'claims_in_channel': (int) number of claims in the channel,\n\n If the url resolves to a claim:\n 'claim': {\n 'address': (str) claim address,\n 'amount': (float) claim amount,\n 'effective_amount': (float) claim amount including supports,\n 'claim_id': (str) claim id,\n 'claim_sequence': (int) claim sequence number,\n 'decoded_claim': (bool) whether or not the claim value was decoded,\n 'height': (int) claim height,\n 'depth': (int) claim depth,\n 'has_signature': (bool) included if decoded_claim\n 'name': (str) claim name,\n 'permanent_url': (str) permanent url of the claim,\n 'channel_name': (str) channel name if claim is in a channel\n 'supports: (list) list of supports [{'txid': (str) txid,\n 'nout': (int) nout,\n 'amount': (float) amount}]\n 'txid': (str) claim txid,\n 'nout': (str) claim nout,\n 'signature_is_valid': (bool), included if has_signature,\n 'value': ClaimDict if decoded, otherwise hex string\n }\n }"
|
||||
},
|
||||
{
|
||||
"name": "routing_table_get",
|
||||
"description": "Get DHT routing information",
|
||||
"arguments": [],
|
||||
"returns": "(dict) dictionary containing routing and contact information\n {\n \"buckets\": {\n <bucket index>: [\n {\n \"address\": (str) peer address,\n \"port\": (int) peer udp port\n \"node_id\": (str) peer node id,\n \"blobs\": (list) blob hashes announced by peer\n }\n ]\n },\n \"contacts\": (list) contact node ids,\n \"blob_hashes\": (list) all of the blob hashes stored by peers in the list of buckets,\n \"node_id\": (str) the local dht node id\n }"
|
||||
"returns": "(dict) dictionary containing routing and peer information\n {\n \"buckets\": {\n <bucket index>: [\n {\n \"address\": (str) peer address,\n \"udp_port\": (int) peer udp port,\n \"tcp_port\": (int) peer tcp port,\n \"node_id\": (str) peer node id,\n }\n ]\n },\n \"node_id\": (str) the local dht node id\n }"
|
||||
},
|
||||
{
|
||||
"name": "settings_get",
|
||||
@@ -1333,131 +1245,21 @@
|
||||
{
|
||||
"name": "settings_set",
|
||||
"description": "Set daemon settings",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "download_directory",
|
||||
"type": "str",
|
||||
"description": "path of download directory",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "data_rate",
|
||||
"type": "float",
|
||||
"description": "0.0001",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "download_timeout",
|
||||
"type": "int",
|
||||
"description": "180",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "peer_port",
|
||||
"type": "int",
|
||||
"description": "3333",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "max_key_fee",
|
||||
"type": "dict",
|
||||
"description": "maximum key fee for downloads, in the format: { 'currency': <currency_symbol>, 'amount': <amount> }. In the CLI, it must be an escaped JSON string Supported currency symbols: LBC, USD, BTC",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "disable_max_key_fee",
|
||||
"type": "bool",
|
||||
"description": "False",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "use_upnp",
|
||||
"type": "bool",
|
||||
"description": "True",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "run_reflector_server",
|
||||
"type": "bool",
|
||||
"description": "False",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "cache_time",
|
||||
"type": "int",
|
||||
"description": "150",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "reflect_uploads",
|
||||
"type": "bool",
|
||||
"description": "True",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "share_usage_data",
|
||||
"type": "bool",
|
||||
"description": "True",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "peer_search_timeout",
|
||||
"type": "int",
|
||||
"description": "3",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "sd_download_timeout",
|
||||
"type": "int",
|
||||
"description": "3",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "auto_renew_claim_height_delta",
|
||||
"type": "int",
|
||||
"description": "0 claims set to expire within this many blocks will be automatically renewed after startup (if set to 0, renews will not be made automatically)",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"arguments": [],
|
||||
"returns": "(dict) Updated dictionary of daemon settings"
|
||||
},
|
||||
{
|
||||
"name": "status",
|
||||
"description": "Get daemon status",
|
||||
"arguments": [],
|
||||
"returns": "(dict) lbrynet-daemon status\n {\n 'installation_id': (str) installation id - base58,\n 'is_running': (bool),\n 'is_first_run': bool,\n 'skipped_components': (list) [names of skipped components (str)],\n 'startup_status': { Does not include components which have been skipped\n 'database': (bool),\n 'wallet': (bool),\n 'session': (bool),\n 'dht': (bool),\n 'hash_announcer': (bool),\n 'stream_identifier': (bool),\n 'file_manager': (bool),\n 'blob_manager': (bool),\n 'blockchain_headers': (bool),\n 'peer_protocol_server': (bool),\n 'reflector': (bool),\n 'upnp': (bool),\n 'exchange_rate_manager': (bool),\n },\n 'connection_status': {\n 'code': (str) connection status code,\n 'message': (str) connection status message\n },\n 'blockchain_headers': {\n 'downloading_headers': (bool),\n 'download_progress': (float) 0-100.0\n },\n 'wallet': {\n 'blocks': (int) local blockchain height,\n 'blocks_behind': (int) remote_height - local_height,\n 'best_blockhash': (str) block hash of most recent block,\n 'is_encrypted': (bool),\n 'is_locked': (bool),\n },\n 'dht': {\n 'node_id': (str) lbry dht node id - hex encoded,\n 'peers_in_routing_table': (int) the number of peers in the routing table,\n },\n 'blob_manager': {\n 'finished_blobs': (int) number of finished blobs in the blob manager,\n },\n 'hash_announcer': {\n 'announce_queue_size': (int) number of blobs currently queued to be announced\n },\n 'file_manager': {\n 'managed_files': (int) count of files in the file manager,\n },\n 'upnp': {\n 'aioupnp_version': (str),\n 'redirects': {\n <TCP | UDP>: (int) external_port,\n },\n 'gateway': (str) manufacturer and model,\n 'dht_redirect_set': (bool),\n 'peer_redirect_set': (bool),\n 'external_ip': (str) external ip address,\n }\n }"
|
||||
"returns": "(dict) lbrynet-daemon status\n {\n 'installation_id': (str) installation id - base58,\n 'is_running': (bool),\n 'skipped_components': (list) [names of skipped components (str)],\n 'startup_status': { Does not include components which have been skipped\n 'blob_manager': (bool),\n 'blockchain_headers': (bool),\n 'database': (bool),\n 'dht': (bool),\n 'exchange_rate_manager': (bool),\n 'hash_announcer': (bool),\n 'peer_protocol_server': (bool),\n 'stream_manager': (bool),\n 'upnp': (bool),\n 'wallet': (bool),\n },\n 'connection_status': {\n 'code': (str) connection status code,\n 'message': (str) connection status message\n },\n 'blockchain_headers': {\n 'downloading_headers': (bool),\n 'download_progress': (float) 0-100.0\n },\n 'wallet': {\n 'blocks': (int) local blockchain height,\n 'blocks_behind': (int) remote_height - local_height,\n 'best_blockhash': (str) block hash of most recent block,\n 'is_encrypted': (bool),\n 'is_locked': (bool),\n },\n 'dht': {\n 'node_id': (str) lbry dht node id - hex encoded,\n 'peers_in_routing_table': (int) the number of peers in the routing table,\n },\n 'blob_manager': {\n 'finished_blobs': (int) number of finished blobs in the blob manager,\n },\n 'hash_announcer': {\n 'announce_queue_size': (int) number of blobs currently queued to be announced\n },\n 'stream_manager': {\n 'managed_files': (int) count of files in the stream manager,\n },\n 'upnp': {\n 'aioupnp_version': (str),\n 'redirects': {\n <TCP | UDP>: (int) external_port,\n },\n 'gateway': (str) manufacturer and model,\n 'dht_redirect_set': (bool),\n 'peer_redirect_set': (bool),\n 'external_ip': (str) external ip address,\n }\n }"
|
||||
},
|
||||
{
|
||||
"name": "stop",
|
||||
"description": "Stop lbrynet",
|
||||
"description": "Stop lbrynet API server.",
|
||||
"arguments": [],
|
||||
"returns": "(string) Shutdown message"
|
||||
},
|
||||
{
|
||||
"name": "stream_availability",
|
||||
"description": "Get stream availability for lbry uri",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "uri",
|
||||
"type": "str",
|
||||
"description": "check availability for this uri",
|
||||
"is_required": true
|
||||
},
|
||||
{
|
||||
"name": "search_timeout",
|
||||
"type": "int",
|
||||
"description": "how long to search for peers for the blob in the dht",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "blob_timeout",
|
||||
"type": "int",
|
||||
"description": "how long to try downloading from a peer",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(dict) {\n 'is_available': <bool>,\n 'did_decode': <bool>,\n 'did_resolve': <bool>,\n 'is_stream': <bool>,\n 'num_blobs_in_stream': <int>,\n 'sd_hash': <str>,\n 'sd_blob_availability': <dict> see `blob_availability`,\n 'head_blob_hash': <str>,\n 'head_blob_availability': <dict> see `blob_availability`,\n 'use_upnp': <bool>,\n 'upnp_redirect_is_set': <bool>,\n 'error': <None> | <str> error message\n }"
|
||||
},
|
||||
{
|
||||
"name": "stream_cost_estimate",
|
||||
"description": "Get estimated cost for a lbry stream",
|
||||
@@ -1467,12 +1269,6 @@
|
||||
"type": "str",
|
||||
"description": "uri to use",
|
||||
"is_required": true
|
||||
},
|
||||
{
|
||||
"name": "size",
|
||||
"type": "float",
|
||||
"description": "stream size in bytes. if provided an sd blob won't be downloaded.",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(float) Estimated cost in lbry credits, returns None if uri is not\n resolvable"
|
||||
@@ -1541,35 +1337,23 @@
|
||||
"returns": "(list) List of unspent transaction outputs (UTXOs)\n [\n {\n \"address\": (str) the output address\n \"amount\": (float) unspent amount\n \"height\": (int) block height\n \"is_claim\": (bool) is the tx a claim\n \"is_coinbase\": (bool) is the tx a coinbase tx\n \"is_support\": (bool) is the tx a support\n \"is_update\": (bool) is the tx an update\n \"nout\": (int) nout of the output\n \"txid\": (str) txid of the output\n },\n ...\n ]"
|
||||
},
|
||||
{
|
||||
"name": "version",
|
||||
"description": "Get lbry version information",
|
||||
"arguments": [],
|
||||
"returns": "(dict) Dictionary of lbry version information\n {\n 'build': (str) build type (e.g. \"dev\", \"rc\", \"release\"),\n 'ip': (str) remote ip, if available,\n 'lbrynet_version': (str) lbrynet_version,\n 'lbryum_version': (str) lbryum_version,\n 'lbryschema_version': (str) lbryschema_version,\n 'os_release': (str) os release string\n 'os_system': (str) os name\n 'platform': (str) platform string\n 'processor': (str) processor type,\n 'python_version': (str) python version,\n }"
|
||||
},
|
||||
{
|
||||
"name": "wallet_prefill_addresses",
|
||||
"description": "Create new UTXOs, each containing `amount` credits",
|
||||
"name": "utxo_release",
|
||||
"description": "When spending a UTXO it is locally locked to prevent double spends;\noccasionally this can result in a UTXO being locked which ultimately\ndid not get spent (failed to broadcast, spend transaction was not\naccepted by blockchain node, etc). This command releases the lock\non all UTXOs in your account.",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "no_broadcast",
|
||||
"type": "bool",
|
||||
"description": "whether to broadcast or not",
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "id of the account to query",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "num_addresses",
|
||||
"type": "int",
|
||||
"description": "num of addresses to create",
|
||||
"is_required": true
|
||||
},
|
||||
{
|
||||
"name": "amount",
|
||||
"type": "decimal",
|
||||
"description": "initial amount in each address",
|
||||
"is_required": true
|
||||
}
|
||||
],
|
||||
"returns": "(dict) the resulting transaction"
|
||||
"returns": "None"
|
||||
},
|
||||
{
|
||||
"name": "version",
|
||||
"description": "Get lbrynet API server version information",
|
||||
"arguments": [],
|
||||
"returns": "(dict) Dictionary of lbry version information\n {\n 'build': (str) build type (e.g. \"dev\", \"rc\", \"release\"),\n 'ip': (str) remote ip, if available,\n 'lbrynet_version': (str) lbrynet_version,\n 'lbryum_version': (str) lbryum_version,\n 'lbryschema_version': (str) lbryschema_version,\n 'os_release': (str) os release string\n 'os_system': (str) os name\n 'platform': (str) platform string\n 'processor': (str) processor type,\n 'python_version': (str) python version,\n }"
|
||||
},
|
||||
{
|
||||
"name": "wallet_send",
|
||||
|
||||
+2
-4
@@ -1409,7 +1409,7 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'claim_sequence': (int) claim sequence number (or -1 if unknown),
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -1954,7 +1954,6 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -1979,7 +1978,6 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -2586,4 +2584,4 @@ Returns:
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
</html>
|
||||
+2
-4
@@ -1339,7 +1339,7 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'claim_sequence': (int) claim sequence number (or -1 if unknown),
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -1786,7 +1786,6 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -1811,7 +1810,6 @@ Returns:
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -2312,4 +2310,4 @@ Returns:
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
</html>
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
__name__ = "lbrynet"
|
||||
__version__ = "0.31.0rc3"
|
||||
__version__ = "0.34.0rc2"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
+31
-21
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
import re
|
||||
import asyncio
|
||||
import binascii
|
||||
import logging
|
||||
@@ -17,8 +18,11 @@ from lbrynet.blob.writer import HashBlobWriter
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
_hexmatch = re.compile("^[a-f,0-9]+$")
|
||||
|
||||
|
||||
def is_valid_hashcharacter(char: str) -> bool:
|
||||
return char in "0123456789abcdef"
|
||||
return len(char) == 1 and _hexmatch.match(char)
|
||||
|
||||
|
||||
def is_valid_blobhash(blobhash: str) -> bool:
|
||||
@@ -29,7 +33,7 @@ def is_valid_blobhash(blobhash: str) -> bool:
|
||||
|
||||
@return: True/False
|
||||
"""
|
||||
return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
|
||||
return len(blobhash) == blobhash_length and _hexmatch.match(blobhash)
|
||||
|
||||
|
||||
def encrypt_blob_bytes(key: bytes, iv: bytes, unencrypted: bytes) -> typing.Tuple[bytes, str]:
|
||||
@@ -66,18 +70,22 @@ class BlobFile:
|
||||
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||
self.finished_writing = asyncio.Event(loop=loop)
|
||||
self.blob_write_lock = asyncio.Lock(loop=loop)
|
||||
if os.path.isfile(os.path.join(blob_dir, blob_hash)):
|
||||
length = length or int(os.stat(os.path.join(blob_dir, blob_hash)).st_size)
|
||||
if self.file_exists:
|
||||
length = int(os.stat(os.path.join(blob_dir, blob_hash)).st_size)
|
||||
self.length = length
|
||||
self.verified.set()
|
||||
self.finished_writing.set()
|
||||
self.saved_verified_blob = False
|
||||
self.blob_completed_callback = blob_completed_callback
|
||||
|
||||
@property
|
||||
def file_exists(self):
|
||||
return os.path.isfile(self.file_path)
|
||||
|
||||
def writer_finished(self, writer: HashBlobWriter):
|
||||
def callback(finished: asyncio.Future):
|
||||
try:
|
||||
error = finished.result()
|
||||
error = finished.exception()
|
||||
except Exception as err:
|
||||
error = err
|
||||
if writer in self.writers: # remove this download attempt
|
||||
@@ -86,37 +94,37 @@ class BlobFile:
|
||||
while self.writers:
|
||||
other = self.writers.pop()
|
||||
other.finished.cancel()
|
||||
t = self.loop.create_task(self.save_verified_blob(writer))
|
||||
t = self.loop.create_task(self.save_verified_blob(writer, finished.result()))
|
||||
t.add_done_callback(lambda *_: self.finished_writing.set())
|
||||
return
|
||||
if isinstance(error, (InvalidBlobHashError, InvalidDataError)):
|
||||
log.error("writer error downloading %s: %s", self.blob_hash[:8], str(error))
|
||||
log.debug("writer error downloading %s: %s", self.blob_hash[:8], str(error))
|
||||
elif not isinstance(error, (DownloadCancelledError, asyncio.CancelledError, asyncio.TimeoutError)):
|
||||
log.exception("something else")
|
||||
raise error
|
||||
return callback
|
||||
|
||||
async def save_verified_blob(self, writer):
|
||||
async def save_verified_blob(self, writer, verified_bytes: bytes):
|
||||
def _save_verified():
|
||||
# log.debug(f"write blob file {self.blob_hash[:8]} from {writer.peer.address}")
|
||||
if not self.saved_verified_blob and not os.path.isfile(self.file_path):
|
||||
if self.get_length() == len(writer.verified_bytes):
|
||||
if self.get_length() == len(verified_bytes):
|
||||
with open(self.file_path, 'wb') as write_handle:
|
||||
write_handle.write(writer.verified_bytes)
|
||||
write_handle.write(verified_bytes)
|
||||
self.saved_verified_blob = True
|
||||
else:
|
||||
raise Exception("length mismatch")
|
||||
|
||||
if self.verified.is_set():
|
||||
return
|
||||
async with self.blob_write_lock:
|
||||
if self.verified.is_set():
|
||||
return
|
||||
await self.loop.run_in_executor(None, _save_verified)
|
||||
if self.blob_completed_callback:
|
||||
await self.blob_completed_callback(self)
|
||||
self.verified.set()
|
||||
|
||||
def open_for_writing(self) -> HashBlobWriter:
|
||||
if os.path.exists(self.file_path):
|
||||
if self.file_exists:
|
||||
raise OSError(f"File already exists '{self.file_path}'")
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
|
||||
@@ -130,18 +138,20 @@ class BlobFile:
|
||||
"""
|
||||
|
||||
with open(self.file_path, 'rb') as handle:
|
||||
return await self.loop.sendfile(writer.transport, handle)
|
||||
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
|
||||
|
||||
async def close(self):
|
||||
def close(self):
|
||||
while self.writers:
|
||||
self.writers.pop().finished.cancel()
|
||||
|
||||
async def delete(self):
|
||||
await self.close()
|
||||
async with self.blob_write_lock:
|
||||
self.saved_verified_blob = False
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
def delete(self):
|
||||
self.close()
|
||||
self.saved_verified_blob = False
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
self.verified.clear()
|
||||
self.finished_writing.clear()
|
||||
self.length = None
|
||||
|
||||
def decrypt(self, key: bytes, iv: bytes) -> bytes:
|
||||
"""
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import os
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
from sqlite3 import IntegrityError
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from lbrynet.blob.blob_file import BlobFile, is_valid_blobhash
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
@@ -30,10 +30,21 @@ class BlobFileManager:
|
||||
self.blobs: typing.Dict[str, BlobFile] = {}
|
||||
|
||||
async def setup(self) -> bool:
|
||||
raw_blob_hashes = await self.get_all_verified_blobs()
|
||||
self.completed_blob_hashes.update(raw_blob_hashes)
|
||||
def get_files_in_blob_dir() -> typing.Set[str]:
|
||||
return {
|
||||
item.name for item in os.scandir(self.blob_dir) if is_valid_blobhash(item.name)
|
||||
}
|
||||
|
||||
in_blobfiles_dir = await self.loop.run_in_executor(None, get_files_in_blob_dir)
|
||||
self.completed_blob_hashes.update(await self.storage.sync_missing_blobs(in_blobfiles_dir))
|
||||
return True
|
||||
|
||||
def stop(self):
|
||||
while self.blobs:
|
||||
_, blob = self.blobs.popitem()
|
||||
blob.close()
|
||||
self.completed_blob_hashes.clear()
|
||||
|
||||
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
|
||||
if blob_hash in self.blobs:
|
||||
if length and self.blobs[blob_hash].length is None:
|
||||
@@ -52,38 +63,28 @@ class BlobFileManager:
|
||||
raise Exception("Blob has a length of 0")
|
||||
if blob.blob_hash not in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.add(blob.blob_hash)
|
||||
await self.storage.add_completed_blob(blob.blob_hash)
|
||||
await self.storage.add_completed_blob(blob.blob_hash, blob.length)
|
||||
|
||||
def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]:
|
||||
"""Returns of the blobhashes_to_check, which are valid"""
|
||||
blobs = [self.get_blob(b) for b in blob_hashes]
|
||||
return [blob.blob_hash for blob in blobs if blob.get_is_verified()]
|
||||
|
||||
async def set_should_announce(self, blob_hash: str, should_announce: bool):
|
||||
now = self.loop.time()
|
||||
return await self.storage.set_should_announce(blob_hash, now, should_announce)
|
||||
def delete_blob(self, blob_hash: str):
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise Exception("invalid blob hash to delete")
|
||||
|
||||
async def get_all_verified_blobs(self) -> typing.List[str]:
|
||||
blob_hashes = await self.storage.get_all_blob_hashes()
|
||||
return self.check_completed_blobs(blob_hashes)
|
||||
|
||||
async def delete_blobs(self, blob_hashes: typing.List[str]):
|
||||
bh_to_delete_from_db = []
|
||||
for blob_hash in blob_hashes:
|
||||
if not blob_hash:
|
||||
continue
|
||||
try:
|
||||
blob = self.get_blob(blob_hash)
|
||||
await blob.delete()
|
||||
bh_to_delete_from_db.append(blob_hash)
|
||||
except Exception as e:
|
||||
log.warning("Failed to delete blob file. Reason: %s", e)
|
||||
if blob_hash not in self.blobs:
|
||||
if os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
os.remove(os.path.join(self.blob_dir, blob_hash))
|
||||
else:
|
||||
self.blobs.pop(blob_hash).delete()
|
||||
if blob_hash in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.remove(blob_hash)
|
||||
if blob_hash in self.blobs:
|
||||
del self.blobs[blob_hash]
|
||||
try:
|
||||
await self.storage.delete_blobs_from_db(bh_to_delete_from_db)
|
||||
except IntegrityError as err:
|
||||
if str(err) != "FOREIGN KEY constraint failed":
|
||||
raise err
|
||||
|
||||
async def delete_blobs(self, blob_hashes: typing.List[str], delete_from_db: typing.Optional[bool] = True):
|
||||
for blob_hash in blob_hashes:
|
||||
self.delete_blob(blob_hash)
|
||||
|
||||
if delete_from_db:
|
||||
await self.storage.delete_blobs_from_db(blob_hashes)
|
||||
@@ -18,7 +18,6 @@ class HashBlobWriter:
|
||||
self.finished.add_done_callback(lambda *_: self.close_handle())
|
||||
self._hashsum = get_lbry_hash_obj()
|
||||
self.len_so_far = 0
|
||||
self.verified_bytes = b''
|
||||
|
||||
def __del__(self):
|
||||
if self.buffer is not None:
|
||||
@@ -46,7 +45,7 @@ class HashBlobWriter:
|
||||
self.len_so_far += len(data)
|
||||
if self.len_so_far > expected_length:
|
||||
self.close_handle()
|
||||
self.finished.set_result(InvalidDataError(
|
||||
self.finished.set_exception(InvalidDataError(
|
||||
f'Length so far is greater than the expected length. {self.len_so_far} to {expected_length}'
|
||||
))
|
||||
return
|
||||
@@ -55,15 +54,12 @@ class HashBlobWriter:
|
||||
blob_hash = self.calculate_blob_hash()
|
||||
if blob_hash != self.expected_blob_hash:
|
||||
self.close_handle()
|
||||
self.finished.set_result(InvalidBlobHashError(
|
||||
self.finished.set_exception(InvalidBlobHashError(
|
||||
f"blob hash is {blob_hash} vs expected {self.expected_blob_hash}"
|
||||
))
|
||||
return
|
||||
self.buffer.seek(0)
|
||||
self.verified_bytes = self.buffer.read()
|
||||
elif self.finished and not (self.finished.done() or self.finished.cancelled()):
|
||||
self.finished.set_result(self.buffer.getvalue())
|
||||
self.close_handle()
|
||||
if self.finished and not (self.finished.done() or self.finished.cancelled()):
|
||||
self.finished.set_result(None)
|
||||
|
||||
def close_handle(self):
|
||||
if self.buffer is not None:
|
||||
|
||||
+116
-80
@@ -1,6 +1,8 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import typing
|
||||
import binascii
|
||||
from lbrynet.error import InvalidBlobHashError, InvalidDataError
|
||||
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
@@ -19,150 +21,184 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
|
||||
self.writer: 'HashBlobWriter' = None
|
||||
self.blob: 'BlobFile' = None
|
||||
self.download_running = asyncio.Event(loop=self.loop)
|
||||
|
||||
self._blob_bytes_received = 0
|
||||
self._response_fut: asyncio.Future = None
|
||||
self._request_lock = asyncio.Lock(loop=self.loop)
|
||||
self.buf = b''
|
||||
|
||||
def handle_data_received(self, data: bytes):
|
||||
if self.transport.is_closing():
|
||||
if self._response_fut and not (self._response_fut.done() or self._response_fut.cancelled()):
|
||||
def data_received(self, data: bytes):
|
||||
log.debug("%s:%d -- got %s bytes -- %s bytes on buffer -- %s blob bytes received",
|
||||
self.peer_address, self.peer_port, len(data), len(self.buf), self._blob_bytes_received)
|
||||
if not self.transport or self.transport.is_closing():
|
||||
log.warning("transport closing, but got more bytes from %s:%i\n%s", self.peer_address, self.peer_port,
|
||||
binascii.hexlify(data))
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return
|
||||
if self._blob_bytes_received and not self.writer.closed():
|
||||
return self._write(data)
|
||||
|
||||
response = BlobResponse.deserialize(data)
|
||||
response = BlobResponse.deserialize(self.buf + data)
|
||||
if not response.responses and not self._response_fut.done():
|
||||
self.buf += data
|
||||
return
|
||||
else:
|
||||
self.buf = b''
|
||||
|
||||
if response.responses and self.blob:
|
||||
blob_response = response.get_blob_response()
|
||||
if blob_response and not blob_response.error and blob_response.blob_hash == self.blob.blob_hash:
|
||||
# set the expected length for the incoming blob if we didn't know it
|
||||
self.blob.set_length(blob_response.length)
|
||||
elif blob_response and not blob_response.error and self.blob.blob_hash != blob_response.blob_hash:
|
||||
# the server started sending a blob we didn't request
|
||||
log.warning("mismatch with self.blob %s", self.blob.blob_hash)
|
||||
return
|
||||
if response.responses:
|
||||
log.debug("got response from %s:%i <- %s", self.peer_address, self.peer_port, response.to_dict())
|
||||
# fire the Future with the response to our request
|
||||
self._response_fut.set_result(response)
|
||||
if response.blob_data and self.writer and not self.writer.closed():
|
||||
self._blob_bytes_received += len(response.blob_data)
|
||||
try:
|
||||
self.writer.write(response.blob_data)
|
||||
except IOError as err:
|
||||
log.error("error downloading blob: %s", err)
|
||||
# log.debug("got %i blob bytes from %s:%i", len(response.blob_data), self.peer_address, self.peer_port)
|
||||
# write blob bytes if we're writing a blob and have blob bytes to write
|
||||
self._write(response.blob_data)
|
||||
|
||||
def data_received(self, data):
|
||||
|
||||
def _write(self, data):
|
||||
if len(data) > (self.blob.get_length() - self._blob_bytes_received):
|
||||
data = data[:(self.blob.get_length() - self._blob_bytes_received)]
|
||||
log.warning("got more than asked from %s:%d, probable sendfile bug", self.peer_address, self.peer_port)
|
||||
else:
|
||||
data = data
|
||||
self._blob_bytes_received += len(data)
|
||||
try:
|
||||
return self.handle_data_received(data)
|
||||
except (asyncio.CancelledError, asyncio.TimeoutError) as err:
|
||||
self.writer.write(data)
|
||||
except IOError as err:
|
||||
log.error("error downloading blob from %s:%i: %s", self.peer_address, self.peer_port, err)
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.set_exception(err)
|
||||
except (asyncio.TimeoutError) as err: # TODO: is this needed?
|
||||
log.error("%s downloading blob from %s:%i", str(err), self.peer_address, self.peer_port)
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.set_exception(err)
|
||||
|
||||
async def _download_blob(self) -> typing.Tuple[bool, bool]:
|
||||
async def _download_blob(self) -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
"""
|
||||
:return: download success (bool), keep connection (bool)
|
||||
"""
|
||||
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
|
||||
try:
|
||||
self.transport.write(request.serialize())
|
||||
msg = request.serialize()
|
||||
log.debug("send request to %s:%i -> %s", self.peer_address, self.peer_port, msg.decode())
|
||||
self.transport.write(msg)
|
||||
response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop)
|
||||
availability_response = response.get_availability_response()
|
||||
price_response = response.get_price_response()
|
||||
blob_response = response.get_blob_response()
|
||||
if (not blob_response or blob_response.error) and\
|
||||
(not availability_response or not availability_response.available_blobs):
|
||||
log.warning("blob not in availability response")
|
||||
return False, True
|
||||
log.warning("%s not in availability response from %s:%i", self.blob.blob_hash, self.peer_address,
|
||||
self.peer_port)
|
||||
log.warning(response.to_dict())
|
||||
return self._blob_bytes_received, self.close()
|
||||
elif availability_response.available_blobs and \
|
||||
availability_response.available_blobs != [self.blob.blob_hash]:
|
||||
log.warning("blob availability response doesn't match our request")
|
||||
return False, False
|
||||
log.warning("blob availability response doesn't match our request from %s:%i",
|
||||
self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
if not price_response or price_response.blob_data_payment_rate != 'RATE_ACCEPTED':
|
||||
log.warning("data rate rejected")
|
||||
return False, False
|
||||
log.warning("data rate rejected by %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
if not blob_response or blob_response.error:
|
||||
log.warning("blob cant be downloaded from this peer")
|
||||
return False, True
|
||||
log.warning("blob cant be downloaded from %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
if not blob_response.error and blob_response.blob_hash != self.blob.blob_hash:
|
||||
log.warning("incoming blob hash mismatch")
|
||||
return False, False
|
||||
log.warning("incoming blob hash mismatch from %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
if self.blob.length is not None and self.blob.length != blob_response.length:
|
||||
log.warning("incoming blob unexpected length")
|
||||
return False, False
|
||||
log.warning("incoming blob unexpected length from %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
msg = f"downloading {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}," \
|
||||
f" timeout in {self.peer_timeout}"
|
||||
log.info(msg)
|
||||
log.debug(msg)
|
||||
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
|
||||
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
|
||||
await self.blob.finished_writing.wait()
|
||||
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}"
|
||||
log.info(msg)
|
||||
return True, True
|
||||
except asyncio.CancelledError:
|
||||
return False, True
|
||||
# await self.blob.finished_writing.wait() not necessary, but a dangerous change. TODO: is it needed?
|
||||
return self._blob_bytes_received, self.transport
|
||||
except asyncio.TimeoutError:
|
||||
return False, False
|
||||
finally:
|
||||
await self.close()
|
||||
return self._blob_bytes_received, self.close()
|
||||
except (InvalidBlobHashError, InvalidDataError):
|
||||
log.warning("invalid blob from %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
|
||||
async def close(self):
|
||||
def close(self):
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
if self.writer and not self.writer.closed():
|
||||
self.writer.close_handle()
|
||||
if self.blob:
|
||||
await self.blob.close()
|
||||
self.download_running.clear()
|
||||
self._response_fut = None
|
||||
self.writer = None
|
||||
self.blob = None
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
self.transport = None
|
||||
self.buf = b''
|
||||
|
||||
async def download_blob(self, blob: 'BlobFile') -> typing.Tuple[bool, bool]:
|
||||
if blob.get_is_verified():
|
||||
return False, True
|
||||
async with self._request_lock:
|
||||
try:
|
||||
if self.download_running.is_set():
|
||||
log.info("wait for download already running")
|
||||
await self.download_running.wait()
|
||||
self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0
|
||||
self.download_running.set()
|
||||
self._response_fut = asyncio.Future(loop=self.loop)
|
||||
return await self._download_blob()
|
||||
except OSError:
|
||||
log.error("race happened")
|
||||
# i'm not sure how to fix this race condition - jack
|
||||
return False, True
|
||||
except asyncio.TimeoutError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return False, False
|
||||
except asyncio.CancelledError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return False, True
|
||||
async def download_blob(self, blob: 'BlobFile') -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
if blob.get_is_verified() or blob.file_exists or blob.blob_write_lock.locked():
|
||||
return 0, self.transport
|
||||
try:
|
||||
self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0
|
||||
self._response_fut = asyncio.Future(loop=self.loop)
|
||||
return await self._download_blob()
|
||||
except OSError as e:
|
||||
log.error("race happened downloading from %s:%i", self.peer_address, self.peer_port)
|
||||
# i'm not sure how to fix this race condition - jack
|
||||
log.exception(e)
|
||||
return self._blob_bytes_received, self.transport
|
||||
except asyncio.TimeoutError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
self.close()
|
||||
return self._blob_bytes_received, None
|
||||
except asyncio.CancelledError:
|
||||
self.close()
|
||||
raise
|
||||
|
||||
def connection_made(self, transport: asyncio.Transport):
|
||||
self.transport = transport
|
||||
self.peer_address, self.peer_port = self.transport.get_extra_info('peername')
|
||||
# log.info("connection made to %s: %s", self.peer_address, transport)
|
||||
log.debug("connection made to %s:%i", self.peer_address, self.peer_port)
|
||||
|
||||
def connection_lost(self, reason):
|
||||
# log.info("connection lost to %s (reason: %s)", self.peer_address, reason)
|
||||
self.transport = None
|
||||
self.loop.create_task(self.close())
|
||||
log.debug("connection lost to %s:%i (reason: %s, %s)", self.peer_address, self.peer_port, str(reason),
|
||||
str(type(reason)))
|
||||
self.close()
|
||||
|
||||
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', protocol: 'BlobExchangeClientProtocol',
|
||||
address: str, tcp_port: int, peer_connect_timeout: float) -> typing.Tuple[bool, bool]:
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', address: str, tcp_port: int,
|
||||
peer_connect_timeout: float, blob_download_timeout: float,
|
||||
connected_transport: asyncio.Transport = None)\
|
||||
-> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
"""
|
||||
Returns [<downloaded blob>, <keep connection>]
|
||||
"""
|
||||
if blob.get_is_verified():
|
||||
return False, True
|
||||
if blob.get_is_verified():
|
||||
log.info("already verified")
|
||||
return False, True
|
||||
|
||||
protocol = BlobExchangeClientProtocol(loop, blob_download_timeout)
|
||||
if connected_transport and not connected_transport.is_closing():
|
||||
connected_transport.set_protocol(protocol)
|
||||
protocol.connection_made(connected_transport)
|
||||
log.debug("reusing connection for %s:%d", address, tcp_port)
|
||||
else:
|
||||
connected_transport = None
|
||||
try:
|
||||
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
|
||||
peer_connect_timeout, loop=loop)
|
||||
if not connected_transport:
|
||||
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
|
||||
peer_connect_timeout, loop=loop)
|
||||
if blob.get_is_verified() or blob.file_exists:
|
||||
# file exists but not verified means someone is writing right now, give it time, come back later
|
||||
return 0, connected_transport
|
||||
return await protocol.download_blob(blob)
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError, ConnectionRefusedError, ConnectionAbortedError, OSError):
|
||||
return False, False
|
||||
except (asyncio.TimeoutError, ConnectionRefusedError, ConnectionAbortedError, OSError):
|
||||
return 0, None
|
||||
+123
-111
@@ -1,10 +1,10 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet import conf
|
||||
from lbrynet.utils import drain_tasks
|
||||
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
|
||||
from lbrynet.blob_exchange.client import request_blob
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
@@ -13,120 +13,132 @@ if typing.TYPE_CHECKING:
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def drain_into(a: list, b: list):
|
||||
while a:
|
||||
b.append(a.pop())
|
||||
|
||||
|
||||
class BlobDownloader: # TODO: refactor to be the base class used by StreamDownloader
|
||||
"""A single blob downloader"""
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', config: conf.Config):
|
||||
class BlobDownloader:
|
||||
BAN_TIME = 10.0 # fixme: when connection manager gets implemented, move it out from here
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager',
|
||||
peer_queue: asyncio.Queue):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
self.blob_manager = blob_manager
|
||||
self.new_peer_event = asyncio.Event(loop=self.loop)
|
||||
self.active_connections: typing.Dict['KademliaPeer', BlobExchangeClientProtocol] = {}
|
||||
self.running_download_requests: typing.List[asyncio.Task] = []
|
||||
self.requested_from: typing.Dict[str, typing.Dict['KademliaPeer', asyncio.Task]] = {}
|
||||
self.lock = asyncio.Lock(loop=self.loop)
|
||||
self.blob: 'BlobFile' = None
|
||||
self.blob_queue = asyncio.Queue(loop=self.loop)
|
||||
self.peer_queue = peer_queue
|
||||
self.active_connections: typing.Dict['KademliaPeer', asyncio.Task] = {} # active request_blob calls
|
||||
self.ignored: typing.Dict['KademliaPeer', int] = {}
|
||||
self.scores: typing.Dict['KademliaPeer', int] = {}
|
||||
self.connections: typing.Dict['KademliaPeer', asyncio.Transport] = {}
|
||||
self.time_since_last_blob = loop.time()
|
||||
|
||||
self.blob_download_timeout = config.get('blob_download_timeout')
|
||||
self.peer_connect_timeout = config.get('peer_connect_timeout')
|
||||
self.max_connections = config.get('max_connections_per_stream')
|
||||
def should_race_continue(self, blob: 'BlobFile'):
|
||||
if len(self.active_connections) >= self.config.max_connections_per_download:
|
||||
return False
|
||||
# if a peer won 3 or more blob races and is active as a downloader, stop the race so bandwidth improves
|
||||
# the safe net side is that any failure will reset the peer score, triggering the race back
|
||||
# TODO: this is a good idea for low bandwidth, but doesnt play nice on high bandwidth
|
||||
# for peer, task in self.active_connections.items():
|
||||
# if self.scores.get(peer, 0) >= 0 and self.rounds_won.get(peer, 0) >= 3 and not task.done():
|
||||
# return False
|
||||
return not (blob.get_is_verified() or blob.file_exists)
|
||||
|
||||
async def _request_blob(self, peer: 'KademliaPeer'):
|
||||
if self.blob.get_is_verified():
|
||||
log.info("already verified")
|
||||
async def request_blob_from_peer(self, blob: 'BlobFile', peer: 'KademliaPeer'):
|
||||
if blob.get_is_verified():
|
||||
return
|
||||
if peer not in self.active_connections:
|
||||
log.warning("not active, adding: %s", str(peer))
|
||||
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.blob_download_timeout)
|
||||
protocol = self.active_connections[peer]
|
||||
success, keep_connection = await request_blob(self.loop, self.blob, protocol, peer.address, peer.tcp_port,
|
||||
self.peer_connect_timeout)
|
||||
await protocol.close()
|
||||
if not keep_connection:
|
||||
log.info("drop peer %s:%i", peer.address, peer.tcp_port)
|
||||
if peer in self.active_connections:
|
||||
async with self.lock:
|
||||
del self.active_connections[peer]
|
||||
return
|
||||
log.info("keep peer %s:%i", peer.address, peer.tcp_port)
|
||||
self.scores[peer] = self.scores.get(peer, 0) - 1 # starts losing score, to account for cancelled ones
|
||||
transport = self.connections.get(peer)
|
||||
start = self.loop.time()
|
||||
bytes_received, transport = await request_blob(
|
||||
self.loop, blob, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
|
||||
self.config.blob_download_timeout, connected_transport=transport
|
||||
)
|
||||
if bytes_received == blob.get_length():
|
||||
self.time_since_last_blob = self.loop.time()
|
||||
if not transport and peer not in self.ignored:
|
||||
self.ignored[peer] = self.loop.time()
|
||||
log.debug("drop peer %s:%i", peer.address, peer.tcp_port)
|
||||
if peer in self.connections:
|
||||
del self.connections[peer]
|
||||
elif transport:
|
||||
log.debug("keep peer %s:%i", peer.address, peer.tcp_port)
|
||||
self.connections[peer] = transport
|
||||
rough_speed = (bytes_received / (self.loop.time() - start)) if bytes_received else 0
|
||||
self.scores[peer] = rough_speed
|
||||
|
||||
def _update_requests(self):
|
||||
self.new_peer_event.clear()
|
||||
if self.blob.blob_hash not in self.requested_from:
|
||||
self.requested_from[self.blob.blob_hash] = {}
|
||||
to_add = []
|
||||
for peer in self.active_connections.keys():
|
||||
if peer not in self.requested_from[self.blob.blob_hash] and peer not in to_add:
|
||||
to_add.append(peer)
|
||||
if to_add or self.running_download_requests:
|
||||
log.info("adding download probes for %i peers to %i already active",
|
||||
min(len(to_add), 8 - len(self.running_download_requests)),
|
||||
len(self.running_download_requests))
|
||||
else:
|
||||
log.info("downloader idle...")
|
||||
for peer in to_add:
|
||||
if len(self.running_download_requests) >= 8:
|
||||
break
|
||||
task = self.loop.create_task(self._request_blob(peer))
|
||||
self.requested_from[self.blob.blob_hash][peer] = task
|
||||
self.running_download_requests.append(task)
|
||||
|
||||
def _add_peer_protocols(self, peers: typing.List['KademliaPeer']):
|
||||
added = 0
|
||||
for peer in peers:
|
||||
if peer not in self.active_connections:
|
||||
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.blob_download_timeout)
|
||||
added += 1
|
||||
if added:
|
||||
if not self.new_peer_event.is_set():
|
||||
log.info("added %i new peers", len(peers))
|
||||
self.new_peer_event.set()
|
||||
|
||||
async def _accumulate_connections(self, node: 'Node'):
|
||||
async def new_peer_or_finished(self, blob: 'BlobFile'):
|
||||
async def get_and_re_add_peers():
|
||||
try:
|
||||
new_peers = await asyncio.wait_for(self.peer_queue.get(), timeout=1.0)
|
||||
self.peer_queue.put_nowait(new_peers)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
tasks = [self.loop.create_task(get_and_re_add_peers()), self.loop.create_task(blob.verified.wait())]
|
||||
active_tasks = list(self.active_connections.values())
|
||||
try:
|
||||
async with node.stream_peer_search_junction(self.blob_queue) as search_junction:
|
||||
async for peers in search_junction:
|
||||
if not isinstance(peers, list): # TODO: what's up with this?
|
||||
log.error("not a list: %s %s", peers, str(type(peers)))
|
||||
else:
|
||||
self._add_peer_protocols(peers)
|
||||
return
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
async def get_blob(self, blob_hash: str, node: 'Node') -> 'BlobFile':
|
||||
self.blob = self.blob_manager.get_blob(blob_hash)
|
||||
if self.blob.get_is_verified():
|
||||
return self.blob
|
||||
accumulator = self.loop.create_task(self._accumulate_connections(node))
|
||||
self.blob_queue.put_nowait(blob_hash)
|
||||
try:
|
||||
while not self.blob.get_is_verified():
|
||||
if len(self.running_download_requests) < self.max_connections:
|
||||
self._update_requests()
|
||||
|
||||
# drain the tasks into a temporary list
|
||||
download_tasks = []
|
||||
drain_into(self.running_download_requests, download_tasks)
|
||||
got_new_peer = self.loop.create_task(self.new_peer_event.wait())
|
||||
|
||||
# wait for a new peer to be added or for a download attempt to finish
|
||||
await asyncio.wait([got_new_peer] + download_tasks, return_when='FIRST_COMPLETED',
|
||||
loop=self.loop)
|
||||
if got_new_peer and not got_new_peer.done():
|
||||
got_new_peer.cancel()
|
||||
if self.blob.get_is_verified():
|
||||
if got_new_peer and not got_new_peer.done():
|
||||
got_new_peer.cancel()
|
||||
drain_tasks(download_tasks)
|
||||
return self.blob
|
||||
except asyncio.CancelledError:
|
||||
drain_tasks(self.running_download_requests)
|
||||
raise
|
||||
await asyncio.wait(tasks + active_tasks, loop=self.loop, return_when='FIRST_COMPLETED')
|
||||
finally:
|
||||
if accumulator and not accumulator.done():
|
||||
accumulator.cancel()
|
||||
drain_tasks(tasks)
|
||||
|
||||
def cleanup_active(self):
|
||||
to_remove = [peer for (peer, task) in self.active_connections.items() if task.done()]
|
||||
for peer in to_remove:
|
||||
del self.active_connections[peer]
|
||||
|
||||
def clearbanned(self):
|
||||
now = self.loop.time()
|
||||
if now - self.time_since_last_blob > 60.0:
|
||||
return
|
||||
forgiven = [banned_peer for banned_peer, when in self.ignored.items() if now - when > self.BAN_TIME]
|
||||
self.peer_queue.put_nowait(forgiven)
|
||||
for banned_peer in forgiven:
|
||||
self.ignored.pop(banned_peer)
|
||||
|
||||
async def download_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
|
||||
blob = self.blob_manager.get_blob(blob_hash, length)
|
||||
if blob.get_is_verified():
|
||||
return blob
|
||||
try:
|
||||
while not blob.get_is_verified():
|
||||
batch: typing.List['KademliaPeer'] = []
|
||||
while not self.peer_queue.empty():
|
||||
batch.extend(self.peer_queue.get_nowait())
|
||||
batch.sort(key=lambda peer: self.scores.get(peer, 0), reverse=True)
|
||||
log.debug(
|
||||
"running, %d peers, %d ignored, %d active",
|
||||
len(batch), len(self.ignored), len(self.active_connections)
|
||||
)
|
||||
for peer in batch:
|
||||
if not self.should_race_continue(blob):
|
||||
break
|
||||
if peer not in self.active_connections and peer not in self.ignored:
|
||||
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
|
||||
t = self.loop.create_task(self.request_blob_from_peer(blob, peer))
|
||||
self.active_connections[peer] = t
|
||||
await self.new_peer_or_finished(blob)
|
||||
self.cleanup_active()
|
||||
if batch:
|
||||
self.peer_queue.put_nowait(set(batch).difference(self.ignored))
|
||||
else:
|
||||
self.clearbanned()
|
||||
blob.close()
|
||||
log.debug("downloaded %s", blob_hash[:8])
|
||||
return blob
|
||||
finally:
|
||||
while self.active_connections:
|
||||
self.active_connections.popitem()[1].cancel()
|
||||
|
||||
def close(self):
|
||||
self.scores.clear()
|
||||
self.ignored.clear()
|
||||
for transport in self.connections.values():
|
||||
transport.close()
|
||||
|
||||
|
||||
async def download_blob(loop, config: 'Config', blob_manager: 'BlobFileManager', node: 'Node',
|
||||
blob_hash: str) -> 'BlobFile':
|
||||
search_queue = asyncio.Queue(loop=loop, maxsize=config.max_connections_per_download)
|
||||
search_queue.put_nowait(blob_hash)
|
||||
peer_queue, accumulate_task = node.accumulate_peers(search_queue)
|
||||
downloader = BlobDownloader(loop, config, blob_manager, peer_queue)
|
||||
try:
|
||||
return await downloader.download_blob(blob_hash)
|
||||
finally:
|
||||
if accumulate_task and not accumulate_task.done():
|
||||
accumulate_task.cancel()
|
||||
downloader.close()
|
||||
@@ -146,32 +146,26 @@ def _parse_blob_response(response_msg: bytes) -> typing.Tuple[typing.Optional[ty
|
||||
# <blob bytes>
|
||||
# <json><blob bytes>
|
||||
|
||||
extra_data = b''
|
||||
response = None
|
||||
curr_pos = 0
|
||||
while True:
|
||||
next_close_paren = response_msg.find(b'}', curr_pos)
|
||||
if next_close_paren == -1:
|
||||
break
|
||||
return None, response_msg
|
||||
curr_pos = next_close_paren + 1
|
||||
try:
|
||||
response = json.loads(response_msg[:curr_pos])
|
||||
if not isinstance(response, dict):
|
||||
raise ValueError()
|
||||
for key in response.keys():
|
||||
if key not in [
|
||||
BlobPaymentAddressResponse.key,
|
||||
BlobAvailabilityResponse.key,
|
||||
BlobPriceResponse.key,
|
||||
BlobDownloadResponse.key]:
|
||||
raise ValueError()
|
||||
extra_data = response_msg[curr_pos:]
|
||||
break
|
||||
except ValueError:
|
||||
response = None
|
||||
if response is None:
|
||||
extra_data = response_msg
|
||||
return response, extra_data
|
||||
continue
|
||||
possible_response_keys = {
|
||||
BlobPaymentAddressResponse.key,
|
||||
BlobAvailabilityResponse.key,
|
||||
BlobPriceResponse.key,
|
||||
BlobDownloadResponse.key
|
||||
}
|
||||
if isinstance(response, dict) and response.keys():
|
||||
if set(response.keys()).issubset(possible_response_keys):
|
||||
return response, response_msg[curr_pos:]
|
||||
return None, response_msg
|
||||
|
||||
|
||||
class BlobRequest:
|
||||
|
||||
@@ -57,8 +57,13 @@ class BlobServerProtocol(asyncio.Protocol):
|
||||
incoming_blob = {'blob_hash': blob.blob_hash, 'length': blob.length}
|
||||
responses.append(BlobDownloadResponse(incoming_blob=incoming_blob))
|
||||
self.send_response(responses)
|
||||
log.info("send %s to %s:%i", blob.blob_hash[:8], peer_address, peer_port)
|
||||
sent = await blob.sendfile(self)
|
||||
log.debug("send %s to %s:%i", blob.blob_hash[:8], peer_address, peer_port)
|
||||
try:
|
||||
sent = await blob.sendfile(self)
|
||||
except (ConnectionResetError, BrokenPipeError, RuntimeError, OSError):
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
return
|
||||
log.info("sent %s (%i bytes) to %s:%i", blob.blob_hash[:8], sent, peer_address, peer_port)
|
||||
if responses:
|
||||
self.send_response(responses)
|
||||
@@ -72,7 +77,7 @@ class BlobServerProtocol(asyncio.Protocol):
|
||||
self.buf += data
|
||||
return
|
||||
try:
|
||||
request = BlobRequest.deserialize(data)
|
||||
request = BlobRequest.deserialize(self.buf + data)
|
||||
self.buf = remainder
|
||||
except JSONDecodeError:
|
||||
addr = self.transport.get_extra_info('peername')
|
||||
@@ -95,6 +100,7 @@ class BlobServer:
|
||||
self.server_task: asyncio.Task = None
|
||||
self.started_listening = asyncio.Event(loop=self.loop)
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
self.server_protocol_class = BlobServerProtocol
|
||||
|
||||
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
|
||||
if self.server_task is not None:
|
||||
@@ -102,7 +108,7 @@ class BlobServer:
|
||||
|
||||
async def _start_server():
|
||||
server = await self.loop.create_server(
|
||||
lambda: BlobServerProtocol(self.loop, self.blob_manager, self.lbrycrd_address),
|
||||
lambda: self.server_protocol_class(self.loop, self.blob_manager, self.lbrycrd_address),
|
||||
interface, port
|
||||
)
|
||||
self.started_listening.set()
|
||||
|
||||
+71
-64
@@ -16,29 +16,11 @@ log = logging.getLogger(__name__)
|
||||
NOT_SET = type(str('NOT_SET'), (object,), {})
|
||||
T = typing.TypeVar('T')
|
||||
|
||||
KB = 2 ** 10
|
||||
MB = 2 ** 20
|
||||
|
||||
ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
|
||||
ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
|
||||
API_ADDRESS = 'lbryapi'
|
||||
APP_NAME = 'LBRY'
|
||||
BLOBFILES_DIR = 'blobfiles'
|
||||
CRYPTSD_FILE_EXTENSION = '.cryptsd'
|
||||
CURRENCIES = {
|
||||
'BTC': {'type': 'crypto'},
|
||||
'LBC': {'type': 'crypto'},
|
||||
'USD': {'type': 'fiat'},
|
||||
}
|
||||
ICON_PATH = 'icons' if 'win' in sys.platform else 'app.icns'
|
||||
LOG_FILE_NAME = 'lbrynet.log'
|
||||
LOG_POST_URL = 'https://lbry.io/log-upload'
|
||||
MAX_BLOB_REQUEST_SIZE = 64 * KB
|
||||
MAX_HANDSHAKE_SIZE = 64 * KB
|
||||
MAX_REQUEST_SIZE = 64 * KB
|
||||
MAX_RESPONSE_INFO_SIZE = 64 * KB
|
||||
MAX_BLOB_INFOS_TO_REQUEST = 20
|
||||
PROTOCOL_PREFIX = 'lbry'
|
||||
SLACK_WEBHOOK = (
|
||||
'nUE0pUZ6Yl9bo29epl5moTSwnl5wo20ip2IlqzywMKZiIQSFZR5'
|
||||
'AHx4mY0VmF0WQZ1ESEP9kMHZlp1WzJwWOoKN3ImR1M2yUAaMyqGZ='
|
||||
@@ -471,7 +453,7 @@ class CLIConfig(BaseConfig):
|
||||
|
||||
|
||||
class Config(CLIConfig):
|
||||
|
||||
# directories
|
||||
data_dir = Path("Directory path to store blobs.", metavar='DIR')
|
||||
download_dir = Path(
|
||||
"Directory path to place assembled files downloaded from LBRY.",
|
||||
@@ -482,60 +464,85 @@ class Config(CLIConfig):
|
||||
previous_names=['lbryum_wallet_dir'], metavar='DIR'
|
||||
)
|
||||
|
||||
share_usage_data = Toggle(
|
||||
"Whether to share usage stats and diagnostic info with LBRY.", True,
|
||||
previous_names=['upload_log', 'upload_log', 'share_debug_info']
|
||||
# network
|
||||
use_upnp = Toggle(
|
||||
"Use UPnP to setup temporary port redirects for the DHT and the hosting of blobs. If you manually forward"
|
||||
"ports or have firewall rules you likely want to disable this.", True
|
||||
)
|
||||
udp_port = Integer("UDP port for communicating on the LBRY DHT", 4444, previous_names=['dht_node_port'])
|
||||
tcp_port = Integer("TCP port to listen for incoming blob requests", 3333, previous_names=['peer_port'])
|
||||
network_interface = String("Interface to use for the DHT and blob exchange", '0.0.0.0')
|
||||
|
||||
# routing table
|
||||
split_buckets_under_index = Integer(
|
||||
"Routing table bucket index below which we always split the bucket if given a new key to add to it and "
|
||||
"the bucket is full. As this value is raised the depth of the routing table (and number of peers in it) "
|
||||
"will increase. This setting is used by seed nodes, you probably don't want to change it during normal "
|
||||
"use.", 1
|
||||
)
|
||||
|
||||
# claims set to expire within this many blocks will be
|
||||
# automatically renewed after startup (if set to 0, renews
|
||||
# will not be made automatically)
|
||||
auto_renew_claim_height_delta = Integer("", 0)
|
||||
cache_time = Integer("", 150)
|
||||
data_rate = Float("points/megabyte", .0001)
|
||||
delete_blobs_on_remove = Toggle("", True)
|
||||
dht_node_port = Integer("", 4444)
|
||||
download_timeout = Float("", 30.0)
|
||||
blob_download_timeout = Float("", 20.0)
|
||||
peer_connect_timeout = Float("", 3.0)
|
||||
node_rpc_timeout = Float("", constants.rpc_timeout)
|
||||
is_generous_host = Toggle("", True)
|
||||
announce_head_blobs_only = Toggle("", True)
|
||||
concurrent_announcers = Integer("", 10)
|
||||
known_dht_nodes = Servers("", [
|
||||
# protocol timeouts
|
||||
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
|
||||
blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
|
||||
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 2.0)
|
||||
node_rpc_timeout = Float("Timeout when making a DHT request", constants.rpc_timeout)
|
||||
|
||||
# blob announcement and download
|
||||
announce_head_and_sd_only = Toggle(
|
||||
"Announce only the descriptor and first (rather than all) data blob for a stream to the DHT", True,
|
||||
previous_names=['announce_head_blobs_only']
|
||||
)
|
||||
concurrent_blob_announcers = Integer(
|
||||
"Number of blobs to iteratively announce at once, set to 0 to disable", 10,
|
||||
previous_names=['concurrent_announcers']
|
||||
)
|
||||
max_connections_per_download = Integer(
|
||||
"Maximum number of peers to connect to while downloading a blob", 8,
|
||||
previous_names=['max_connections_per_stream']
|
||||
)
|
||||
fixed_peer_delay = Float(
|
||||
"Amount of seconds before adding the reflector servers as potential peers to download from in case dht"
|
||||
"peers are not found or are slow", 2.0
|
||||
)
|
||||
max_key_fee = MaxKeyFee(
|
||||
"Don't download streams with fees exceeding this amount", {'currency': 'USD', 'amount': 50.0}
|
||||
)
|
||||
|
||||
# reflector settings
|
||||
reflect_streams = Toggle(
|
||||
"Upload completed streams (published and downloaded) reflector in order to re-host them", True,
|
||||
previous_names=['reflect_uploads']
|
||||
)
|
||||
concurrent_reflector_uploads = Integer(
|
||||
"Maximum number of streams to upload to a reflector server at a time", 10
|
||||
)
|
||||
|
||||
# servers
|
||||
reflector_servers = Servers("Reflector re-hosting servers", [
|
||||
('reflector.lbry.io', 5566)
|
||||
])
|
||||
lbryum_servers = Servers("SPV wallet servers", [
|
||||
('lbryumx1.lbry.io', 50001),
|
||||
('lbryumx2.lbry.io', 50001)
|
||||
])
|
||||
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
|
||||
('lbrynet1.lbry.io', 4444), # US EAST
|
||||
('lbrynet2.lbry.io', 4444), # US WEST
|
||||
('lbrynet3.lbry.io', 4444), # EU
|
||||
('lbrynet4.lbry.io', 4444) # ASIA
|
||||
])
|
||||
max_connections_per_stream = Integer("", 5)
|
||||
seek_head_blob_first = Toggle("", True)
|
||||
max_key_fee = MaxKeyFee("", {'currency': 'USD', 'amount': 50.0})
|
||||
min_info_rate = Float("points/1000 infos", .02)
|
||||
min_valuable_hash_rate = Float("points/1000 infos", .05)
|
||||
min_valuable_info_rate = Float("points/1000 infos", .05)
|
||||
peer_port = Integer("", 3333)
|
||||
pointtrader_server = String("", 'http://127.0.0.1:2424')
|
||||
reflector_port = Integer("", 5566)
|
||||
# if reflect_uploads is True, send files to reflector after publishing (as well as a periodic check in the
|
||||
# event the initial upload failed or was disconnected part way through, provided the auto_re_reflect_interval > 0)
|
||||
reflect_uploads = Toggle("", True)
|
||||
auto_re_reflect_interval = Integer("set to 0 to disable", 86400)
|
||||
reflector_servers = Servers("", [
|
||||
('reflector.lbry.io', 5566)
|
||||
])
|
||||
run_reflector_server = Toggle("adds reflector to components_to_skip unless True", False)
|
||||
sd_download_timeout = Integer("", 3)
|
||||
peer_search_timeout = Integer("", 60)
|
||||
use_upnp = Toggle("", True)
|
||||
use_keyring = Toggle("", False)
|
||||
blockchain_name = String("", 'lbrycrd_main')
|
||||
lbryum_servers = Servers("", [
|
||||
('lbryumx1.lbry.io', 50001),
|
||||
('lbryumx2.lbry.io', 50001)
|
||||
])
|
||||
|
||||
# blockchain
|
||||
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
|
||||
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
|
||||
cache_time = Integer("Time to cache resolved claims", 150) # TODO: use this
|
||||
|
||||
# daemon
|
||||
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
|
||||
share_usage_data = Toggle(
|
||||
"Whether to share usage stats and diagnostic info with LBRY.", True,
|
||||
previous_names=['upload_log', 'upload_log', 'share_debug_info']
|
||||
)
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
@@ -19,6 +19,8 @@ class BlobAnnouncer:
|
||||
self.announce_queue: typing.List[str] = []
|
||||
|
||||
async def _announce(self, batch_size: typing.Optional[int] = 10):
|
||||
if not batch_size:
|
||||
return
|
||||
if not self.node.joined.is_set():
|
||||
await self.node.joined.wait()
|
||||
blob_hashes = await self.storage.get_blobs_to_announce()
|
||||
@@ -39,7 +41,7 @@ class BlobAnnouncer:
|
||||
to_await.append(batch.pop())
|
||||
if to_await:
|
||||
await asyncio.gather(*tuple(to_await), loop=self.loop)
|
||||
await self.storage.update_last_announced_blobs(announced, self.loop.time())
|
||||
await self.storage.update_last_announced_blobs(announced)
|
||||
log.info("announced %i blobs", len(announced))
|
||||
if self.running:
|
||||
self.pending_call = self.loop.call_later(60, self.announce, batch_size)
|
||||
|
||||
@@ -6,6 +6,7 @@ hash_length = hash_class().digest_size
|
||||
hash_bits = hash_length * 8
|
||||
alpha = 5
|
||||
k = 8
|
||||
split_buckets_under_index = 1
|
||||
replacement_cache_size = 8
|
||||
rpc_timeout = 5.0
|
||||
rpc_attempts = 5
|
||||
@@ -15,6 +16,7 @@ refresh_interval = 3600 # 1 hour
|
||||
replicate_interval = refresh_interval
|
||||
data_expiration = 86400 # 24 hours
|
||||
token_secret_refresh_interval = 300 # 5 minutes
|
||||
maybe_ping_delay = 300 # 5 minutes
|
||||
check_refresh_interval = refresh_interval / 5
|
||||
max_datagram_size = 8192 # 8 KB
|
||||
rpc_id_length = 20
|
||||
|
||||
+62
-41
@@ -1,9 +1,9 @@
|
||||
import logging
|
||||
import asyncio
|
||||
import typing
|
||||
import socket
|
||||
import binascii
|
||||
import contextlib
|
||||
from lbrynet.utils import resolve_host
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.error import RemoteException
|
||||
from lbrynet.dht.protocol.async_generator_junction import AsyncGeneratorJunction
|
||||
@@ -20,10 +20,12 @@ log = logging.getLogger(__name__)
|
||||
|
||||
class Node:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int,
|
||||
internal_udp_port: int, peer_port: int, external_ip: str):
|
||||
internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: float = constants.rpc_timeout,
|
||||
split_buckets_under_index: int = constants.split_buckets_under_index):
|
||||
self.loop = loop
|
||||
self.internal_udp_port = internal_udp_port
|
||||
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port)
|
||||
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port, rpc_timeout,
|
||||
split_buckets_under_index)
|
||||
self.listening_port: asyncio.DatagramTransport = None
|
||||
self.joined = asyncio.Event(loop=self.loop)
|
||||
self._join_task: asyncio.Task = None
|
||||
@@ -64,7 +66,7 @@ class Node:
|
||||
# ping the set of peers; upon success/failure the routing able and last replied/failed time will be updated
|
||||
to_ping = [peer for peer in set(total_peers) if self.protocol.peer_manager.peer_is_good(peer) is not True]
|
||||
if to_ping:
|
||||
await self.protocol.ping_queue.enqueue_maybe_ping(*to_ping, delay=0)
|
||||
self.protocol.ping_queue.enqueue_maybe_ping(*to_ping, delay=0)
|
||||
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
self.loop.call_later(constants.refresh_interval, fut.set_result, None)
|
||||
@@ -79,10 +81,9 @@ class Node:
|
||||
|
||||
if not self.protocol.external_ip:
|
||||
raise Exception("Cannot determine external IP")
|
||||
log.info("Store to %i peers", len(peers))
|
||||
log.info(peers)
|
||||
log.debug("Store to %i peers", len(peers))
|
||||
for peer in peers:
|
||||
log.info("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
|
||||
log.debug("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
|
||||
stored_to_tup = await asyncio.gather(
|
||||
*(self.protocol.store_to_peer(hash_value, peer) for peer in peers), loop=self.loop
|
||||
)
|
||||
@@ -116,39 +117,50 @@ class Node:
|
||||
log.warning("Already bound to port %s", self.listening_port)
|
||||
|
||||
async def join_network(self, interface: typing.Optional[str] = '',
|
||||
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
known_node_addresses: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
|
||||
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
|
||||
if not self.listening_port:
|
||||
await self.start_listening(interface)
|
||||
self.protocol.ping_queue.start()
|
||||
self._refresh_task = self.loop.create_task(self.refresh_node())
|
||||
|
||||
known_node_addresses = known_node_addresses or []
|
||||
# resolve the known node urls
|
||||
known_node_addresses = []
|
||||
url_to_addr = {}
|
||||
|
||||
if known_node_urls:
|
||||
for host, port in known_node_urls:
|
||||
info = await self.loop.getaddrinfo(
|
||||
host, 'https',
|
||||
proto=socket.IPPROTO_TCP,
|
||||
)
|
||||
if (info[0][4][0], port) not in known_node_addresses:
|
||||
known_node_addresses.append((info[0][4][0], port))
|
||||
futs = []
|
||||
for address, port in known_node_addresses:
|
||||
peer = self.protocol.get_rpc_peer(KademliaPeer(self.loop, address, udp_port=port))
|
||||
futs.append(peer.ping())
|
||||
if futs:
|
||||
await asyncio.wait(futs, loop=self.loop)
|
||||
address = await resolve_host(host, port, proto='udp')
|
||||
if (address, port) not in known_node_addresses and address != self.protocol.external_ip:
|
||||
known_node_addresses.append((address, port))
|
||||
url_to_addr[address] = host
|
||||
|
||||
async with self.peer_search_junction(self.protocol.node_id, max_results=16) as junction:
|
||||
async for peers in junction:
|
||||
for peer in peers:
|
||||
if known_node_addresses:
|
||||
while not self.protocol.routing_table.get_peers():
|
||||
success = False
|
||||
# ping the seed nodes, this will set their node ids (since we don't know them ahead of time)
|
||||
for address, port in known_node_addresses:
|
||||
peer = self.protocol.get_rpc_peer(KademliaPeer(self.loop, address, udp_port=port))
|
||||
try:
|
||||
await self.protocol.get_rpc_peer(peer).ping()
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
pass
|
||||
self.joined.set()
|
||||
await peer.ping()
|
||||
success = True
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("seed node (%s:%i) timed out in %s", url_to_addr.get(address, address), port,
|
||||
round(self.protocol.rpc_timeout, 2))
|
||||
if success:
|
||||
break
|
||||
# now that we have the seed nodes in routing, to an iterative lookup of our own id to populate the buckets
|
||||
# in the routing table with good peers who are near us
|
||||
async with self.peer_search_junction(self.protocol.node_id, max_results=16) as junction:
|
||||
async for peers in junction:
|
||||
for peer in peers:
|
||||
try:
|
||||
await self.protocol.get_rpc_peer(peer).ping()
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
pass
|
||||
|
||||
log.info("Joined DHT, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
|
||||
self.protocol.routing_table.buckets_with_contacts())
|
||||
self.joined.set()
|
||||
|
||||
def start(self, interface: str, known_node_urls: typing.List[typing.Tuple[str, int]]):
|
||||
self._join_task = self.loop.create_task(
|
||||
@@ -178,10 +190,7 @@ class Node:
|
||||
|
||||
async def _add_hashes_from_queue():
|
||||
while True:
|
||||
try:
|
||||
blob_hash = await hash_queue.get()
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
blob_hash = await hash_queue.get()
|
||||
peer_generator.add_generator(
|
||||
self.get_iterative_value_finder(
|
||||
binascii.unhexlify(blob_hash.encode()), bottom_out_limit=bottom_out_limit,
|
||||
@@ -193,10 +202,6 @@ class Node:
|
||||
async with peer_generator as junction:
|
||||
yield junction
|
||||
await peer_generator.finished.wait()
|
||||
except asyncio.CancelledError:
|
||||
if add_hashes_task and not (add_hashes_task.done() or add_hashes_task.cancelled()):
|
||||
add_hashes_task.cancel()
|
||||
raise
|
||||
finally:
|
||||
if add_hashes_task and not (add_hashes_task.done() or add_hashes_task.cancelled()):
|
||||
add_hashes_task.cancel()
|
||||
@@ -214,13 +219,29 @@ class Node:
|
||||
async def peer_search(self, node_id: bytes, count=constants.k, max_results=constants.k*2,
|
||||
bottom_out_limit=20) -> typing.List['KademliaPeer']:
|
||||
accumulated: typing.List['KademliaPeer'] = []
|
||||
async with self.peer_search_junction(self.protocol.node_id, max_results=max_results,
|
||||
async with self.peer_search_junction(node_id, max_results=max_results,
|
||||
bottom_out_limit=bottom_out_limit) as junction:
|
||||
async for peers in junction:
|
||||
log.info("peer search: %s", peers)
|
||||
accumulated.extend(peers)
|
||||
log.info("junction done")
|
||||
log.info("context done")
|
||||
distance = Distance(node_id)
|
||||
accumulated.sort(key=lambda peer: distance(peer.node_id))
|
||||
return accumulated[:count]
|
||||
|
||||
async def _accumulate_search_junction(self, search_queue: asyncio.Queue,
|
||||
result_queue: asyncio.Queue):
|
||||
async with self.stream_peer_search_junction(search_queue) as search_junction: # pylint: disable=E1701
|
||||
async for peers in search_junction:
|
||||
if peers:
|
||||
result_queue.put_nowait([
|
||||
peer for peer in peers
|
||||
if not (
|
||||
peer.address == self.protocol.external_ip
|
||||
and peer.tcp_port == self.protocol.peer_port
|
||||
)
|
||||
])
|
||||
|
||||
def accumulate_peers(self, search_queue: asyncio.Queue,
|
||||
peer_queue: typing.Optional[asyncio.Queue] = None) -> typing.Tuple[
|
||||
asyncio.Queue, asyncio.Task]:
|
||||
q = peer_queue or asyncio.Queue()
|
||||
return q, asyncio.create_task(self._accumulate_search_junction(search_queue, q))
|
||||
+36
-54
@@ -3,6 +3,8 @@ import asyncio
|
||||
import logging
|
||||
import ipaddress
|
||||
from binascii import hexlify
|
||||
from functools import lru_cache
|
||||
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
|
||||
|
||||
@@ -30,37 +32,30 @@ class PeerManager:
|
||||
self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = {}
|
||||
self._node_tokens: typing.Dict[bytes, (float, bytes)] = {}
|
||||
self._kademlia_peers: typing.Dict[typing.Tuple[bytes, str, int], 'KademliaPeer']
|
||||
self._lock = asyncio.Lock(loop=loop)
|
||||
|
||||
async def report_failure(self, address: str, udp_port: int):
|
||||
def report_failure(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
async with self._lock:
|
||||
_, previous = self._rpc_failures.pop((address, udp_port), (None, None))
|
||||
self._rpc_failures[(address, udp_port)] = (previous, now)
|
||||
_, previous = self._rpc_failures.pop((address, udp_port), (None, None))
|
||||
self._rpc_failures[(address, udp_port)] = (previous, now)
|
||||
|
||||
async def report_last_sent(self, address: str, udp_port: int):
|
||||
def report_last_sent(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
async with self._lock:
|
||||
self._last_sent[(address, udp_port)] = now
|
||||
self._last_sent[(address, udp_port)] = now
|
||||
|
||||
async def report_last_replied(self, address: str, udp_port: int):
|
||||
def report_last_replied(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
async with self._lock:
|
||||
self._last_replied[(address, udp_port)] = now
|
||||
self._last_replied[(address, udp_port)] = now
|
||||
|
||||
async def report_last_requested(self, address: str, udp_port: int):
|
||||
def report_last_requested(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
async with self._lock:
|
||||
self._last_requested[(address, udp_port)] = now
|
||||
self._last_requested[(address, udp_port)] = now
|
||||
|
||||
async def clear_token(self, node_id: bytes):
|
||||
async with self._lock:
|
||||
self._node_tokens.pop(node_id, None)
|
||||
def clear_token(self, node_id: bytes):
|
||||
self._node_tokens.pop(node_id, None)
|
||||
|
||||
async def update_token(self, node_id: bytes, token: bytes):
|
||||
def update_token(self, node_id: bytes, token: bytes):
|
||||
now = self._loop.time()
|
||||
async with self._lock:
|
||||
self._node_tokens[node_id] = (now, token)
|
||||
self._node_tokens[node_id] = (now, token)
|
||||
|
||||
def get_node_token(self, node_id: bytes) -> typing.Optional[bytes]:
|
||||
ts, token = self._node_tokens.get(node_id, (None, None))
|
||||
@@ -70,50 +65,37 @@ class PeerManager:
|
||||
def get_last_replied(self, address: str, udp_port: int) -> typing.Optional[float]:
|
||||
return self._last_replied.get((address, udp_port))
|
||||
|
||||
def get_node_id(self, address: str, udp_port: int) -> typing.Optional[bytes]:
|
||||
return self._node_id_mapping.get((address, udp_port))
|
||||
|
||||
def get_node_address(self, node_id: bytes) -> typing.Optional[typing.Tuple[str, int]]:
|
||||
return self._node_id_reverse_mapping.get(node_id)
|
||||
|
||||
async def get_node_address_and_port(self, node_id: bytes) -> typing.Optional[typing.Tuple[str, int]]:
|
||||
async with self._lock:
|
||||
addr_tuple = self._node_id_reverse_mapping.get(node_id)
|
||||
if addr_tuple and addr_tuple in self._node_id_mapping:
|
||||
return addr_tuple
|
||||
|
||||
async def update_contact_triple(self, node_id: bytes, address: str, udp_port: int):
|
||||
def update_contact_triple(self, node_id: bytes, address: str, udp_port: int):
|
||||
"""
|
||||
Update the mapping of node_id -> address tuple and that of address tuple -> node_id
|
||||
This is to handle peers changing addresses and ids while assuring that the we only ever have
|
||||
one node id / address tuple mapped to each other
|
||||
"""
|
||||
async with self._lock:
|
||||
if (address, udp_port) in self._node_id_mapping:
|
||||
self._node_id_reverse_mapping.pop(self._node_id_mapping.pop((address, udp_port)))
|
||||
if node_id in self._node_id_reverse_mapping:
|
||||
self._node_id_mapping.pop(self._node_id_reverse_mapping.pop(node_id))
|
||||
self._node_id_mapping[(address, udp_port)] = node_id
|
||||
self._node_id_reverse_mapping[node_id] = (address, udp_port)
|
||||
if (address, udp_port) in self._node_id_mapping:
|
||||
self._node_id_reverse_mapping.pop(self._node_id_mapping.pop((address, udp_port)))
|
||||
if node_id in self._node_id_reverse_mapping:
|
||||
self._node_id_mapping.pop(self._node_id_reverse_mapping.pop(node_id))
|
||||
self._node_id_mapping[(address, udp_port)] = node_id
|
||||
self._node_id_reverse_mapping[node_id] = (address, udp_port)
|
||||
|
||||
@lru_cache(maxsize=400)
|
||||
def get_kademlia_peer(self, node_id: bytes, address: str, udp_port: int) -> 'KademliaPeer':
|
||||
return KademliaPeer(self._loop, address, node_id, udp_port)
|
||||
|
||||
async def prune(self):
|
||||
def prune(self): # TODO: periodically call this
|
||||
now = self._loop.time()
|
||||
async with self._lock:
|
||||
to_pop = []
|
||||
for (address, udp_port), (_, last_failure) in self._rpc_failures.items():
|
||||
if last_failure and last_failure < now - constants.rpc_attempts_pruning_window:
|
||||
to_pop.append((address, udp_port))
|
||||
while to_pop:
|
||||
del self._rpc_failures[to_pop.pop()]
|
||||
to_pop = []
|
||||
for node_id, (age, token) in self._node_tokens.items():
|
||||
if age < now - constants.token_secret_refresh_interval:
|
||||
to_pop.append(node_id)
|
||||
while to_pop:
|
||||
del self._node_tokens[to_pop.pop()]
|
||||
to_pop = []
|
||||
for (address, udp_port), (_, last_failure) in self._rpc_failures.items():
|
||||
if last_failure and last_failure < now - constants.rpc_attempts_pruning_window:
|
||||
to_pop.append((address, udp_port))
|
||||
while to_pop:
|
||||
del self._rpc_failures[to_pop.pop()]
|
||||
to_pop = []
|
||||
for node_id, (age, token) in self._node_tokens.items():
|
||||
if age < now - constants.token_secret_refresh_interval:
|
||||
to_pop.append(node_id)
|
||||
while to_pop:
|
||||
del self._node_tokens[to_pop.pop()]
|
||||
|
||||
def contact_triple_is_good(self, node_id: bytes, address: str, udp_port: int):
|
||||
"""
|
||||
|
||||
@@ -100,6 +100,6 @@ class AsyncGeneratorJunction:
|
||||
pass
|
||||
finally:
|
||||
if exc_type:
|
||||
if exc_type not in (asyncio.CancelledError, asyncio.TimeoutError, StopAsyncIteration):
|
||||
if exc_type not in (asyncio.CancelledError, asyncio.TimeoutError, StopAsyncIteration, GeneratorExit):
|
||||
err = traceback.format_exception(exc_type, exc, tb)
|
||||
log.error(err)
|
||||
@@ -1,7 +1,9 @@
|
||||
import asyncio
|
||||
from binascii import hexlify
|
||||
from itertools import chain
|
||||
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet.utils import drain_tasks
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.error import RemoteException
|
||||
from lbrynet.dht.protocol.distance import Distance
|
||||
@@ -90,8 +92,8 @@ class IterativeFinder:
|
||||
self.exclude = exclude or []
|
||||
|
||||
self.shortlist: typing.List['KademliaPeer'] = get_shortlist(routing_table, key, shortlist)
|
||||
self.active: typing.List['KademliaPeer'] = []
|
||||
self.contacted: typing.List[typing.Tuple[str, int]] = []
|
||||
self.active: typing.Set['KademliaPeer'] = set()
|
||||
self.contacted: typing.Set[typing.Tuple[str, int]] = set()
|
||||
self.distance = Distance(key)
|
||||
|
||||
self.closest_peer: typing.Optional['KademliaPeer'] = None if not self.shortlist else self.shortlist[0]
|
||||
@@ -99,14 +101,12 @@ class IterativeFinder:
|
||||
|
||||
self.iteration_queue = asyncio.Queue(loop=self.loop)
|
||||
|
||||
self.running_probes: typing.List[asyncio.Task] = []
|
||||
self.lock = asyncio.Lock(loop=self.loop)
|
||||
self.running_probes: typing.Set[asyncio.Task] = set()
|
||||
self.iteration_count = 0
|
||||
self.bottom_out_count = 0
|
||||
self.running = False
|
||||
self.tasks: typing.List[asyncio.Task] = []
|
||||
self.delayed_calls: typing.List[asyncio.Handle] = []
|
||||
self.finished = asyncio.Event(loop=self.loop)
|
||||
|
||||
async def send_probe(self, peer: 'KademliaPeer') -> FindResponse:
|
||||
"""
|
||||
@@ -114,9 +114,16 @@ class IterativeFinder:
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def search_exhausted(self):
|
||||
"""
|
||||
This method ends the iterator due no more peers to contact.
|
||||
Override to provide last time results.
|
||||
"""
|
||||
self.iteration_queue.put_nowait(None)
|
||||
|
||||
def check_result_ready(self, response: FindResponse):
|
||||
"""
|
||||
Called with a lock after adding peers from an rpc result to the shortlist.
|
||||
Called after adding peers from an rpc result to the shortlist.
|
||||
This method is responsible for putting a result for the generator into the Queue
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
@@ -129,9 +136,7 @@ class IterativeFinder:
|
||||
return []
|
||||
|
||||
def _is_closer(self, peer: 'KademliaPeer') -> bool:
|
||||
if not self.closest_peer:
|
||||
return True
|
||||
return self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
|
||||
return not self.closest_peer or self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
|
||||
|
||||
def _update_closest(self):
|
||||
self.shortlist.sort(key=lambda peer: self.distance(peer.node_id), reverse=True)
|
||||
@@ -141,35 +146,28 @@ class IterativeFinder:
|
||||
self.closest_peer = self.shortlist[-1]
|
||||
|
||||
async def _handle_probe_result(self, peer: 'KademliaPeer', response: FindResponse):
|
||||
async with self.lock:
|
||||
if peer not in self.shortlist:
|
||||
self.shortlist.append(peer)
|
||||
if peer not in self.active:
|
||||
self.active.append(peer)
|
||||
for contact_triple in response.get_close_triples():
|
||||
addr_tuple = (contact_triple[1], contact_triple[2])
|
||||
if addr_tuple not in self.contacted: # and not self.peer_manager.is_ignored(addr_tuple)
|
||||
found_peer = self.peer_manager.get_kademlia_peer(
|
||||
contact_triple[0], contact_triple[1], contact_triple[2]
|
||||
)
|
||||
if found_peer not in self.shortlist and self.peer_manager.peer_is_good(peer) is not False:
|
||||
self.shortlist.append(found_peer)
|
||||
self._update_closest()
|
||||
self.check_result_ready(response)
|
||||
if peer not in self.shortlist:
|
||||
self.shortlist.append(peer)
|
||||
if peer not in self.active:
|
||||
self.active.add(peer)
|
||||
for contact_triple in response.get_close_triples():
|
||||
node_id, address, udp_port = contact_triple
|
||||
if (address, udp_port) not in self.contacted: # and not self.peer_manager.is_ignored(addr_tuple)
|
||||
found_peer = self.peer_manager.get_kademlia_peer(node_id, address, udp_port)
|
||||
if found_peer not in self.shortlist and self.peer_manager.peer_is_good(peer) is not False:
|
||||
self.shortlist.append(found_peer)
|
||||
self._update_closest()
|
||||
self.check_result_ready(response)
|
||||
|
||||
async def _send_probe(self, peer: 'KademliaPeer'):
|
||||
try:
|
||||
response = await self.send_probe(peer)
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
except asyncio.TimeoutError:
|
||||
if peer in self.active:
|
||||
self.active.remove(peer)
|
||||
self.active.discard(peer)
|
||||
return
|
||||
except ValueError as err:
|
||||
log.warning(str(err))
|
||||
if peer in self.active:
|
||||
self.active.remove(peer)
|
||||
self.active.discard(peer)
|
||||
return
|
||||
except RemoteException:
|
||||
return
|
||||
@@ -181,31 +179,35 @@ class IterativeFinder:
|
||||
"""
|
||||
|
||||
added = 0
|
||||
async with self.lock:
|
||||
self.shortlist.sort(key=lambda p: self.distance(p.node_id), reverse=True)
|
||||
while self.running and len(self.shortlist) and added < constants.alpha:
|
||||
peer = self.shortlist.pop()
|
||||
origin_address = (peer.address, peer.udp_port)
|
||||
if origin_address in self.exclude or self.peer_manager.peer_is_good(peer) is False:
|
||||
continue
|
||||
if peer.node_id == self.protocol.node_id:
|
||||
continue
|
||||
if (peer.address, peer.udp_port) == (self.protocol.external_ip, self.protocol.udp_port):
|
||||
continue
|
||||
if (peer.address, peer.udp_port) not in self.contacted:
|
||||
self.contacted.append((peer.address, peer.udp_port))
|
||||
self.shortlist.sort(key=lambda p: self.distance(p.node_id), reverse=True)
|
||||
while self.running and len(self.shortlist) and added < constants.alpha:
|
||||
peer = self.shortlist.pop()
|
||||
origin_address = (peer.address, peer.udp_port)
|
||||
if origin_address in self.exclude or self.peer_manager.peer_is_good(peer) is False:
|
||||
continue
|
||||
if peer.node_id == self.protocol.node_id:
|
||||
continue
|
||||
if (peer.address, peer.udp_port) == (self.protocol.external_ip, self.protocol.udp_port):
|
||||
continue
|
||||
if (peer.address, peer.udp_port) not in self.contacted:
|
||||
self.contacted.add((peer.address, peer.udp_port))
|
||||
|
||||
t = self.loop.create_task(self._send_probe(peer))
|
||||
t = self.loop.create_task(self._send_probe(peer))
|
||||
|
||||
def callback(_):
|
||||
if t and t in self.running_probes:
|
||||
self.running_probes.remove(t)
|
||||
if not self.running_probes and self.shortlist:
|
||||
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
|
||||
def callback(_):
|
||||
self.running_probes.difference_update({
|
||||
probe for probe in self.running_probes if probe.done() or probe == t
|
||||
})
|
||||
if not self.running_probes and self.shortlist:
|
||||
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
|
||||
|
||||
t.add_done_callback(callback)
|
||||
self.running_probes.append(t)
|
||||
added += 1
|
||||
t.add_done_callback(callback)
|
||||
self.running_probes.add(t)
|
||||
added += 1
|
||||
log.debug("running %d probes", len(self.running_probes))
|
||||
if not added and not self.running_probes:
|
||||
log.debug("search for %s exhausted", hexlify(self.key)[:8])
|
||||
self.search_exhausted()
|
||||
|
||||
async def _search_task(self, delay: typing.Optional[float] = constants.iterative_lookup_delay):
|
||||
try:
|
||||
@@ -215,70 +217,40 @@ class IterativeFinder:
|
||||
self.delayed_calls.append(self.loop.call_later(delay, self._search))
|
||||
except (asyncio.CancelledError, StopAsyncIteration):
|
||||
if self.running:
|
||||
drain_tasks(self.running_probes)
|
||||
self.running = False
|
||||
self.loop.call_soon(self.aclose)
|
||||
|
||||
def _search(self):
|
||||
self.tasks.append(self.loop.create_task(self._search_task()))
|
||||
|
||||
def search(self):
|
||||
def __aiter__(self):
|
||||
if self.running:
|
||||
raise Exception("already running")
|
||||
self.running = True
|
||||
self._search()
|
||||
|
||||
async def next_queue_or_finished(self) -> typing.List['KademliaPeer']:
|
||||
peers = self.loop.create_task(self.iteration_queue.get())
|
||||
finished = self.loop.create_task(self.finished.wait())
|
||||
err = None
|
||||
try:
|
||||
await asyncio.wait([peers, finished], loop=self.loop, return_when='FIRST_COMPLETED')
|
||||
if peers.done():
|
||||
return peers.result()
|
||||
raise StopAsyncIteration()
|
||||
except asyncio.CancelledError as error:
|
||||
err = error
|
||||
finally:
|
||||
if not finished.done() and not finished.cancelled():
|
||||
finished.cancel()
|
||||
if not peers.done() and not peers.cancelled():
|
||||
peers.cancel()
|
||||
if err:
|
||||
raise err
|
||||
|
||||
def __aiter__(self):
|
||||
self.search()
|
||||
return self
|
||||
|
||||
async def __anext__(self) -> typing.List['KademliaPeer']:
|
||||
try:
|
||||
if self.iteration_count == 0:
|
||||
initial_results = self.get_initial_result()
|
||||
if initial_results:
|
||||
self.iteration_queue.put_nowait(initial_results)
|
||||
result = await self.next_queue_or_finished()
|
||||
result = self.get_initial_result() or await self.iteration_queue.get()
|
||||
else:
|
||||
result = await self.iteration_queue.get()
|
||||
if not result:
|
||||
raise StopAsyncIteration
|
||||
self.iteration_count += 1
|
||||
return result
|
||||
except (asyncio.CancelledError, StopAsyncIteration):
|
||||
await self.aclose()
|
||||
self.loop.call_soon(self.aclose)
|
||||
raise
|
||||
|
||||
def aclose(self):
|
||||
self.running = False
|
||||
|
||||
async def _aclose():
|
||||
async with self.lock:
|
||||
self.running = False
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
drain_tasks(self.tasks)
|
||||
drain_tasks(self.running_probes)
|
||||
while self.delayed_calls:
|
||||
timer = self.delayed_calls.pop()
|
||||
if timer:
|
||||
timer.cancel()
|
||||
|
||||
return asyncio.ensure_future(_aclose(), loop=self.loop)
|
||||
self.iteration_queue.put_nowait(None)
|
||||
for task in chain(self.tasks, self.running_probes, self.delayed_calls):
|
||||
task.cancel()
|
||||
self.tasks.clear()
|
||||
self.running_probes.clear()
|
||||
self.delayed_calls.clear()
|
||||
|
||||
|
||||
class IterativeNodeFinder(IterativeFinder):
|
||||
@@ -295,24 +267,26 @@ class IterativeNodeFinder(IterativeFinder):
|
||||
response = await self.protocol.get_rpc_peer(peer).find_node(self.key)
|
||||
return FindNodeResponse(self.key, response)
|
||||
|
||||
def put_result(self, from_list: typing.List['KademliaPeer']):
|
||||
not_yet_yielded = [peer for peer in from_list if peer not in self.yielded_peers]
|
||||
def search_exhausted(self):
|
||||
self.put_result(self.active, finish=True)
|
||||
|
||||
def put_result(self, from_iter: typing.Iterable['KademliaPeer'], finish=False):
|
||||
not_yet_yielded = [peer for peer in from_iter if peer not in self.yielded_peers]
|
||||
not_yet_yielded.sort(key=lambda peer: self.distance(peer.node_id))
|
||||
to_yield = not_yet_yielded[:min(constants.k, len(not_yet_yielded))]
|
||||
if to_yield:
|
||||
for peer in to_yield:
|
||||
self.yielded_peers.add(peer)
|
||||
self.iteration_queue.put_nowait(to_yield)
|
||||
if finish:
|
||||
self.iteration_queue.put_nowait(None)
|
||||
|
||||
def check_result_ready(self, response: FindNodeResponse):
|
||||
found = response.found and self.key != self.protocol.node_id
|
||||
|
||||
if found:
|
||||
log.info("found")
|
||||
self.put_result(self.shortlist)
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return
|
||||
log.debug("found")
|
||||
return self.put_result(self.shortlist, finish=True)
|
||||
if self.prev_closest_peer and self.closest_peer and not self._is_closer(self.prev_closest_peer):
|
||||
# log.info("improving, %i %i %i %i %i", len(self.shortlist), len(self.active), len(self.contacted),
|
||||
# self.bottom_out_count, self.iteration_count)
|
||||
@@ -323,16 +297,10 @@ class IterativeNodeFinder(IterativeFinder):
|
||||
self.bottom_out_count)
|
||||
if self.bottom_out_count >= self.bottom_out_limit or self.iteration_count >= self.bottom_out_limit:
|
||||
log.info("limit hit")
|
||||
self.put_result(self.active)
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return
|
||||
if self.max_results and len(self.active) - len(self.yielded_peers) >= self.max_results:
|
||||
log.info("max results")
|
||||
self.put_result(self.active)
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return
|
||||
self.put_result(self.active, True)
|
||||
elif self.max_results and len(self.active) - len(self.yielded_peers) >= self.max_results:
|
||||
log.debug("max results")
|
||||
self.put_result(self.active, True)
|
||||
|
||||
|
||||
class IterativeValueFinder(IterativeFinder):
|
||||
@@ -366,14 +334,11 @@ class IterativeValueFinder(IterativeFinder):
|
||||
# log.info("enough blob peers found")
|
||||
# if not self.finished.is_set():
|
||||
# self.finished.set()
|
||||
return
|
||||
if self.prev_closest_peer and self.closest_peer:
|
||||
elif self.prev_closest_peer and self.closest_peer:
|
||||
self.bottom_out_count += 1
|
||||
if self.bottom_out_count >= self.bottom_out_limit:
|
||||
log.info("blob peer search bottomed out")
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return
|
||||
self.iteration_queue.put_nowait(None)
|
||||
|
||||
def get_initial_result(self) -> typing.List['KademliaPeer']:
|
||||
if self.protocol.data_store.has_peers_for_blob(self.key):
|
||||
|
||||
+104
-103
@@ -77,24 +77,23 @@ class KademliaRPC:
|
||||
|
||||
response = {
|
||||
b'token': self.make_token(rpc_contact.compact_ip()),
|
||||
b'contacts': self.find_node(rpc_contact, key)
|
||||
}
|
||||
|
||||
if self.protocol.protocol_version:
|
||||
response[b'protocolVersion'] = self.protocol.protocol_version
|
||||
|
||||
# get peers we have stored for this blob_exchange
|
||||
has_other_peers = self.protocol.data_store.has_peers_for_blob(key)
|
||||
peers = []
|
||||
if has_other_peers:
|
||||
peers.extend(self.protocol.data_store.get_peers_for_blob(key))
|
||||
|
||||
peers = [
|
||||
peer.compact_address_tcp()
|
||||
for peer in self.protocol.data_store.get_peers_for_blob(key)
|
||||
if not rpc_contact.tcp_port or peer.compact_address_tcp() != rpc_contact.compact_address_tcp()
|
||||
]
|
||||
# if we don't have k storing peers to return and we have this hash locally, include our contact information
|
||||
if len(peers) < constants.k and binascii.hexlify(key).decode() in self.protocol.data_store.completed_blobs:
|
||||
peers.append(self.compact_address())
|
||||
if peers:
|
||||
response[key] = peers
|
||||
else:
|
||||
response[b'contacts'] = self.find_node(rpc_contact, key)
|
||||
return response
|
||||
|
||||
def refresh_token(self): # TODO: this needs to be called periodically
|
||||
@@ -180,7 +179,7 @@ class RemoteKademliaRPC:
|
||||
response = await self.protocol.send_request(
|
||||
self.peer, RequestDatagram.make_find_value(self.protocol.node_id, key)
|
||||
)
|
||||
await self.peer_tracker.update_token(self.peer.node_id, response.response[b'token'])
|
||||
self.peer_tracker.update_token(self.peer.node_id, response.response[b'token'])
|
||||
return response.response
|
||||
|
||||
|
||||
@@ -188,59 +187,46 @@ class PingQueue:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, protocol: 'KademliaProtocol'):
|
||||
self._loop = loop
|
||||
self._protocol = protocol
|
||||
self._enqueued_contacts: typing.List['KademliaPeer'] = []
|
||||
self._pending_contacts: typing.Dict['KademliaPeer', float] = {}
|
||||
self._process_task: asyncio.Task = None
|
||||
self._next_task: asyncio.Future = None
|
||||
self._next_timer: asyncio.TimerHandle = None
|
||||
self._lock = asyncio.Lock()
|
||||
self._running = False
|
||||
self._running_pings: typing.Set[asyncio.Task] = set()
|
||||
|
||||
@property
|
||||
def running(self):
|
||||
return self._running
|
||||
|
||||
async def enqueue_maybe_ping(self, *peers: 'KademliaPeer', delay: typing.Optional[float] = None):
|
||||
delay = constants.check_refresh_interval if delay is None else delay
|
||||
async with self._lock:
|
||||
for peer in peers:
|
||||
if delay and peer not in self._enqueued_contacts:
|
||||
self._pending_contacts[peer] = self._loop.time() + delay
|
||||
elif peer not in self._enqueued_contacts:
|
||||
self._enqueued_contacts.append(peer)
|
||||
if peer in self._pending_contacts:
|
||||
del self._pending_contacts[peer]
|
||||
def enqueue_maybe_ping(self, *peers: 'KademliaPeer', delay: float = constants.maybe_ping_delay):
|
||||
now = self._loop.time()
|
||||
for peer in peers:
|
||||
if peer not in self._pending_contacts or now + delay < self._pending_contacts[peer]:
|
||||
self._pending_contacts[peer] = delay + now
|
||||
|
||||
async def _process(self):
|
||||
async def _ping(p: 'KademliaPeer'):
|
||||
def maybe_ping(self, peer: 'KademliaPeer'):
|
||||
async def ping_task():
|
||||
try:
|
||||
if self._protocol.peer_manager.peer_is_good(p):
|
||||
await self._protocol.add_peer(p)
|
||||
if self._protocol.peer_manager.peer_is_good(peer):
|
||||
if peer not in self._protocol.routing_table.get_peers():
|
||||
await self._protocol.add_peer(peer)
|
||||
return
|
||||
await self._protocol.get_rpc_peer(p).ping()
|
||||
except TimeoutError:
|
||||
await self._protocol.get_rpc_peer(peer).ping()
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
task = self._loop.create_task(ping_task())
|
||||
task.add_done_callback(lambda _: None if task not in self._running_pings else self._running_pings.remove(task))
|
||||
self._running_pings.add(task)
|
||||
|
||||
async def _process(self): # send up to 1 ping per second
|
||||
while True:
|
||||
tasks = []
|
||||
|
||||
async with self._lock:
|
||||
if self._enqueued_contacts or self._pending_contacts:
|
||||
now = self._loop.time()
|
||||
scheduled = [k for k, d in self._pending_contacts.items() if now >= d]
|
||||
for k in scheduled:
|
||||
del self._pending_contacts[k]
|
||||
if k not in self._enqueued_contacts:
|
||||
self._enqueued_contacts.append(k)
|
||||
while self._enqueued_contacts:
|
||||
peer = self._enqueued_contacts.pop()
|
||||
tasks.append(self._loop.create_task(_ping(peer)))
|
||||
if tasks:
|
||||
await asyncio.wait(tasks, loop=self._loop)
|
||||
|
||||
f = self._loop.create_future()
|
||||
self._loop.call_later(1.0, lambda: None if f.done() else f.set_result(None))
|
||||
await f
|
||||
enqueued = list(self._pending_contacts.keys())
|
||||
now = self._loop.time()
|
||||
for peer in enqueued:
|
||||
if self._pending_contacts[peer] <= now:
|
||||
del self._pending_contacts[peer]
|
||||
self.maybe_ping(peer)
|
||||
break
|
||||
await asyncio.sleep(1, loop=self._loop)
|
||||
|
||||
def start(self):
|
||||
assert not self._running
|
||||
@@ -254,17 +240,14 @@ class PingQueue:
|
||||
if self._process_task:
|
||||
self._process_task.cancel()
|
||||
self._process_task = None
|
||||
if self._next_task:
|
||||
self._next_task.cancel()
|
||||
self._next_task = None
|
||||
if self._next_timer:
|
||||
self._next_timer.cancel()
|
||||
self._next_timer = None
|
||||
while self._running_pings:
|
||||
self._running_pings.pop().cancel()
|
||||
|
||||
|
||||
class KademliaProtocol(DatagramProtocol):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', node_id: bytes, external_ip: str,
|
||||
udp_port: int, peer_port: int, rpc_timeout: float = 5.0):
|
||||
udp_port: int, peer_port: int, rpc_timeout: float = constants.rpc_timeout,
|
||||
split_buckets_under_index: int = constants.split_buckets_under_index):
|
||||
self.peer_manager = peer_manager
|
||||
self.loop = loop
|
||||
self.node_id = node_id
|
||||
@@ -279,11 +262,10 @@ class KademliaProtocol(DatagramProtocol):
|
||||
self.transport: DatagramTransport = None
|
||||
self.old_token_secret = constants.generate_id()
|
||||
self.token_secret = constants.generate_id()
|
||||
self.routing_table = TreeRoutingTable(self.loop, self.peer_manager, self.node_id)
|
||||
self.routing_table = TreeRoutingTable(self.loop, self.peer_manager, self.node_id, split_buckets_under_index)
|
||||
self.data_store = DictDataStore(self.loop, self.peer_manager)
|
||||
self.ping_queue = PingQueue(self.loop, self)
|
||||
self.node_rpc = KademliaRPC(self, self.loop, self.peer_port)
|
||||
self.lock = asyncio.Lock(loop=self.loop)
|
||||
self.rpc_timeout = rpc_timeout
|
||||
self._split_lock = asyncio.Lock(loop=self.loop)
|
||||
|
||||
@@ -317,9 +299,14 @@ class KademliaProtocol(DatagramProtocol):
|
||||
return args, {}
|
||||
|
||||
async def _add_peer(self, peer: 'KademliaPeer'):
|
||||
for p in self.routing_table.get_peers():
|
||||
if (p.address, p.udp_port) == (peer.address, peer.udp_port) and p.node_id != peer.node_id:
|
||||
self.routing_table.remove_peer(p)
|
||||
self.routing_table.join_buckets()
|
||||
bucket_index = self.routing_table.kbucket_index(peer.node_id)
|
||||
if self.routing_table.buckets[bucket_index].add_peer(peer):
|
||||
return True
|
||||
|
||||
# The bucket is full; see if it can be split (by checking if its range includes the host node's node_id)
|
||||
if self.routing_table.should_split(bucket_index, peer.node_id):
|
||||
self.routing_table.split_bucket(bucket_index)
|
||||
@@ -413,22 +400,32 @@ class KademliaProtocol(DatagramProtocol):
|
||||
sender_contact, ResponseDatagram(RESPONSE_TYPE, message.rpc_id, self.node_id, result),
|
||||
)
|
||||
|
||||
async def handle_request_datagram(self, address, request_datagram: RequestDatagram):
|
||||
async def handle_request_datagram(self, address: typing.Tuple[str, int], request_datagram: RequestDatagram):
|
||||
# This is an RPC method request
|
||||
await self.peer_manager.report_last_requested(address[0], address[1])
|
||||
await self.peer_manager.update_contact_triple(request_datagram.node_id, address[0], address[1])
|
||||
# only add a requesting contact to the routing table if it has replied to one of our requests
|
||||
peer = self.peer_manager.get_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
self.peer_manager.report_last_requested(address[0], address[1])
|
||||
try:
|
||||
peer = self.routing_table.get_peer(request_datagram.node_id)
|
||||
except IndexError:
|
||||
peer = self.peer_manager.get_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
try:
|
||||
await self._handle_rpc(peer, request_datagram)
|
||||
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
|
||||
# will be added to our routing table if successful
|
||||
is_good = self.peer_manager.peer_is_good(peer)
|
||||
if is_good is None:
|
||||
await self.ping_queue.enqueue_maybe_ping(peer)
|
||||
self.ping_queue.enqueue_maybe_ping(peer)
|
||||
# only add a requesting contact to the routing table if it has replied to one of our requests
|
||||
elif is_good is True:
|
||||
await self.add_peer(peer)
|
||||
|
||||
except ValueError as err:
|
||||
log.debug("error raised handling %s request from %s:%i - %s(%s)",
|
||||
request_datagram.method, peer.address, peer.udp_port, str(type(err)),
|
||||
str(err))
|
||||
await self.send_error(
|
||||
peer,
|
||||
ErrorDatagram(ERROR_TYPE, request_datagram.rpc_id, self.node_id, str(type(err)).encode(),
|
||||
str(err).encode())
|
||||
)
|
||||
except Exception as err:
|
||||
log.warning("error raised handling %s request from %s:%i - %s(%s)",
|
||||
request_datagram.method, peer.address, peer.udp_port, str(type(err)),
|
||||
@@ -445,11 +442,10 @@ class KademliaProtocol(DatagramProtocol):
|
||||
peer, df, request = self.sent_messages[response_datagram.rpc_id]
|
||||
if peer.address != address[0]:
|
||||
df.set_exception(RemoteException(
|
||||
f"response from {address[0]}:{address[1]}, "
|
||||
f"expected {peer.address}:{peer.udp_port}")
|
||||
f"response from {address[0]}, expected {peer.address}")
|
||||
)
|
||||
return
|
||||
peer.set_id(response_datagram.node_id)
|
||||
|
||||
# We got a result from the RPC
|
||||
if peer.node_id == self.node_id:
|
||||
df.set_exception(RemoteException("node has our node id"))
|
||||
@@ -457,8 +453,10 @@ class KademliaProtocol(DatagramProtocol):
|
||||
elif response_datagram.node_id == self.node_id:
|
||||
df.set_exception(RemoteException("incoming message is from our node id"))
|
||||
return
|
||||
await self.peer_manager.report_last_replied(address[0], address[1])
|
||||
await self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
|
||||
peer.set_id(response_datagram.node_id)
|
||||
peer.update_udp_port(address[1])
|
||||
self.peer_manager.report_last_replied(address[0], address[1])
|
||||
self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
|
||||
if not df.cancelled():
|
||||
df.set_result(response_datagram)
|
||||
await self.add_peer(peer)
|
||||
@@ -475,21 +473,23 @@ class KademliaProtocol(DatagramProtocol):
|
||||
remote_exception = RemoteException(f"{error_datagram.exception_type}({error_datagram.response})")
|
||||
if error_datagram.rpc_id in self.sent_messages:
|
||||
peer, df, request = self.sent_messages.pop(error_datagram.rpc_id)
|
||||
|
||||
if (peer.address, peer.udp_port) != address:
|
||||
df.set_exception(RemoteException(
|
||||
f"response from {address[0]}:{address[1]}, "
|
||||
f"expected {peer.address}:{peer.udp_port}")
|
||||
)
|
||||
return
|
||||
error_msg = f"" \
|
||||
f"Error sending '{request.method}' to {peer.address}:{peer.udp_port}\n" \
|
||||
f"Args: {request.args}\n" \
|
||||
f"Raised: {str(remote_exception)}"
|
||||
if error_datagram.response not in old_protocol_errors:
|
||||
if 'Invalid token' in error_msg:
|
||||
log.debug(error_msg)
|
||||
elif error_datagram.response not in old_protocol_errors:
|
||||
log.warning(error_msg)
|
||||
else:
|
||||
log.warning("known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)",
|
||||
peer.address, peer.udp_port, old_protocol_errors[error_datagram.response])
|
||||
|
||||
# reject replies coming from a different address than what we sent our request to
|
||||
if (peer.address, peer.udp_port) != address:
|
||||
log.error("node id mismatch in reply")
|
||||
remote_exception = TimeoutError(peer.node_id)
|
||||
df.set_exception(remote_exception)
|
||||
return
|
||||
else:
|
||||
@@ -505,7 +505,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
try:
|
||||
message = decode_datagram(datagram)
|
||||
except (ValueError, TypeError):
|
||||
self.loop.create_task(self.peer_manager.report_failure(address[0], address[1]))
|
||||
self.peer_manager.report_failure(address[0], address[1])
|
||||
log.warning("Couldn't decode dht datagram from %s: %s", address, binascii.hexlify(datagram).decode())
|
||||
return
|
||||
|
||||
@@ -522,12 +522,14 @@ class KademliaProtocol(DatagramProtocol):
|
||||
response_fut = self.sent_messages[request.rpc_id][1]
|
||||
try:
|
||||
response = await asyncio.wait_for(response_fut, self.rpc_timeout)
|
||||
await self.peer_manager.report_last_replied(peer.address, peer.udp_port)
|
||||
self.peer_manager.report_last_replied(peer.address, peer.udp_port)
|
||||
return response
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
await self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
if self.peer_manager.peer_is_good(peer) is False:
|
||||
self.routing_table.remove_peer(peer)
|
||||
async with self._split_lock:
|
||||
self.routing_table.remove_peer(peer)
|
||||
self.routing_table.join_buckets()
|
||||
raise
|
||||
|
||||
async def send_response(self, peer: 'KademliaPeer', response: ResponseDatagram):
|
||||
@@ -554,30 +556,29 @@ class KademliaProtocol(DatagramProtocol):
|
||||
if message.rpc_id in self.sent_messages:
|
||||
self.sent_messages.pop(message.rpc_id)
|
||||
|
||||
async with self.lock:
|
||||
if isinstance(message, RequestDatagram):
|
||||
response_fut = self.loop.create_future()
|
||||
response_fut.add_done_callback(pop_from_sent_messages)
|
||||
self.sent_messages[message.rpc_id] = (peer, response_fut, message)
|
||||
try:
|
||||
self.transport.sendto(data, (peer.address, peer.udp_port))
|
||||
except OSError as err:
|
||||
# TODO: handle ENETUNREACH
|
||||
if err.errno == socket.EWOULDBLOCK:
|
||||
# i'm scared this may swallow important errors, but i get a million of these
|
||||
# on Linux and it doesn't seem to affect anything -grin
|
||||
log.warning("Can't send data to dht: EWOULDBLOCK")
|
||||
else:
|
||||
log.error("DHT socket error sending %i bytes to %s:%i - %s (code %i)",
|
||||
len(data), peer.address, peer.udp_port, str(err), err.errno)
|
||||
if isinstance(message, RequestDatagram):
|
||||
self.sent_messages[message.rpc_id][1].set_exception(err)
|
||||
else:
|
||||
raise err
|
||||
if isinstance(message, RequestDatagram):
|
||||
await self.peer_manager.report_last_sent(peer.address, peer.udp_port)
|
||||
response_fut = self.loop.create_future()
|
||||
response_fut.add_done_callback(pop_from_sent_messages)
|
||||
self.sent_messages[message.rpc_id] = (peer, response_fut, message)
|
||||
try:
|
||||
self.transport.sendto(data, (peer.address, peer.udp_port))
|
||||
except OSError as err:
|
||||
# TODO: handle ENETUNREACH
|
||||
if err.errno == socket.EWOULDBLOCK:
|
||||
# i'm scared this may swallow important errors, but i get a million of these
|
||||
# on Linux and it doesn't seem to affect anything -grin
|
||||
log.warning("Can't send data to dht: EWOULDBLOCK")
|
||||
else:
|
||||
log.error("DHT socket error sending %i bytes to %s:%i - %s (code %i)",
|
||||
len(data), peer.address, peer.udp_port, str(err), err.errno)
|
||||
if isinstance(message, RequestDatagram):
|
||||
self.sent_messages[message.rpc_id][1].set_exception(err)
|
||||
else:
|
||||
raise err
|
||||
if isinstance(message, RequestDatagram):
|
||||
self.peer_manager.report_last_sent(peer.address, peer.udp_port)
|
||||
elif isinstance(message, ErrorDatagram):
|
||||
await self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
|
||||
def change_token(self):
|
||||
self.old_token_secret = self.token_secret
|
||||
@@ -601,7 +602,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
res = await self.get_rpc_peer(peer).store(hash_value)
|
||||
if res != b"OK":
|
||||
raise ValueError(res)
|
||||
log.info("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
log.debug("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
return peer.node_id, True
|
||||
except asyncio.TimeoutError:
|
||||
log.debug("Timeout while storing blob_hash %s at %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
@@ -609,7 +610,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
log.error("Unexpected response: %s" % err)
|
||||
except Exception as err:
|
||||
if 'Invalid token' in str(err):
|
||||
await self.peer_manager.clear_token(peer.node_id)
|
||||
self.peer_manager.clear_token(peer.node_id)
|
||||
else:
|
||||
log.exception("Unexpected error while storing blob_hash")
|
||||
return peer.node_id, False
|
||||
|
||||
@@ -29,6 +29,7 @@ class KBucket:
|
||||
self.range_max = range_max
|
||||
self.peers: typing.List['KademliaPeer'] = []
|
||||
self._node_id = node_id
|
||||
self._distance_to_self = Distance(node_id)
|
||||
|
||||
def add_peer(self, peer: 'KademliaPeer') -> bool:
|
||||
""" Add contact to _contact list in the right order. This will move the
|
||||
@@ -48,7 +49,14 @@ class KBucket:
|
||||
self.peers.remove(peer)
|
||||
self.peers.append(peer)
|
||||
return True
|
||||
elif len(self.peers) < constants.k:
|
||||
else:
|
||||
for i in range(len(self.peers)):
|
||||
p = self.peers[i]
|
||||
if p.node_id == peer.node_id:
|
||||
self.peers.remove(p)
|
||||
self.peers.append(peer)
|
||||
return True
|
||||
if len(self.peers) < constants.k:
|
||||
self.peers.append(peer)
|
||||
return True
|
||||
else:
|
||||
@@ -130,7 +138,7 @@ class KBucket:
|
||||
if not.
|
||||
@rtype: bool
|
||||
"""
|
||||
return self.range_min <= int.from_bytes(key, 'big') < self.range_max
|
||||
return self.range_min <= self._distance_to_self(key) < self.range_max
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.peers)
|
||||
@@ -155,10 +163,12 @@ class TreeRoutingTable:
|
||||
that paper.
|
||||
"""
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', parent_node_id: bytes):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', parent_node_id: bytes,
|
||||
split_buckets_under_index: int = constants.split_buckets_under_index):
|
||||
self._loop = loop
|
||||
self._peer_manager = peer_manager
|
||||
self._parent_node_id = parent_node_id
|
||||
self._split_buckets_under_index = split_buckets_under_index
|
||||
self.buckets: typing.List[KBucket] = [
|
||||
KBucket(
|
||||
self._peer_manager, range_min=0, range_max=2 ** constants.hash_bits, node_id=self._parent_node_id
|
||||
@@ -170,7 +180,7 @@ class TreeRoutingTable:
|
||||
|
||||
def should_split(self, bucket_index: int, to_add: bytes) -> bool:
|
||||
# https://stackoverflow.com/questions/32129978/highly-unbalanced-kademlia-routing-table/32187456#32187456
|
||||
if self.buckets[bucket_index].key_in_range(self._parent_node_id):
|
||||
if bucket_index < self._split_buckets_under_index:
|
||||
return True
|
||||
contacts = self.get_peers()
|
||||
distance = Distance(self._parent_node_id)
|
||||
@@ -183,8 +193,6 @@ class TreeRoutingTable:
|
||||
exclude = [self._parent_node_id]
|
||||
if sender_node_id:
|
||||
exclude.append(sender_node_id)
|
||||
if key in exclude:
|
||||
exclude.remove(key)
|
||||
count = count or constants.k
|
||||
distance = Distance(key)
|
||||
contacts = self.get_peers()
|
||||
@@ -238,11 +246,15 @@ class TreeRoutingTable:
|
||||
|
||||
def random_id_in_bucket_range(self, bucket_index: int) -> bytes:
|
||||
random_id = int(random.randrange(self.buckets[bucket_index].range_min, self.buckets[bucket_index].range_max))
|
||||
return random_id.to_bytes(constants.hash_length, 'big')
|
||||
return Distance(
|
||||
self._parent_node_id
|
||||
)(random_id.to_bytes(constants.hash_length, 'big')).to_bytes(constants.hash_length, 'big')
|
||||
|
||||
def midpoint_id_in_bucket_range(self, bucket_index: int) -> bytes:
|
||||
half = int((self.buckets[bucket_index].range_max - self.buckets[bucket_index].range_min) // 2)
|
||||
return int(self.buckets[bucket_index].range_min + half).to_bytes(constants.hash_length, 'big')
|
||||
return Distance(self._parent_node_id)(
|
||||
int(self.buckets[bucket_index].range_min + half).to_bytes(constants.hash_length, 'big')
|
||||
).to_bytes(constants.hash_length, 'big')
|
||||
|
||||
def split_bucket(self, old_bucket_index: int) -> None:
|
||||
""" Splits the specified k-bucket into two new buckets which together
|
||||
@@ -269,6 +281,8 @@ class TreeRoutingTable:
|
||||
old_bucket.remove_peer(contact)
|
||||
|
||||
def join_buckets(self):
|
||||
if len(self.buckets) == 1:
|
||||
return
|
||||
to_pop = [i for i, bucket in enumerate(self.buckets) if not len(bucket)]
|
||||
if not to_pop:
|
||||
return
|
||||
|
||||
@@ -67,8 +67,7 @@ class RequestDatagram(KademliaDatagramBase):
|
||||
def make_ping(cls, from_node_id: bytes, rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
|
||||
if rpc_id and len(rpc_id) != constants.rpc_id_length:
|
||||
raise ValueError("invalid rpc id length")
|
||||
elif not rpc_id:
|
||||
rpc_id = constants.generate_id()[:constants.rpc_id_length]
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(from_node_id) != constants.hash_bits // 8:
|
||||
raise ValueError("invalid node id")
|
||||
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'ping')
|
||||
|
||||
+1
-2
@@ -28,8 +28,7 @@ class DownloadTimeoutError(Exception):
|
||||
|
||||
class DownloadDataTimeout(Exception):
|
||||
def __init__(self, download):
|
||||
super().__init__('Failed to download data blobs for sd hash '
|
||||
'{} within timeout'.format(download))
|
||||
super().__init__(f'Failed to download data blobs for sd hash {download} within timeout')
|
||||
self.download = download
|
||||
|
||||
|
||||
|
||||
+29
-2
@@ -1,4 +1,7 @@
|
||||
import os
|
||||
import sys
|
||||
import signal
|
||||
import pathlib
|
||||
import json
|
||||
import asyncio
|
||||
import argparse
|
||||
@@ -216,12 +219,19 @@ def get_argument_parser():
|
||||
return main
|
||||
|
||||
|
||||
def ensure_directory_exists(path: str):
|
||||
if not os.path.isdir(path):
|
||||
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
argv = argv or sys.argv[1:]
|
||||
parser = get_argument_parser()
|
||||
args, command_args = parser.parse_known_args(argv)
|
||||
|
||||
conf = Config.create_from_arguments(args)
|
||||
for directory in (conf.data_dir, conf.download_dir, conf.wallet_dir):
|
||||
ensure_directory_exists(directory)
|
||||
|
||||
if args.cli_version:
|
||||
print(f"{lbrynet_name} {lbrynet_version}")
|
||||
@@ -238,17 +248,23 @@ def main(argv=None):
|
||||
)
|
||||
file_handler.setFormatter(default_formatter)
|
||||
log.addHandler(file_handler)
|
||||
logging.getLogger('torba').addHandler(file_handler)
|
||||
|
||||
if not args.quiet:
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(default_formatter)
|
||||
log.addHandler(handler)
|
||||
logging.getLogger('torba').addHandler(handler)
|
||||
logging.getLogger('torba').setLevel(logging.INFO)
|
||||
|
||||
logging.getLogger('aioupnp').setLevel(logging.WARNING)
|
||||
logging.getLogger('aiohttp').setLevel(logging.CRITICAL)
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
if args.verbose:
|
||||
log.setLevel(logging.DEBUG)
|
||||
loop.set_debug(True)
|
||||
else:
|
||||
log.setLevel(logging.INFO)
|
||||
if conf.share_usage_data:
|
||||
@@ -257,10 +273,21 @@ def main(argv=None):
|
||||
log.addHandler(loggly_handler)
|
||||
|
||||
daemon = Daemon(conf)
|
||||
loop = asyncio.get_event_loop()
|
||||
started = False
|
||||
def __exit():
|
||||
if started:
|
||||
daemon.stop_event.set()
|
||||
else:
|
||||
raise GracefulExit()
|
||||
try:
|
||||
loop.add_signal_handler(signal.SIGINT, __exit)
|
||||
loop.add_signal_handler(signal.SIGTERM, __exit)
|
||||
except NotImplementedError:
|
||||
pass # Not implemented on Windows
|
||||
try:
|
||||
loop.run_until_complete(daemon.start())
|
||||
loop.run_forever()
|
||||
started = True
|
||||
loop.run_until_complete(daemon.stop_event.wait())
|
||||
except (GracefulExit, KeyboardInterrupt):
|
||||
pass
|
||||
finally:
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
import os
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import logging
|
||||
import math
|
||||
import binascii
|
||||
import typing
|
||||
import socket
|
||||
from hashlib import sha256
|
||||
from types import SimpleNamespace
|
||||
import base58
|
||||
@@ -18,7 +16,6 @@ import lbrynet.schema
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import HEADERS_FILE_SHA256_CHECKSUM
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.dht.blob_announcer import BlobAnnouncer
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob_exchange.server import BlobServer
|
||||
@@ -54,25 +51,6 @@ async def gather_dict(tasks: dict):
|
||||
)))
|
||||
|
||||
|
||||
async def get_external_ip(): # used if upnp is disabled or non-functioning
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get("https://api.lbry.io/ip") as resp:
|
||||
response = await resp.json()
|
||||
if response['success']:
|
||||
return response['data']['ip']
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
|
||||
async def resolve_host(loop: asyncio.BaseEventLoop, url: str):
|
||||
info = await loop.getaddrinfo(
|
||||
url, 'https',
|
||||
proto=socket.IPPROTO_TCP,
|
||||
)
|
||||
return info[0][4][0]
|
||||
|
||||
|
||||
class DatabaseComponent(Component):
|
||||
component_name = DATABASE_COMPONENT
|
||||
|
||||
@@ -116,7 +94,7 @@ class DatabaseComponent(Component):
|
||||
from lbrynet.extras.daemon.migrator import dbmigrator
|
||||
log.info("Upgrading your databases (revision %i to %i)", old_revision, self.get_current_db_revision())
|
||||
await asyncio.get_event_loop().run_in_executor(
|
||||
None, dbmigrator.migrate_db, self.conf.data_dir, old_revision, self.get_current_db_revision()
|
||||
None, dbmigrator.migrate_db, self.conf, old_revision, self.get_current_db_revision()
|
||||
)
|
||||
self._write_db_revision_file(self.get_current_db_revision())
|
||||
log.info("Finished upgrading the databases.")
|
||||
@@ -157,40 +135,31 @@ class HeadersComponent(Component):
|
||||
}
|
||||
|
||||
async def fetch_headers_from_s3(self):
|
||||
def collector(d, h_file):
|
||||
h_file.write(d)
|
||||
local_size = float(h_file.tell())
|
||||
final_size = float(final_size_after_download)
|
||||
self._headers_progress_percent = math.ceil(local_size / final_size * 100)
|
||||
|
||||
local_header_size = self.local_header_file_size()
|
||||
resume_header = {"Range": f"bytes={local_header_size}-"}
|
||||
async with aiohttp.request('get', HEADERS_URL, headers=resume_header) as response:
|
||||
got_406 = response.status == 406 # our file is bigger
|
||||
final_size_after_download = response.content_length + local_header_size
|
||||
if got_406:
|
||||
async with utils.aiohttp_request('get', HEADERS_URL, headers=resume_header) as response:
|
||||
if response.status == 406 or response.content_length < HEADER_SIZE: # our file is bigger
|
||||
log.warning("s3 is more out of date than we are")
|
||||
# should have something to download and a final length divisible by the header size
|
||||
elif final_size_after_download and not final_size_after_download % HEADER_SIZE:
|
||||
s3_height = (final_size_after_download / HEADER_SIZE) - 1
|
||||
local_height = self.local_header_file_height()
|
||||
if s3_height > local_height:
|
||||
data = await response.read()
|
||||
|
||||
if local_header_size:
|
||||
log.info("Resuming download of %i bytes from s3", response.content_length)
|
||||
with open(self.headers_file, "a+b") as headers_file:
|
||||
collector(data, headers_file)
|
||||
else:
|
||||
with open(self.headers_file, "wb") as headers_file:
|
||||
collector(data, headers_file)
|
||||
log.info("fetched headers from s3 (s3 height: %i), now verifying integrity after download.",
|
||||
s3_height)
|
||||
self._check_header_file_integrity()
|
||||
else:
|
||||
log.warning("s3 is more out of date than we are")
|
||||
else:
|
||||
log.error("invalid size for headers from s3")
|
||||
return
|
||||
if response.content_length % HEADER_SIZE != 0:
|
||||
log.warning("s3 appears to have corrupted header")
|
||||
return
|
||||
final_size_after_download = response.content_length + local_header_size
|
||||
write_mode = "wb"
|
||||
if local_header_size > 0:
|
||||
log.info("Resuming download of %i bytes from s3", response.content_length)
|
||||
write_mode = "a+b"
|
||||
with open(self.headers_file, write_mode) as fd:
|
||||
while True:
|
||||
chunk = await response.content.read(512)
|
||||
if not chunk:
|
||||
break
|
||||
fd.write(chunk)
|
||||
self._headers_progress_percent = math.ceil(
|
||||
float(fd.tell()) / float(final_size_after_download) * 100
|
||||
)
|
||||
log.info("fetched headers from s3, now verifying integrity after download.")
|
||||
self._check_header_file_integrity()
|
||||
|
||||
def local_header_file_height(self):
|
||||
return max((self.local_header_file_size() / HEADER_SIZE) - 1, 0)
|
||||
@@ -331,9 +300,7 @@ class BlobComponent(Component):
|
||||
return await self.blob_manager.setup()
|
||||
|
||||
async def stop(self):
|
||||
while self.blob_manager and self.blob_manager.blobs:
|
||||
_, blob = self.blob_manager.blobs.popitem()
|
||||
await blob.close()
|
||||
self.blob_manager.stop()
|
||||
|
||||
async def get_status(self):
|
||||
count = 0
|
||||
@@ -359,7 +326,7 @@ class DHTComponent(Component):
|
||||
|
||||
async def get_status(self):
|
||||
return {
|
||||
'node_id': binascii.hexlify(self.dht_node.protocol.node_id),
|
||||
'node_id': None if not self.dht_node else binascii.hexlify(self.dht_node.protocol.node_id),
|
||||
'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.protocol.routing_table.get_peers())
|
||||
}
|
||||
|
||||
@@ -376,12 +343,12 @@ class DHTComponent(Component):
|
||||
async def start(self):
|
||||
log.info("start the dht")
|
||||
self.upnp_component = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.peer_port)
|
||||
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.dht_node_port)
|
||||
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.tcp_port)
|
||||
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.udp_port)
|
||||
external_ip = self.upnp_component.external_ip
|
||||
if not external_ip:
|
||||
log.warning("UPnP component failed to get external ip")
|
||||
external_ip = await get_external_ip()
|
||||
external_ip = await utils.get_external_ip()
|
||||
if not external_ip:
|
||||
log.warning("failed to get external ip")
|
||||
|
||||
@@ -389,13 +356,15 @@ class DHTComponent(Component):
|
||||
asyncio.get_event_loop(),
|
||||
self.component_manager.peer_manager,
|
||||
node_id=self.get_node_id(),
|
||||
internal_udp_port=self.conf.dht_node_port,
|
||||
internal_udp_port=self.conf.udp_port,
|
||||
udp_port=self.external_udp_port,
|
||||
external_ip=external_ip,
|
||||
peer_port=self.external_peer_port
|
||||
peer_port=self.external_peer_port,
|
||||
rpc_timeout=self.conf.node_rpc_timeout,
|
||||
split_buckets_under_index=self.conf.split_buckets_under_index
|
||||
)
|
||||
self.dht_node.start(
|
||||
interface='0.0.0.0', known_node_urls=self.conf.known_dht_nodes
|
||||
interface=self.conf.network_interface, known_node_urls=self.conf.known_dht_nodes
|
||||
)
|
||||
log.info("Started the dht")
|
||||
|
||||
@@ -419,7 +388,7 @@ class HashAnnouncerComponent(Component):
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
self.hash_announcer = BlobAnnouncer(asyncio.get_event_loop(), dht_node, storage)
|
||||
self.hash_announcer.start(self.conf.concurrent_announcers)
|
||||
self.hash_announcer.start(self.conf.concurrent_blob_announcers)
|
||||
log.info("Started blob announcer")
|
||||
|
||||
async def stop(self):
|
||||
@@ -462,17 +431,13 @@ class StreamManagerComponent(Component):
|
||||
log.info('Starting the file manager')
|
||||
loop = asyncio.get_event_loop()
|
||||
self.stream_manager = StreamManager(
|
||||
loop, blob_manager, wallet, storage, node, self.conf.blob_download_timeout,
|
||||
self.conf.peer_connect_timeout, [
|
||||
KademliaPeer(loop, address=(await resolve_host(loop, url)), tcp_port=port + 1)
|
||||
for url, port in self.conf.reflector_servers
|
||||
], self.conf.reflector_servers
|
||||
loop, self.conf, blob_manager, wallet, storage, node, self.component_manager.analytics_manager
|
||||
)
|
||||
await self.stream_manager.start()
|
||||
log.info('Done setting up file manager')
|
||||
|
||||
async def stop(self):
|
||||
await self.stream_manager.stop()
|
||||
self.stream_manager.stop()
|
||||
|
||||
|
||||
class PeerProtocolServerComponent(Component):
|
||||
@@ -492,10 +457,10 @@ class PeerProtocolServerComponent(Component):
|
||||
upnp = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
blob_manager: BlobFileManager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
wallet: LbryWalletManager = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
peer_port = upnp.upnp_redirects.get("TCP", self.conf.peer_port)
|
||||
peer_port = self.conf.tcp_port
|
||||
address = await wallet.get_unused_address()
|
||||
self.blob_server = BlobServer(asyncio.get_event_loop(), blob_manager, address)
|
||||
self.blob_server.start_server(peer_port, interface='0.0.0.0')
|
||||
self.blob_server.start_server(peer_port, interface=self.conf.network_interface)
|
||||
await self.blob_server.started_listening.wait()
|
||||
|
||||
async def stop(self):
|
||||
@@ -508,8 +473,8 @@ class UPnPComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self._int_peer_port = self.conf.peer_port
|
||||
self._int_dht_node_port = self.conf.dht_node_port
|
||||
self._int_peer_port = self.conf.tcp_port
|
||||
self._int_dht_node_port = self.conf.udp_port
|
||||
self.use_upnp = self.conf.use_upnp
|
||||
self.upnp = None
|
||||
self.upnp_redirects = {}
|
||||
@@ -548,7 +513,7 @@ class UPnPComponent(Component):
|
||||
|
||||
if external_ip == "0.0.0.0" or not external_ip:
|
||||
log.warning("unable to get external ip from UPnP, checking lbry.io fallback")
|
||||
external_ip = await get_external_ip()
|
||||
external_ip = await utils.get_external_ip()
|
||||
if self.external_ip and self.external_ip != external_ip:
|
||||
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
|
||||
self.external_ip = external_ip
|
||||
@@ -599,7 +564,7 @@ class UPnPComponent(Component):
|
||||
async def start(self):
|
||||
log.info("detecting external ip")
|
||||
if not self.use_upnp:
|
||||
self.external_ip = await get_external_ip()
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
return
|
||||
success = False
|
||||
await self._maintain_redirects()
|
||||
@@ -614,7 +579,9 @@ class UPnPComponent(Component):
|
||||
else:
|
||||
log.error("failed to setup upnp")
|
||||
if self.component_manager.analytics_manager:
|
||||
self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, await self.get_status())
|
||||
await self.component_manager.analytics_manager.send_upnp_setup_success_fail(
|
||||
success, await self.get_status()
|
||||
)
|
||||
self._maintain_redirects_task = asyncio.create_task(self._repeatedly_maintain_redirects(now=False))
|
||||
|
||||
async def stop(self):
|
||||
@@ -640,7 +607,7 @@ class ExchangeRateManagerComponent(Component):
|
||||
component_name = EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
Component.__init__(self, component_manager)
|
||||
super().__init__(component_manager)
|
||||
self.exchange_rate_manager = ExchangeRateManager()
|
||||
|
||||
@property
|
||||
|
||||
+228
-151
@@ -2,25 +2,27 @@ import os
|
||||
import asyncio
|
||||
import logging
|
||||
import json
|
||||
import time
|
||||
import inspect
|
||||
import typing
|
||||
import aiohttp
|
||||
import base58
|
||||
import random
|
||||
from urllib.parse import urlencode, quote
|
||||
from typing import Callable, Optional, List
|
||||
from binascii import hexlify, unhexlify
|
||||
from copy import deepcopy
|
||||
from traceback import format_exc
|
||||
from aiohttp import web
|
||||
from functools import wraps
|
||||
from torba.client.wallet import Wallet
|
||||
from torba.client.baseaccount import SingleKey, HierarchicalDeterministic
|
||||
|
||||
from lbrynet import __version__, utils
|
||||
from lbrynet.conf import Config, Setting, SLACK_WEBHOOK
|
||||
from lbrynet.blob.blob_file import is_valid_blobhash
|
||||
from lbrynet.blob_exchange.downloader import BlobDownloader
|
||||
from lbrynet.blob_exchange.downloader import download_blob
|
||||
from lbrynet.error import InsufficientFundsError, DownloadSDTimeout, ComponentsNotStarted
|
||||
from lbrynet.error import NullFundsError, NegativeFundsError, ResolveError, ComponentStartConditionNotMet
|
||||
from lbrynet.error import NullFundsError, NegativeFundsError, ComponentStartConditionNotMet
|
||||
from lbrynet.extras import system_info
|
||||
from lbrynet.extras.daemon import analytics
|
||||
from lbrynet.extras.daemon.Components import WALLET_COMPONENT, DATABASE_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT
|
||||
@@ -29,6 +31,7 @@ from lbrynet.extras.daemon.Components import EXCHANGE_RATE_MANAGER_COMPONENT, UP
|
||||
from lbrynet.extras.daemon.ComponentManager import RequiredCondition
|
||||
from lbrynet.extras.daemon.ComponentManager import ComponentManager
|
||||
from lbrynet.extras.daemon.json_response_encoder import JSONResponseEncoder
|
||||
from lbrynet.extras.daemon.mime_types import guess_media_type
|
||||
from lbrynet.extras.daemon.undecorated import undecorated
|
||||
from lbrynet.extras.wallet.account import Account as LBCAccount
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
||||
@@ -249,12 +252,14 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
self._node_id = None
|
||||
self._installation_id = None
|
||||
self.session_id = base58.b58encode(utils.generate_id()).decode()
|
||||
self.analytics_manager = analytics.Manager(conf, self.installation_id, self.session_id)
|
||||
self.analytics_manager = analytics.AnalyticsManager(conf, self.installation_id, self.session_id)
|
||||
self.component_manager = component_manager or ComponentManager(
|
||||
conf, analytics_manager=self.analytics_manager,
|
||||
skip_components=conf.components_to_skip or []
|
||||
)
|
||||
self.component_startup_task = None
|
||||
self._connection_status: typing.Tuple[float, bool] = [self.component_manager.loop.time(), False]
|
||||
self.stop_event = asyncio.Event()
|
||||
|
||||
logging.getLogger('aiohttp.access').setLevel(logging.WARN)
|
||||
app = web.Application()
|
||||
@@ -367,6 +372,18 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
if not os.path.isdir(self.conf.download_dir):
|
||||
os.makedirs(self.conf.download_dir)
|
||||
|
||||
async def update_connection_status(self):
|
||||
connected = await utils.async_check_connection()
|
||||
self._connection_status = (self.component_manager.loop.time(), connected)
|
||||
|
||||
async def get_connection_status(self) -> str:
|
||||
if self._connection_status[0] + 300 > self.component_manager.loop.time():
|
||||
if not self._connection_status[1]:
|
||||
await self.update_connection_status()
|
||||
else:
|
||||
await self.update_connection_status()
|
||||
return CONNECTION_STATUS_CONNECTED if self._connection_status[1] else CONNECTION_STATUS_NETWORK
|
||||
|
||||
async def start(self):
|
||||
log.info("Starting LBRYNet Daemon")
|
||||
log.debug("Settings: %s", json.dumps(self.conf.settings_dict, indent=2))
|
||||
@@ -401,7 +418,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
self.ensure_wallet_dir()
|
||||
self.ensure_download_dir()
|
||||
if not self.analytics_manager.is_started:
|
||||
self.analytics_manager.start()
|
||||
await self.analytics_manager.start()
|
||||
self.component_startup_task = asyncio.create_task(self.component_manager.start())
|
||||
await self.component_startup_task
|
||||
|
||||
@@ -474,6 +491,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
if asyncio.iscoroutine(result):
|
||||
result = await result
|
||||
return result
|
||||
except asyncio.CancelledError:
|
||||
log.info("cancelled API call for: %s", function_name)
|
||||
raise
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
log.exception("error handling api request")
|
||||
return JSONRPCError(
|
||||
@@ -590,6 +610,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
(string) Shutdown message
|
||||
"""
|
||||
log.info("Shutting down lbrynet daemon")
|
||||
self.stop_event.set()
|
||||
return "Shutting down"
|
||||
|
||||
async def jsonrpc_status(self):
|
||||
@@ -609,19 +630,16 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
'is_running': (bool),
|
||||
'skipped_components': (list) [names of skipped components (str)],
|
||||
'startup_status': { Does not include components which have been skipped
|
||||
'database': (bool),
|
||||
'wallet': (bool),
|
||||
'session': (bool),
|
||||
'dht': (bool),
|
||||
'hash_announcer': (bool),
|
||||
'stream_identifier': (bool),
|
||||
'file_manager': (bool),
|
||||
'blob_manager': (bool),
|
||||
'blockchain_headers': (bool),
|
||||
'peer_protocol_server': (bool),
|
||||
'reflector': (bool),
|
||||
'upnp': (bool),
|
||||
'database': (bool),
|
||||
'dht': (bool),
|
||||
'exchange_rate_manager': (bool),
|
||||
'hash_announcer': (bool),
|
||||
'peer_protocol_server': (bool),
|
||||
'stream_manager': (bool),
|
||||
'upnp': (bool),
|
||||
'wallet': (bool),
|
||||
},
|
||||
'connection_status': {
|
||||
'code': (str) connection status code,
|
||||
@@ -648,8 +666,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
'hash_announcer': {
|
||||
'announce_queue_size': (int) number of blobs currently queued to be announced
|
||||
},
|
||||
'file_manager': {
|
||||
'managed_files': (int) count of files in the file manager,
|
||||
'stream_manager': {
|
||||
'managed_files': (int) count of files in the stream manager,
|
||||
},
|
||||
'upnp': {
|
||||
'aioupnp_version': (str),
|
||||
@@ -664,7 +682,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
}
|
||||
"""
|
||||
|
||||
connection_code = CONNECTION_STATUS_NETWORK
|
||||
connection_code = await self.get_connection_status()
|
||||
|
||||
response = {
|
||||
'installation_id': self.installation_id,
|
||||
'is_running': all(self.component_manager.get_components_status().values()),
|
||||
@@ -761,7 +780,10 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
Set daemon settings
|
||||
|
||||
Usage:
|
||||
settings_set <key> <value>
|
||||
settings_set (<key>) (<value>)
|
||||
|
||||
Options:
|
||||
None
|
||||
|
||||
Returns:
|
||||
(dict) Updated dictionary of daemon settings
|
||||
@@ -823,12 +845,12 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
amount = self.get_dewies_or_error("amount", amount)
|
||||
if not amount:
|
||||
raise NullFundsError
|
||||
elif amount < 0:
|
||||
if amount < 0:
|
||||
raise NegativeFundsError()
|
||||
|
||||
if address and claim_id:
|
||||
raise Exception("Given both an address and a claim id")
|
||||
elif not address and not claim_id:
|
||||
if not address and not claim_id:
|
||||
raise Exception("Not given an address or a claim id")
|
||||
|
||||
if address:
|
||||
@@ -1060,6 +1082,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
change_made = True
|
||||
|
||||
if change_made:
|
||||
account.modified_on = time.time()
|
||||
self.default_wallet.save()
|
||||
|
||||
result = account.to_dict()
|
||||
@@ -1218,7 +1241,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
amount = self.get_dewies_or_error("amount", amount)
|
||||
if not amount:
|
||||
raise NullFundsError
|
||||
elif amount < 0:
|
||||
if amount < 0:
|
||||
raise NegativeFundsError()
|
||||
|
||||
for address in addresses:
|
||||
@@ -1229,6 +1252,68 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
await self.analytics_manager.send_credits_sent()
|
||||
return result
|
||||
|
||||
SYNC_DOC = """
|
||||
Wallet synchronization.
|
||||
"""
|
||||
|
||||
@requires("wallet")
|
||||
def jsonrpc_sync_hash(self):
|
||||
"""
|
||||
Deterministic hash of the wallet.
|
||||
|
||||
Usage:
|
||||
sync hash
|
||||
|
||||
Options:
|
||||
|
||||
Returns:
|
||||
(str) sha256 hash of wallet
|
||||
"""
|
||||
return hexlify(self.default_wallet.hash).decode()
|
||||
|
||||
@requires("wallet")
|
||||
def jsonrpc_sync_apply(self, password, data=None, encrypt_password=None):
|
||||
"""
|
||||
Apply incoming synchronization data, if provided, and then produce a sync hash and
|
||||
an encrypted wallet.
|
||||
|
||||
Usage:
|
||||
sync apply <password> [--data=<data>] [--encrypt-password=<encrypt_password>]
|
||||
|
||||
Options:
|
||||
--password=<password> : (str) password to decrypt incoming and encrypt outgoing data
|
||||
--data=<data> : (str) incoming sync data, if any
|
||||
--encrypt-password=<encrypt_password> : (str) password to encrypt outgoing data if different
|
||||
from the decrypt password, used during password changes
|
||||
|
||||
Returns:
|
||||
(map) sync hash and data
|
||||
|
||||
"""
|
||||
if data is not None:
|
||||
decrypted_data = Wallet.unpack(password, data)
|
||||
for account_data in decrypted_data['accounts']:
|
||||
_, _, pubkey = LBCAccount.keys_from_dict(self.ledger, account_data)
|
||||
account_id = pubkey.address
|
||||
local_match = None
|
||||
for local_account in self.default_wallet.accounts:
|
||||
if account_id == local_account.id:
|
||||
local_match = local_account
|
||||
break
|
||||
if local_match is not None:
|
||||
local_match.apply(account_data)
|
||||
else:
|
||||
new_account = LBCAccount.from_dict(self.ledger, self.default_wallet, account_data)
|
||||
if self.ledger.network.is_connected:
|
||||
asyncio.create_task(self.ledger.subscribe_account(new_account))
|
||||
self.default_wallet.save()
|
||||
|
||||
encrypted = self.default_wallet.pack(encrypt_password or password)
|
||||
return {
|
||||
'hash': self.jsonrpc_sync_hash(),
|
||||
'data': encrypted.decode()
|
||||
}
|
||||
|
||||
ADDRESS_DOC = """
|
||||
Address management.
|
||||
"""
|
||||
@@ -1307,8 +1392,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
file_list [--sd_hash=<sd_hash>] [--file_name=<file_name>] [--stream_hash=<stream_hash>]
|
||||
[--rowid=<rowid>] [--claim_id=<claim_id>] [--outpoint=<outpoint>] [--txid=<txid>] [--nout=<nout>]
|
||||
[--channel_claim_id=<channel_claim_id>] [--channel_name=<channel_name>]
|
||||
[--claim_name=<claim_name>] [--sort=<sort_by>] [--reverse] [--comparison=<comparison>]
|
||||
[--full_status=<full_status>]
|
||||
[--claim_name=<claim_name>] [--blobs_in_stream=<blobs_in_stream>]
|
||||
[--blobs_remaining=<blobs_remaining>] [--sort=<sort_by>]
|
||||
[--comparison=<comparison>] [--full_status=<full_status>] [--reverse]
|
||||
|
||||
Options:
|
||||
--sd_hash=<sd_hash> : (str) get file with matching sd hash
|
||||
@@ -1321,11 +1407,12 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--txid=<txid> : (str) get file with matching claim txid
|
||||
--nout=<nout> : (int) get file with matching claim nout
|
||||
--channel_claim_id=<channel_claim_id> : (str) get file with matching channel claim id
|
||||
--channel_name=<channel_name> : (str) get file with matching channel name
|
||||
--channel_name=<channel_name> : (str) get file with matching channel name
|
||||
--claim_name=<claim_name> : (str) get file with matching claim name
|
||||
--sort=<sort_method> : (str) sort by any property, like 'file_name'
|
||||
or 'metadata.author'; to specify direction
|
||||
append ',asc' or ',desc'
|
||||
--blobs_in_stream<blobs_in_stream> : (int) get file with matching blobs in stream
|
||||
--blobs_remaining=<blobs_remaining> : (int) amount of remaining blobs to download
|
||||
--sort=<sort_by> : (str) field to sort by (one of the above filter fields)
|
||||
--comparison=<comparison> : (str) logical comparision, (eq | ne | g | ge | l | le)
|
||||
|
||||
Returns:
|
||||
(list) List of files
|
||||
@@ -1344,23 +1431,26 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
'download_path': (str) download path of file,
|
||||
'mime_type': (str) mime type of file,
|
||||
'key': (str) key attached to file,
|
||||
'total_bytes': (int) file size in bytes,
|
||||
'total_bytes_lower_bound': (int) lower bound file size in bytes,
|
||||
'total_bytes': (int) file upper bound size in bytes,
|
||||
'written_bytes': (int) written size in bytes,
|
||||
'blobs_completed': (int) number of fully downloaded blobs,
|
||||
'blobs_in_stream': (int) total blobs on stream,
|
||||
'blobs_remaining': (int) total blobs remaining to download,
|
||||
'status': (str) downloader status
|
||||
'claim_id': (str) None if claim is not found else the claim id,
|
||||
'outpoint': (str) None if claim is not found else the tx and output,
|
||||
'txid': (str) None if claim is not found else the transaction id,
|
||||
'nout': (int) None if claim is not found else the transaction output index,
|
||||
'outpoint': (str) None if claim is not found else the tx and output,
|
||||
'metadata': (dict) None if claim is not found else the claim metadata,
|
||||
'channel_claim_id': (str) None if claim is not found or not signed,
|
||||
'channel_name': (str) None if claim is not found or not signed,
|
||||
'claim_name': (str) None if claim is not found else the claim name
|
||||
},
|
||||
]
|
||||
}
|
||||
"""
|
||||
sort = sort or 'status'
|
||||
sort = sort or 'rowid'
|
||||
comparison = comparison or 'eq'
|
||||
return [
|
||||
stream.as_dict() for stream in self.stream_manager.get_filtered_streams(
|
||||
@@ -1417,31 +1507,29 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
return claim_results
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
async def jsonrpc_resolve(self, force=False, uri=None, uris=None):
|
||||
async def jsonrpc_resolve(self, urls: typing.Union[str, list]):
|
||||
"""
|
||||
Resolve given LBRY URIs
|
||||
Get the claim that a URL refers to.
|
||||
|
||||
Usage:
|
||||
resolve [--force] (<uri> | --uri=<uri>) [<uris>...]
|
||||
resolve <urls>...
|
||||
|
||||
Options:
|
||||
--force : (bool) force refresh and ignore cache
|
||||
--uri=<uri> : (str) uri to resolve
|
||||
--uris=<uris> : (list) uris to resolve
|
||||
--urls=<urls> : (str, list) one or more urls to resolve
|
||||
|
||||
Returns:
|
||||
Dictionary of results, keyed by uri
|
||||
'<uri>': {
|
||||
Dictionary of results, keyed by url
|
||||
'<url>': {
|
||||
If a resolution error occurs:
|
||||
'error': Error message
|
||||
|
||||
If the uri resolves to a channel or a claim in a channel:
|
||||
If the url resolves to a channel or a claim in a channel:
|
||||
'certificate': {
|
||||
'address': (str) claim address,
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'claim_sequence': (int) claim sequence number (or -1 if unknown),
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -1457,16 +1545,16 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
'value': ClaimDict if decoded, otherwise hex string
|
||||
}
|
||||
|
||||
If the uri resolves to a channel:
|
||||
If the url resolves to a channel:
|
||||
'claims_in_channel': (int) number of claims in the channel,
|
||||
|
||||
If the uri resolves to a claim:
|
||||
If the url resolves to a claim:
|
||||
'claim': {
|
||||
'address': (str) claim address,
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'claim_sequence': (int) claim sequence number (or -1 if unknown),
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -1485,23 +1573,24 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
}
|
||||
"""
|
||||
|
||||
uris = tuple(uris or [])
|
||||
if uri is not None:
|
||||
uris += (uri,)
|
||||
if isinstance(urls, str):
|
||||
urls = [urls]
|
||||
|
||||
results = {}
|
||||
|
||||
valid_uris = tuple()
|
||||
for u in uris:
|
||||
valid_urls = set()
|
||||
for u in urls:
|
||||
try:
|
||||
parse_lbry_uri(u)
|
||||
valid_uris += (u,)
|
||||
valid_urls.add(u)
|
||||
except URIParseError:
|
||||
results[u] = {"error": "%s is not a valid uri" % u}
|
||||
results[u] = {"error": "%s is not a valid url" % u}
|
||||
|
||||
resolved = await self.wallet_manager.resolve(*tuple(valid_urls))
|
||||
|
||||
resolved = await self.wallet_manager.resolve(*valid_uris, check_cache=not force)
|
||||
for resolved_uri in resolved:
|
||||
results[resolved_uri] = resolved[resolved_uri]
|
||||
|
||||
return results
|
||||
|
||||
@requires(WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT,
|
||||
@@ -1551,44 +1640,17 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
}
|
||||
"""
|
||||
|
||||
timeout = timeout if timeout is not None else self.conf.download_timeout
|
||||
|
||||
parsed_uri = parse_lbry_uri(uri)
|
||||
if parsed_uri.is_channel:
|
||||
raise Exception("cannot download a channel claim, specify a /path")
|
||||
|
||||
resolved = (await self.wallet_manager.resolve(uri)).get(uri, {})
|
||||
resolved = resolved if 'value' in resolved else resolved.get('claim')
|
||||
|
||||
if not resolved:
|
||||
raise ResolveError(
|
||||
"Failed to resolve stream at lbry://{}".format(uri.replace("lbry://", ""))
|
||||
try:
|
||||
stream = await self.stream_manager.download_stream_from_uri(
|
||||
uri, self.exchange_rate_manager, file_name, timeout
|
||||
)
|
||||
if 'error' in resolved:
|
||||
raise ResolveError(f"error resolving stream: {resolved['error']}")
|
||||
|
||||
claim = ClaimDict.load_dict(resolved['value'])
|
||||
fee_amount, fee_address = None, None
|
||||
if claim.has_fee:
|
||||
fee_amount = round(self.exchange_rate_manager.convert_currency(
|
||||
claim.source_fee.currency, "LBC", claim.source_fee.amount
|
||||
), 5)
|
||||
fee_address = claim.source_fee.address
|
||||
outpoint = f"{resolved['txid']}:{resolved['nout']}"
|
||||
existing = self.stream_manager.get_filtered_streams(outpoint=outpoint)
|
||||
if not existing:
|
||||
existing.extend(self.stream_manager.get_filtered_streams(claim_id=resolved['claim_id'],
|
||||
sd_hash=claim.source_hash))
|
||||
if existing:
|
||||
log.info("already have matching stream for %s", uri)
|
||||
stream = existing[0]
|
||||
if not stream:
|
||||
raise DownloadSDTimeout(uri)
|
||||
except Exception as e:
|
||||
log.warning("Error downloading %s: %s", uri, str(e))
|
||||
return {"error": str(e)}
|
||||
else:
|
||||
stream = await self.stream_manager.download_stream_from_claim(
|
||||
self.dht_node, self.conf.download_dir, resolved, file_name, timeout, fee_amount, fee_address
|
||||
)
|
||||
if stream:
|
||||
return stream.as_dict()
|
||||
raise DownloadSDTimeout(resolved['value']['stream']['source']['source'])
|
||||
|
||||
@requires(STREAM_MANAGER_COMPONENT)
|
||||
async def jsonrpc_file_set_status(self, status, **kwargs):
|
||||
@@ -1618,11 +1680,11 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
if not streams:
|
||||
raise Exception(f'Unable to find a file for {kwargs}')
|
||||
stream = streams[0]
|
||||
if status == 'start' and not stream.running and not stream.finished:
|
||||
stream.downloader.download(self.dht_node)
|
||||
if status == 'start' and not stream.running:
|
||||
await self.stream_manager.start_stream(stream)
|
||||
msg = "Resumed download"
|
||||
elif status == 'stop' and stream.running:
|
||||
await stream.stop_download()
|
||||
await self.stream_manager.stop_stream(stream)
|
||||
msg = "Stopped download"
|
||||
else:
|
||||
msg = (
|
||||
@@ -1679,8 +1741,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
return False
|
||||
else:
|
||||
for stream in streams:
|
||||
message = f"Deleted file {stream.file_name}"
|
||||
await self.stream_manager.delete_stream(stream, delete_file=delete_from_download_dir)
|
||||
log.info("Deleted file: %s", stream.file_name)
|
||||
log.info(message)
|
||||
result = True
|
||||
return result
|
||||
|
||||
@@ -1829,7 +1892,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
async def jsonrpc_publish(
|
||||
self, name, bid, metadata=None, file_path=None, fee=None, title=None,
|
||||
description=None, author=None, language=None, license=None,
|
||||
license_url=None, thumbnail=None, preview=None, nsfw=None, sources=None,
|
||||
license_url=None, thumbnail=None, preview=None, nsfw=None,
|
||||
channel_name=None, channel_id=None, channel_account_id=None, account_id=None,
|
||||
claim_address=None, change_address=None):
|
||||
"""
|
||||
@@ -1853,7 +1916,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
[--file_path=<file_path>] [--fee=<fee>] [--title=<title>]
|
||||
[--description=<description>] [--author=<author>] [--language=<language>]
|
||||
[--license=<license>] [--license_url=<license_url>] [--thumbnail=<thumbnail>]
|
||||
[--preview=<preview>] [--nsfw=<nsfw>] [--sources=<sources>]
|
||||
[--preview=<preview>] [--nsfw=<nsfw>]
|
||||
[--channel_name=<channel_name>] [--channel_id=<channel_id>]
|
||||
[--channel_account_id=<channel_account_id>...] [--account_id=<account_id>]
|
||||
[--claim_address=<claim_address>] [--change_address=<change_address>]
|
||||
@@ -1889,7 +1952,6 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--thumbnail=<thumbnail> : (str) thumbnail url
|
||||
--preview=<preview> : (str) preview url
|
||||
--nsfw=<nsfw> : (bool) whether the content is nsfw
|
||||
--sources=<sources> : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file
|
||||
--channel_name=<channel_name> : (str) name of the publisher channel name in the wallet
|
||||
--channel_id=<channel_id> : (str) claim id of the publisher channel, does not check
|
||||
for channel claim being in the wallet. This allows
|
||||
@@ -1913,7 +1975,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
"""
|
||||
|
||||
try:
|
||||
parse_lbry_uri(name)
|
||||
parsed = parse_lbry_uri(name)
|
||||
if parsed.name != name:
|
||||
raise Exception("Name given to publish has invalid characters")
|
||||
except (TypeError, URIParseError):
|
||||
raise Exception("Invalid name given to publish")
|
||||
|
||||
@@ -1929,9 +1993,10 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
account = self.get_account_or_default(account_id)
|
||||
|
||||
available = await account.get_balance()
|
||||
existing_claims = []
|
||||
existing_claims = await account.get_claims(claim_name=name)
|
||||
if len(existing_claims) > 1:
|
||||
log.warning("found %i claims for the name %s", len(existing_claims), name)
|
||||
if amount >= available:
|
||||
existing_claims = await account.get_claims(claim_name=name)
|
||||
if len(existing_claims) == 1:
|
||||
available += existing_claims[0].get_estimator(self.ledger).effective_amount
|
||||
if amount >= available:
|
||||
@@ -1990,39 +2055,62 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
}
|
||||
}
|
||||
|
||||
# this will be used to verify the format with lbrynet.schema
|
||||
claim_copy = deepcopy(claim_dict)
|
||||
if sources is not None:
|
||||
claim_dict['stream']['source'] = sources
|
||||
claim_copy['stream']['source'] = sources
|
||||
elif file_path is not None:
|
||||
sd_to_delete = None
|
||||
|
||||
if file_path:
|
||||
if not os.path.isfile(file_path):
|
||||
raise Exception("invalid file path to publish")
|
||||
if os.path.getsize(file_path) == 0:
|
||||
raise Exception(f"Cannot publish empty file {file_path}")
|
||||
if existing_claims:
|
||||
sd_to_delete = existing_claims[-1].claim_dict['stream']['source']['source']
|
||||
|
||||
# since the file hasn't yet been made into a stream, we don't have
|
||||
# a valid Source for the claim when validating the format, we'll use a fake one
|
||||
claim_copy['stream']['source'] = {
|
||||
claim_dict['stream']['source'] = {
|
||||
'version': '_0_0_1',
|
||||
'sourceType': 'lbry_sd_hash',
|
||||
'source': '0' * 96,
|
||||
'contentType': ''
|
||||
}
|
||||
elif not existing_claims:
|
||||
raise Exception("no previous stream to update")
|
||||
else:
|
||||
# there is no existing source to use, and a file was not provided to make a new one
|
||||
raise Exception("no source provided to publish")
|
||||
claim_dict['stream']['source'] = existing_claims[-1].claim_dict['stream']['source']
|
||||
try:
|
||||
ClaimDict.load_dict(claim_copy)
|
||||
claim_dict = ClaimDict.load_dict(claim_dict).claim_dict
|
||||
# the metadata to use in the claim can be serialized by lbrynet.schema
|
||||
except DecodeError as err:
|
||||
# there was a problem with a metadata field, raise an error here rather than
|
||||
# waiting to find out when we go to publish the claim (after having made the stream)
|
||||
raise Exception(f"invalid publish metadata: {err}")
|
||||
|
||||
certificate = None
|
||||
if channel_id or channel_name:
|
||||
certificate = await self.get_channel_or_error(
|
||||
self.get_accounts_or_all(channel_account_id), channel_id, channel_name
|
||||
)
|
||||
|
||||
if file_path:
|
||||
stream = await self.stream_manager.create_stream(file_path)
|
||||
claim_dict['stream']['source']['source'] = stream.sd_hash
|
||||
claim_dict['stream']['source']['contentType'] = guess_media_type(file_path)
|
||||
|
||||
if sd_to_delete:
|
||||
stream_hash_to_delete = await self.storage.get_stream_hash_for_sd_hash(sd_to_delete)
|
||||
stream_to_delete = self.stream_manager.get_stream_by_stream_hash(stream_hash_to_delete)
|
||||
if stream_to_delete:
|
||||
await self.stream_manager.delete_stream(stream_to_delete, delete_file=False)
|
||||
log.info("updating claim to stream generated from %s, deleted previous stream %s",
|
||||
sd_to_delete[:8], file_path)
|
||||
else:
|
||||
log.info("previous stream %s from claim was not saved locally, nothing to delete",
|
||||
sd_to_delete[:8])
|
||||
else:
|
||||
log.info("generated stream from %s for claim", file_path)
|
||||
else:
|
||||
log.info("updating claim with stream %s from previous", claim_dict['stream']['source']['source'][:8])
|
||||
|
||||
sd_hash = claim_dict['stream']['source']['source']
|
||||
log.info("Publish: %s", {
|
||||
'name': name,
|
||||
'file_path': file_path,
|
||||
@@ -2033,35 +2121,14 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
'channel_id': channel_id,
|
||||
'channel_name': channel_name
|
||||
})
|
||||
|
||||
from lbrynet.extras.daemon.mime_types import guess_media_type
|
||||
|
||||
if file_path:
|
||||
if not os.path.isfile(file_path):
|
||||
raise Exception(f"File {file_path} not found")
|
||||
if os.path.getsize(file_path) == 0:
|
||||
raise Exception(f"Cannot publish empty file {file_path}")
|
||||
claim_dict['stream']['source'] = {}
|
||||
|
||||
stream = await self.stream_manager.create_stream(file_path)
|
||||
stream_hash = stream.stream_hash
|
||||
await self.storage.save_published_file(stream_hash, os.path.basename(file_path),
|
||||
os.path.dirname(file_path), 0)
|
||||
claim_dict['stream']['source']['source'] = stream.sd_hash
|
||||
claim_dict['stream']['source']['sourceType'] = 'lbry_sd_hash'
|
||||
claim_dict['stream']['source']['contentType'] = guess_media_type(file_path)
|
||||
claim_dict['stream']['source']['version'] = "_0_0_1" # need current version here
|
||||
else:
|
||||
if not ('source' not in claim_dict['stream'] and existing_claims):
|
||||
raise Exception("no previous stream to update")
|
||||
claim_dict['stream']['source'] = existing_claims[-1].claim_dict['stream']['source']
|
||||
stream_hash = await self.storage.get_stream_hash_for_sd_hash(claim_dict['stream']['source']['source'])
|
||||
tx = await self.wallet_manager.claim_name(
|
||||
account, name, amount, claim_dict, certificate, claim_address
|
||||
)
|
||||
await self.storage.save_content_claim(
|
||||
stream_hash, tx.outputs[0].id
|
||||
)
|
||||
stream_hash = await self.storage.get_stream_hash_for_sd_hash(sd_hash)
|
||||
if stream_hash:
|
||||
await self.storage.save_content_claim(
|
||||
stream_hash, tx.outputs[0].id
|
||||
)
|
||||
await self.analytics_manager.send_claim_action('publish')
|
||||
nout = 0
|
||||
txo = tx.outputs[nout]
|
||||
@@ -2289,7 +2356,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
}
|
||||
"""
|
||||
claims = await self.wallet_manager.get_claims_for_name(name) # type: dict
|
||||
sort_claim_results(claims['claims'])
|
||||
claims['claims'] = sort_claim_results(claims['claims'])
|
||||
return claims
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
@@ -2327,7 +2394,6 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
'amount': (float) claim amount,
|
||||
'effective_amount': (float) claim amount including supports,
|
||||
'claim_id': (str) claim id,
|
||||
'claim_sequence': (int) claim sequence number,
|
||||
'decoded_claim': (bool) whether or not the claim value was decoded,
|
||||
'height': (int) claim height,
|
||||
'depth': (int) claim depth,
|
||||
@@ -2569,8 +2635,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
(str) Success/Fail message or (dict) decoded data
|
||||
"""
|
||||
|
||||
downloader = BlobDownloader(asyncio.get_event_loop(), self.blob_manager, self.conf)
|
||||
blob = await downloader.get_blob(blob_hash, self.dht_node)
|
||||
blob = await download_blob(asyncio.get_event_loop(), self.conf, self.blob_manager, self.dht_node, blob_hash)
|
||||
if read:
|
||||
with open(blob.file_path, 'rb') as handle:
|
||||
return handle.read().decode()
|
||||
@@ -2591,7 +2656,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
Returns:
|
||||
(str) Success/fail message
|
||||
"""
|
||||
|
||||
if not blob_hash or not is_valid_blobhash(blob_hash):
|
||||
return f"Invalid blob hash to delete '{blob_hash}'"
|
||||
streams = self.stream_manager.get_filtered_streams(sd_hash=blob_hash)
|
||||
if streams:
|
||||
await self.stream_manager.delete_stream(streams[0])
|
||||
@@ -2721,7 +2787,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
else:
|
||||
blobs = []
|
||||
if stream_hash:
|
||||
blobs.extend([b.blob_hash for b in await self.storage.get_blobs_for_stream(stream_hash)])
|
||||
blobs.extend([b.blob_hash for b in (await self.storage.get_blobs_for_stream(stream_hash))[:-1]])
|
||||
else:
|
||||
blobs = list(self.blob_manager.completed_blob_hashes)
|
||||
if needed:
|
||||
@@ -2791,7 +2857,19 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
(list) list of blobs reflected
|
||||
"""
|
||||
|
||||
raise NotImplementedError()
|
||||
server, port = kwargs.get('server'), kwargs.get('port')
|
||||
if server and port:
|
||||
port = int(port)
|
||||
else:
|
||||
server, port = random.choice(self.conf.reflector_servers)
|
||||
reflected = await asyncio.gather(*[
|
||||
stream.upload_to_reflector(server, port)
|
||||
for stream in self.stream_manager.get_filtered_streams(**kwargs)
|
||||
])
|
||||
total = []
|
||||
for reflected_for_stream in reflected:
|
||||
total.extend(reflected_for_stream)
|
||||
return total
|
||||
|
||||
@requires(DHT_COMPONENT)
|
||||
async def jsonrpc_peer_ping(self, node_id, address, port):
|
||||
@@ -2802,23 +2880,22 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
Usage:
|
||||
peer_ping (<node_id> | --node_id=<node_id>) (<address> | --address=<address>) (<port> | --port=<port>)
|
||||
|
||||
Options:
|
||||
None
|
||||
|
||||
Returns:
|
||||
(str) pong, or {'error': <error message>} if an error is encountered
|
||||
"""
|
||||
peer = None
|
||||
if node_id and address and port:
|
||||
peer = self.component_manager.peer_manager.get_peer(address, unhexlify(node_id), udp_port=int(port))
|
||||
if not peer:
|
||||
peer = self.component_manager.peer_manager.make_peer(
|
||||
address, unhexlify(node_id), udp_port=int(port)
|
||||
)
|
||||
peer = self.component_manager.peer_manager.get_kademlia_peer(unhexlify(node_id), address,
|
||||
udp_port=int(port))
|
||||
try:
|
||||
return await self.dht_node.protocol.get_rpc_peer(peer).ping()
|
||||
except asyncio.TimeoutError:
|
||||
return {'error': 'timeout'}
|
||||
if not peer:
|
||||
return {'error': 'peer not found'}
|
||||
try:
|
||||
result = await peer.ping()
|
||||
return result.decode()
|
||||
except TimeoutError:
|
||||
return {'error': 'ping timeout'}
|
||||
|
||||
@requires(DHT_COMPONENT)
|
||||
def jsonrpc_routing_table_get(self):
|
||||
|
||||
+114
-122
@@ -1,13 +1,15 @@
|
||||
import asyncio
|
||||
import collections
|
||||
import logging
|
||||
|
||||
import aiohttp
|
||||
|
||||
import typing
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import Config, ANALYTICS_ENDPOINT, ANALYTICS_TOKEN
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras import system_info
|
||||
|
||||
ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
|
||||
ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
|
||||
|
||||
# Things We Track
|
||||
SERVER_STARTUP = 'Server Startup'
|
||||
SERVER_STARTUP_SUCCESS = 'Server Startup Success'
|
||||
@@ -19,102 +21,137 @@ HEARTBEAT = 'Heartbeat'
|
||||
CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon
|
||||
NEW_CHANNEL = 'New Channel'
|
||||
CREDITS_SENT = 'Credits Sent'
|
||||
NEW_DOWNLOAD_STAT = 'Download'
|
||||
UPNP_SETUP = "UPnP Setup"
|
||||
|
||||
BLOB_BYTES_UPLOADED = 'Blob Bytes Uploaded'
|
||||
|
||||
|
||||
TIME_TO_FIRST_BYTES = "Time To First Bytes"
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Manager:
|
||||
def _event_properties(installation_id: str, session_id: str,
|
||||
event_properties: typing.Optional[typing.Dict]) -> typing.Dict:
|
||||
properties = {
|
||||
'lbry_id': installation_id,
|
||||
'session_id': session_id,
|
||||
}
|
||||
properties.update(event_properties or {})
|
||||
return properties
|
||||
|
||||
|
||||
def _download_properties(conf: Config, external_ip: str, resolve_duration: float,
|
||||
total_duration: typing.Optional[float], download_id: str, name: str,
|
||||
outpoint: str, active_peer_count: int, tried_peers_count: int,
|
||||
added_fixed_peers: bool, fixed_peer_delay: float, sd_hash: str,
|
||||
sd_download_duration: typing.Optional[float] = None,
|
||||
head_blob_hash: typing.Optional[str] = None,
|
||||
head_blob_length: typing.Optional[int] = None,
|
||||
head_blob_download_duration: typing.Optional[float] = None,
|
||||
error: typing.Optional[str] = None) -> typing.Dict:
|
||||
return {
|
||||
"external_ip": external_ip,
|
||||
"download_id": download_id,
|
||||
"total_duration": round(total_duration, 4),
|
||||
"resolve_duration": None if not resolve_duration else round(resolve_duration, 4),
|
||||
"error": error,
|
||||
'name': name,
|
||||
"outpoint": outpoint,
|
||||
|
||||
"node_rpc_timeout": conf.node_rpc_timeout,
|
||||
"peer_connect_timeout": conf.peer_connect_timeout,
|
||||
"blob_download_timeout": conf.blob_download_timeout,
|
||||
"use_fixed_peers": len(conf.reflector_servers) > 0,
|
||||
"fixed_peer_delay": fixed_peer_delay,
|
||||
"added_fixed_peers": added_fixed_peers,
|
||||
"active_peer_count": active_peer_count,
|
||||
"tried_peers_count": tried_peers_count,
|
||||
|
||||
"sd_blob_hash": sd_hash,
|
||||
"sd_blob_duration": None if not sd_download_duration else round(sd_download_duration, 4),
|
||||
|
||||
"head_blob_hash": head_blob_hash,
|
||||
"head_blob_length": head_blob_length,
|
||||
"head_blob_duration": None if not head_blob_download_duration else round(head_blob_download_duration, 4)
|
||||
}
|
||||
|
||||
|
||||
def _make_context(platform):
|
||||
# see https://segment.com/docs/spec/common/#context
|
||||
# they say they'll ignore fields outside the spec, but evidently they don't
|
||||
context = {
|
||||
'app': {
|
||||
'version': platform['lbrynet_version'],
|
||||
'build': platform['build'],
|
||||
},
|
||||
# TODO: expand os info to give linux/osx specific info
|
||||
'os': {
|
||||
'name': platform['os_system'],
|
||||
'version': platform['os_release']
|
||||
},
|
||||
}
|
||||
if 'desktop' in platform and 'distro' in platform:
|
||||
context['os']['desktop'] = platform['desktop']
|
||||
context['os']['distro'] = platform['distro']
|
||||
return context
|
||||
|
||||
|
||||
class AnalyticsManager:
|
||||
|
||||
def __init__(self, conf: Config, installation_id: str, session_id: str):
|
||||
self.conf = conf
|
||||
self.cookies = {}
|
||||
self.url = ANALYTICS_ENDPOINT
|
||||
self._write_key = utils.deobfuscate(ANALYTICS_TOKEN)
|
||||
self._enabled = conf.share_usage_data
|
||||
self._tracked_data = collections.defaultdict(list)
|
||||
self.context = self._make_context(system_info.get_platform(), 'torba')
|
||||
self.context = _make_context(system_info.get_platform())
|
||||
self.installation_id = installation_id
|
||||
self.session_id = session_id
|
||||
self.task: asyncio.Task = None
|
||||
self.external_ip: typing.Optional[str] = None
|
||||
|
||||
@property
|
||||
def is_started(self):
|
||||
return self.task is not None
|
||||
|
||||
def start(self):
|
||||
async def start(self):
|
||||
if self._enabled and self.task is None:
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
self.task = asyncio.create_task(self.run())
|
||||
|
||||
async def run(self):
|
||||
while True:
|
||||
await self._send_heartbeat()
|
||||
await asyncio.sleep(1800)
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
|
||||
def stop(self):
|
||||
if self.task is not None and not self.task.done():
|
||||
self.task.cancel()
|
||||
|
||||
async def _post(self, endpoint, data):
|
||||
# there is an issue with a timing condition with keep-alive
|
||||
# that is best explained here: https://github.com/mikem23/keepalive-race
|
||||
#
|
||||
# If you make a request, wait just the right amount of time,
|
||||
# then make another request, the requests module may opt to
|
||||
# reuse the connection, but by the time the server gets it the
|
||||
# timeout will have expired.
|
||||
#
|
||||
# by forcing the connection to close, we will disable the keep-alive.
|
||||
|
||||
assert endpoint[0] == '/'
|
||||
async def _post(self, data: typing.Dict):
|
||||
request_kwargs = {
|
||||
'method': 'POST',
|
||||
'url': self.url + endpoint,
|
||||
'url': self.url + '/track',
|
||||
'headers': {'Connection': 'Close'},
|
||||
'auth': aiohttp.BasicAuth(self._write_key, ''),
|
||||
'json': data,
|
||||
'cookies': self.cookies
|
||||
}
|
||||
try:
|
||||
async with aiohttp.request(**request_kwargs) as response:
|
||||
async with utils.aiohttp_request(**request_kwargs) as response:
|
||||
self.cookies.update(response.cookies)
|
||||
except Exception as e:
|
||||
log.exception('Encountered an exception while POSTing to %s: ', self.url + endpoint, exc_info=e)
|
||||
log.debug('Encountered an exception while POSTing to %s: ', self.url + '/track', exc_info=e)
|
||||
|
||||
async def track(self, event):
|
||||
async def track(self, event: typing.Dict):
|
||||
"""Send a single tracking event"""
|
||||
if self._enabled:
|
||||
log.debug('Sending track event: %s', event)
|
||||
await self._post('/track', event)
|
||||
|
||||
async def send_new_download_start(self, download_id, name, claim_dict):
|
||||
await self._send_new_download_stats("start", download_id, name, claim_dict)
|
||||
|
||||
async def send_new_download_success(self, download_id, name, claim_dict):
|
||||
await self._send_new_download_stats("success", download_id, name, claim_dict)
|
||||
|
||||
async def send_new_download_fail(self, download_id, name, claim_dict, e):
|
||||
await self._send_new_download_stats("failure", download_id, name, claim_dict, {
|
||||
'name': type(e).__name__ if hasattr(type(e), "__name__") else str(type(e)),
|
||||
'message': str(e),
|
||||
})
|
||||
|
||||
async def _send_new_download_stats(self, action, download_id, name, claim_dict, e=None):
|
||||
await self.track({
|
||||
'userId': 'lbry', # required, see https://segment.com/docs/sources/server/http/#track
|
||||
'event': NEW_DOWNLOAD_STAT,
|
||||
'properties': self._event_properties({
|
||||
'download_id': download_id,
|
||||
'name': name,
|
||||
'sd_hash': None if not claim_dict else claim_dict.source_hash.decode(),
|
||||
'action': action,
|
||||
'error': e,
|
||||
}),
|
||||
'context': self.context,
|
||||
'timestamp': utils.isonow(),
|
||||
})
|
||||
await self._post(event)
|
||||
|
||||
async def send_upnp_setup_success_fail(self, success, status):
|
||||
await self.track(
|
||||
@@ -133,20 +170,33 @@ class Manager:
|
||||
async def send_server_startup_error(self, message):
|
||||
await self.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
|
||||
|
||||
async def send_download_started(self, id_, name, claim_dict=None):
|
||||
async def send_time_to_first_bytes(self, resolve_duration: typing.Optional[float],
|
||||
total_duration: typing.Optional[float], download_id: str,
|
||||
name: str, outpoint: str, found_peers_count: int,
|
||||
tried_peers_count: int, added_fixed_peers: bool,
|
||||
fixed_peers_delay: float, sd_hash: str,
|
||||
sd_download_duration: typing.Optional[float] = None,
|
||||
head_blob_hash: typing.Optional[str] = None,
|
||||
head_blob_length: typing.Optional[int] = None,
|
||||
head_blob_duration: typing.Optional[int] = None,
|
||||
error: typing.Optional[str] = None):
|
||||
await self.track(self._event(TIME_TO_FIRST_BYTES, _download_properties(
|
||||
self.conf, self.external_ip, resolve_duration, total_duration, download_id, name, outpoint,
|
||||
found_peers_count, tried_peers_count, added_fixed_peers, fixed_peers_delay, sd_hash,
|
||||
sd_download_duration, head_blob_hash, head_blob_length, head_blob_duration, error
|
||||
)))
|
||||
|
||||
async def send_download_finished(self, download_id, name, sd_hash):
|
||||
await self.track(
|
||||
self._event(DOWNLOAD_STARTED, self._download_properties(id_, name, claim_dict))
|
||||
self._event(
|
||||
DOWNLOAD_FINISHED, {
|
||||
'download_id': download_id,
|
||||
'name': name,
|
||||
'stream_info': sd_hash
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
async def send_download_errored(self, err, id_, name, claim_dict, report):
|
||||
download_error_properties = self._download_error_properties(err, id_, name, claim_dict,
|
||||
report)
|
||||
await self.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
|
||||
|
||||
async def send_download_finished(self, id_, name, report, claim_dict=None):
|
||||
download_properties = self._download_properties(id_, name, claim_dict, report)
|
||||
await self.track(self._event(DOWNLOAD_FINISHED, download_properties))
|
||||
|
||||
async def send_claim_action(self, action):
|
||||
await self.track(self._event(CLAIM_ACTION, {'action': action}))
|
||||
|
||||
@@ -159,69 +209,11 @@ class Manager:
|
||||
async def _send_heartbeat(self):
|
||||
await self.track(self._event(HEARTBEAT))
|
||||
|
||||
def _event(self, event, event_properties=None):
|
||||
def _event(self, event, properties: typing.Optional[typing.Dict] = None):
|
||||
return {
|
||||
'userId': 'lbry',
|
||||
'event': event,
|
||||
'properties': self._event_properties(event_properties),
|
||||
'properties': _event_properties(self.installation_id, self.session_id, properties),
|
||||
'context': self.context,
|
||||
'timestamp': utils.isonow()
|
||||
}
|
||||
|
||||
def _metric_event(self, metric_name, value):
|
||||
return self._event(metric_name, {'value': value})
|
||||
|
||||
def _event_properties(self, event_properties=None):
|
||||
properties = {
|
||||
'lbry_id': self.installation_id,
|
||||
'session_id': self.session_id,
|
||||
}
|
||||
properties.update(event_properties or {})
|
||||
return properties
|
||||
|
||||
@staticmethod
|
||||
def _download_properties(id_, name, claim_dict=None, report=None):
|
||||
sd_hash = None if not claim_dict else claim_dict.source_hash.decode()
|
||||
p = {
|
||||
'download_id': id_,
|
||||
'name': name,
|
||||
'stream_info': sd_hash
|
||||
}
|
||||
if report:
|
||||
p['report'] = report
|
||||
return p
|
||||
|
||||
@staticmethod
|
||||
def _download_error_properties(error, id_, name, claim_dict, report):
|
||||
def error_name(err):
|
||||
if not hasattr(type(err), "__name__"):
|
||||
return str(type(err))
|
||||
return type(err).__name__
|
||||
return {
|
||||
'download_id': id_,
|
||||
'name': name,
|
||||
'stream_info': claim_dict.source_hash.decode(),
|
||||
'error': error_name(error),
|
||||
'reason': str(error),
|
||||
'report': report
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _make_context(platform, wallet):
|
||||
# see https://segment.com/docs/spec/common/#context
|
||||
# they say they'll ignore fields outside the spec, but evidently they don't
|
||||
context = {
|
||||
'app': {
|
||||
'version': platform['lbrynet_version'],
|
||||
'build': platform['build'],
|
||||
},
|
||||
# TODO: expand os info to give linux/osx specific info
|
||||
'os': {
|
||||
'name': platform['os_system'],
|
||||
'version': platform['os_release']
|
||||
},
|
||||
}
|
||||
if 'desktop' in platform and 'distro' in platform:
|
||||
context['os']['desktop'] = platform['desktop']
|
||||
context['os']['distro'] = platform['distro']
|
||||
return context
|
||||
@@ -1,10 +1,10 @@
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import time
|
||||
import logging
|
||||
import json
|
||||
|
||||
from lbrynet.error import InvalidExchangeRateResponse, CurrencyConversionError
|
||||
from lbrynet.utils import aiohttp_request
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -52,8 +52,8 @@ class MarketFeed:
|
||||
def is_online(self):
|
||||
return self._online
|
||||
|
||||
async def _make_request(self):
|
||||
async with aiohttp.request('get', self.url, params=self.params) as response:
|
||||
async def _make_request(self) -> str:
|
||||
async with aiohttp_request('get', self.url, params=self.params) as response:
|
||||
return (await response.read()).decode()
|
||||
|
||||
def _handle_response(self, response):
|
||||
@@ -219,7 +219,7 @@ class ExchangeRateManager:
|
||||
|
||||
def convert_currency(self, from_currency, to_currency, amount):
|
||||
rates = [market.rate for market in self.market_feeds]
|
||||
log.info("Converting %f %s to %s, rates: %s" % (amount, from_currency, to_currency, rates))
|
||||
log.debug("Converting %f %s to %s, rates: %s" % (amount, from_currency, to_currency, rates))
|
||||
if from_currency == to_currency:
|
||||
return amount
|
||||
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import json
|
||||
import logging.handlers
|
||||
import traceback
|
||||
@@ -51,7 +50,7 @@ class HTTPSLogglyHandler(logging.Handler):
|
||||
|
||||
async def _emit(self, record):
|
||||
payload = self.format(record)
|
||||
async with aiohttp.request('post', self.url, data=payload.encode(), cookies=self.cookies) as response:
|
||||
async with utils.aiohttp_request('post', self.url, data=payload.encode(), cookies=self.cookies) as response:
|
||||
self.cookies.update(response.cookies)
|
||||
|
||||
def emit(self, record):
|
||||
|
||||
@@ -154,5 +154,7 @@ types_map = {
|
||||
|
||||
def guess_media_type(path):
|
||||
_, ext = os.path.splitext(path)
|
||||
default = f'application/x-ext-{ext}' if ext else 'application/octet-stream'
|
||||
return types_map.get(ext, default)
|
||||
return types_map.get(
|
||||
ext.strip().lower(),
|
||||
'application/octet-stream' if not (ext and ext[1:].strip()) else f'application/x-ext-{ext[1:].strip().lower()}'
|
||||
)
|
||||
+193
-107
@@ -1,8 +1,10 @@
|
||||
import os
|
||||
import logging
|
||||
import sqlite3
|
||||
import typing
|
||||
import asyncio
|
||||
import binascii
|
||||
import time
|
||||
from torba.client.basedatabase import SQLiteMixin
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
||||
@@ -104,14 +106,104 @@ def get_content_claim_from_outpoint(transaction: sqlite3.Connection,
|
||||
return StoredStreamClaim(*claim_fields)
|
||||
|
||||
|
||||
def _batched_select(transaction, query, parameters):
|
||||
for start_index in range(0, len(parameters), 900):
|
||||
current_batch = parameters[start_index:start_index+900]
|
||||
def _batched_select(transaction, query, parameters, batch_size=900):
|
||||
for start_index in range(0, len(parameters), batch_size):
|
||||
current_batch = parameters[start_index:start_index+batch_size]
|
||||
bind = "({})".format(','.join(['?'] * len(current_batch)))
|
||||
for result in transaction.execute(query.format(bind), current_batch):
|
||||
yield result
|
||||
|
||||
|
||||
def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
|
||||
files = []
|
||||
signed_claims = {}
|
||||
for (rowid, stream_hash, file_name, download_dir, data_rate, status, _, sd_hash, stream_key,
|
||||
stream_name, suggested_file_name, *claim_args) in _batched_select(
|
||||
transaction, "select file.rowid, file.*, stream.*, c.* "
|
||||
"from file inner join stream on file.stream_hash=stream.stream_hash "
|
||||
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
|
||||
"inner join claim c on cc.claim_outpoint=c.claim_outpoint "
|
||||
"where file.stream_hash in {} "
|
||||
"order by c.rowid desc", [
|
||||
stream_hash for (stream_hash,) in transaction.execute("select stream_hash from file")]):
|
||||
|
||||
claim = StoredStreamClaim(stream_hash, *claim_args)
|
||||
if claim.channel_claim_id:
|
||||
if claim.channel_claim_id not in signed_claims:
|
||||
signed_claims[claim.channel_claim_id] = []
|
||||
signed_claims[claim.channel_claim_id].append(claim)
|
||||
files.append(
|
||||
{
|
||||
"rowid": rowid,
|
||||
"stream_hash": stream_hash,
|
||||
"file_name": file_name, # hex
|
||||
"download_directory": download_dir, # hex
|
||||
"blob_data_rate": data_rate,
|
||||
"status": status,
|
||||
"sd_hash": sd_hash,
|
||||
"key": stream_key,
|
||||
"stream_name": stream_name, # hex
|
||||
"suggested_file_name": suggested_file_name, # hex
|
||||
"claim": claim
|
||||
}
|
||||
)
|
||||
for claim_name, claim_id in _batched_select(
|
||||
transaction, "select c.claim_name, c.claim_id from claim c where c.claim_id in {}",
|
||||
list(signed_claims.keys())):
|
||||
for claim in signed_claims[claim_id]:
|
||||
claim.channel_name = claim_name
|
||||
return files
|
||||
|
||||
|
||||
def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'):
|
||||
# add the head blob and set it to be announced
|
||||
transaction.execute(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
sd_blob.blob_hash, sd_blob.length, 0, 1, "pending", 0, 0,
|
||||
descriptor.blobs[0].blob_hash, descriptor.blobs[0].length, 0, 1, "pending", 0, 0
|
||||
)
|
||||
)
|
||||
# add the rest of the blobs with announcement off
|
||||
if len(descriptor.blobs) > 2:
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
[(blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
|
||||
for blob in descriptor.blobs[1:-1]]
|
||||
)
|
||||
# associate the blobs to the stream
|
||||
transaction.execute("insert or ignore into stream values (?, ?, ?, ?, ?)",
|
||||
(descriptor.stream_hash, sd_blob.blob_hash, descriptor.key,
|
||||
binascii.hexlify(descriptor.stream_name.encode()).decode(),
|
||||
binascii.hexlify(descriptor.suggested_file_name.encode()).decode()))
|
||||
# add the stream
|
||||
transaction.executemany(
|
||||
"insert or ignore into stream_blob values (?, ?, ?, ?)",
|
||||
[(descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
|
||||
for blob in descriptor.blobs]
|
||||
)
|
||||
|
||||
|
||||
def delete_stream(transaction: sqlite3.Connection, descriptor: 'StreamDescriptor'):
|
||||
blob_hashes = [(blob.blob_hash, ) for blob in descriptor.blobs[:-1]]
|
||||
blob_hashes.append((descriptor.sd_hash, ))
|
||||
transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,))
|
||||
transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash,))
|
||||
transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash,))
|
||||
transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash,))
|
||||
transaction.executemany("delete from blob where blob_hash=?", blob_hashes)
|
||||
|
||||
|
||||
def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: str, download_directory: str,
|
||||
data_payment_rate: float, status: str) -> int:
|
||||
transaction.execute(
|
||||
"insert or replace into file values (?, ?, ?, ?, ?)",
|
||||
(stream_hash, binascii.hexlify(file_name.encode()).decode(),
|
||||
binascii.hexlify(download_directory.encode()).decode(), data_payment_rate, status)
|
||||
)
|
||||
return transaction.execute("select rowid from file where stream_hash=?", (stream_hash, )).fetchone()[0]
|
||||
|
||||
|
||||
class SQLiteStorage(SQLiteMixin):
|
||||
CREATE_TABLES_QUERY = """
|
||||
pragma foreign_keys=on;
|
||||
@@ -184,11 +276,12 @@ class SQLiteStorage(SQLiteMixin):
|
||||
);
|
||||
"""
|
||||
|
||||
def __init__(self, conf: Config, path, loop=None):
|
||||
def __init__(self, conf: Config, path, loop=None, time_getter: typing.Optional[typing.Callable[[], float]] = None):
|
||||
super().__init__(path)
|
||||
self.conf = conf
|
||||
self.content_claim_callbacks = {}
|
||||
self.loop = loop or asyncio.get_event_loop()
|
||||
self.time_getter = time_getter or time.time
|
||||
|
||||
async def run_and_return_one_or_none(self, query, *args):
|
||||
for row in await self.db.execute_fetchall(query, args):
|
||||
@@ -202,15 +295,16 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
# # # # # # # # # blob functions # # # # # # # # #
|
||||
|
||||
def add_completed_blob(self, blob_hash: str):
|
||||
log.debug("Adding a completed blob. blob_hash=%s", blob_hash)
|
||||
return self.db.execute("update blob set status='finished' where blob.blob_hash=?", (blob_hash, ))
|
||||
|
||||
def set_should_announce(self, blob_hash: str, next_announce_time: int, should_announce: int):
|
||||
return self.db.execute(
|
||||
"update blob set next_announce_time=?, should_announce=? where blob_hash=?",
|
||||
(next_announce_time or 0, int(bool(should_announce)), blob_hash)
|
||||
)
|
||||
def add_completed_blob(self, blob_hash: str, length: int):
|
||||
def _add_blob(transaction: sqlite3.Connection):
|
||||
transaction.execute(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
(blob_hash, length, 0, 0, "pending", 0, 0)
|
||||
)
|
||||
transaction.execute(
|
||||
"update blob set status='finished' where blob.blob_hash=?", (blob_hash, )
|
||||
)
|
||||
return self.db.run(_add_blob)
|
||||
|
||||
def get_blob_status(self, blob_hash: str):
|
||||
return self.run_and_return_one_or_none(
|
||||
@@ -247,8 +341,9 @@ class SQLiteStorage(SQLiteMixin):
|
||||
"select count(*) from blob where status='finished'"
|
||||
)
|
||||
|
||||
def update_last_announced_blobs(self, blob_hashes: typing.List[str], last_announced: float):
|
||||
def update_last_announced_blobs(self, blob_hashes: typing.List[str]):
|
||||
def _update_last_announced_blobs(transaction: sqlite3.Connection):
|
||||
last_announced = self.time_getter()
|
||||
return transaction.executemany(
|
||||
"update blob set next_announce_time=?, last_announced_time=?, single_announce=0 "
|
||||
"where blob_hash=?",
|
||||
@@ -259,7 +354,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
def should_single_announce_blobs(self, blob_hashes, immediate=False):
|
||||
def set_single_announce(transaction):
|
||||
now = self.loop.time()
|
||||
now = int(self.time_getter())
|
||||
for blob_hash in blob_hashes:
|
||||
if immediate:
|
||||
transaction.execute(
|
||||
@@ -274,21 +369,23 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
def get_blobs_to_announce(self):
|
||||
def get_and_update(transaction):
|
||||
timestamp = self.loop.time()
|
||||
if self.conf.announce_head_blobs_only:
|
||||
timestamp = int(self.time_getter())
|
||||
if self.conf.announce_head_and_sd_only:
|
||||
r = transaction.execute(
|
||||
"select blob_hash from blob "
|
||||
"where blob_hash is not null and "
|
||||
"(should_announce=1 or single_announce=1) and next_announce_time<? and status='finished'",
|
||||
(timestamp,)
|
||||
"(should_announce=1 or single_announce=1) and next_announce_time<? and status='finished' "
|
||||
"order by next_announce_time asc limit ?",
|
||||
(timestamp, int(self.conf.concurrent_blob_announcers * 10))
|
||||
)
|
||||
else:
|
||||
r = transaction.execute(
|
||||
"select blob_hash from blob where blob_hash is not null "
|
||||
"and next_announce_time<? and status='finished'", (timestamp,)
|
||||
"and next_announce_time<? and status='finished' "
|
||||
"order by next_announce_time asc limit ?",
|
||||
(timestamp, int(self.conf.concurrent_blob_announcers * 10))
|
||||
)
|
||||
blobs = [b[0] for b in r.fetchall()]
|
||||
return blobs
|
||||
return [b[0] for b in r.fetchall()]
|
||||
return self.db.run(get_and_update)
|
||||
|
||||
def delete_blobs_from_db(self, blob_hashes):
|
||||
@@ -296,11 +393,31 @@ class SQLiteStorage(SQLiteMixin):
|
||||
transaction.executemany(
|
||||
"delete from blob where blob_hash=?;", [(blob_hash,) for blob_hash in blob_hashes]
|
||||
)
|
||||
return self.db.run(delete_blobs)
|
||||
return self.db.run_with_foreign_keys_disabled(delete_blobs)
|
||||
|
||||
def get_all_blob_hashes(self):
|
||||
return self.run_and_return_list("select blob_hash from blob")
|
||||
|
||||
def sync_missing_blobs(self, blob_files: typing.Set[str]) -> typing.Awaitable[typing.Set[str]]:
|
||||
def _sync_blobs(transaction: sqlite3.Connection) -> typing.Set[str]:
|
||||
to_update = [
|
||||
(blob_hash, )
|
||||
for (blob_hash, ) in transaction.execute("select blob_hash from blob where status='finished'")
|
||||
if blob_hash not in blob_files
|
||||
]
|
||||
transaction.executemany(
|
||||
"update blob set status='pending' where blob_hash=?",
|
||||
to_update
|
||||
)
|
||||
return {
|
||||
blob_hash
|
||||
for blob_hash, in _batched_select(
|
||||
transaction, "select blob_hash from blob where status='finished' and blob_hash in {}",
|
||||
list(blob_files)
|
||||
)
|
||||
}
|
||||
return self.db.run(_sync_blobs)
|
||||
|
||||
# # # # # # # # # stream functions # # # # # # # # #
|
||||
|
||||
async def stream_exists(self, sd_hash: str) -> bool:
|
||||
@@ -313,42 +430,20 @@ class SQLiteStorage(SQLiteMixin):
|
||||
"s.stream_hash=f.stream_hash and s.sd_hash=?", sd_hash)
|
||||
return streams is not None
|
||||
|
||||
def rowid_for_stream(self, stream_hash: str) -> typing.Awaitable[typing.Optional[int]]:
|
||||
return self.run_and_return_one_or_none(
|
||||
"select rowid from file where stream_hash=?", stream_hash
|
||||
)
|
||||
|
||||
def store_stream(self, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'):
|
||||
def _store_stream(transaction: sqlite3.Connection):
|
||||
# add the head blob and set it to be announced
|
||||
transaction.execute(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
sd_blob.blob_hash, sd_blob.length, 0, 1, "pending", 0, 0,
|
||||
descriptor.blobs[0].blob_hash, descriptor.blobs[0].length, 0, 1, "pending", 0, 0
|
||||
)
|
||||
)
|
||||
# add the rest of the blobs with announcement off
|
||||
if len(descriptor.blobs) > 2:
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
[(blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
|
||||
for blob in descriptor.blobs[1:-1]]
|
||||
)
|
||||
# associate the blobs to the stream
|
||||
transaction.execute("insert or ignore into stream values (?, ?, ?, ?, ?)",
|
||||
(descriptor.stream_hash, sd_blob.blob_hash, descriptor.key,
|
||||
binascii.hexlify(descriptor.stream_name.encode()).decode(),
|
||||
binascii.hexlify(descriptor.suggested_file_name.encode()).decode()))
|
||||
# add the stream
|
||||
transaction.executemany(
|
||||
"insert or ignore into stream_blob values (?, ?, ?, ?)",
|
||||
[(descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
|
||||
for blob in descriptor.blobs]
|
||||
)
|
||||
return self.db.run(store_stream, sd_blob, descriptor)
|
||||
|
||||
return self.db.run(_store_stream)
|
||||
|
||||
def get_blobs_for_stream(self, stream_hash, only_completed=False):
|
||||
def get_blobs_for_stream(self, stream_hash, only_completed=False) -> typing.Awaitable[typing.List[BlobInfo]]:
|
||||
def _get_blobs_for_stream(transaction):
|
||||
crypt_blob_infos = []
|
||||
stream_blobs = transaction.execute(
|
||||
"select blob_hash, position, iv from stream_blob where stream_hash=?", (stream_hash, )
|
||||
"select blob_hash, position, iv from stream_blob where stream_hash=? "
|
||||
"order by position asc", (stream_hash, )
|
||||
).fetchall()
|
||||
if only_completed:
|
||||
lengths = transaction.execute(
|
||||
@@ -370,7 +465,8 @@ class SQLiteStorage(SQLiteMixin):
|
||||
for blob_hash, position, iv in stream_blobs:
|
||||
blob_length = blob_length_dict.get(blob_hash, 0)
|
||||
crypt_blob_infos.append(BlobInfo(position, blob_length, iv, blob_hash))
|
||||
crypt_blob_infos = sorted(crypt_blob_infos, key=lambda info: info.blob_num)
|
||||
if not blob_hash:
|
||||
break
|
||||
return crypt_blob_infos
|
||||
return self.db.run(_get_blobs_for_stream)
|
||||
|
||||
@@ -385,68 +481,58 @@ class SQLiteStorage(SQLiteMixin):
|
||||
)
|
||||
|
||||
def delete_stream(self, descriptor: 'StreamDescriptor'):
|
||||
def _delete_stream(transaction: sqlite3.Connection):
|
||||
transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,))
|
||||
transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash, ))
|
||||
transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash, ))
|
||||
transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash, ))
|
||||
transaction.execute("delete from blob where blob_hash=?", (descriptor.sd_hash, ))
|
||||
transaction.executemany("delete from blob where blob_hash=?",
|
||||
[(blob.blob_hash, ) for blob in descriptor.blobs[:-1]])
|
||||
return self.db.run(_delete_stream)
|
||||
return self.db.run_with_foreign_keys_disabled(delete_stream, descriptor)
|
||||
|
||||
# # # # # # # # # file stuff # # # # # # # # #
|
||||
|
||||
def save_downloaded_file(self, stream_hash, file_name, download_directory, data_payment_rate):
|
||||
def save_downloaded_file(self, stream_hash, file_name, download_directory,
|
||||
data_payment_rate) -> typing.Awaitable[int]:
|
||||
return self.save_published_file(
|
||||
stream_hash, binascii.hexlify(file_name.encode()).decode(),
|
||||
binascii.hexlify(download_directory.encode()).decode(), data_payment_rate,
|
||||
status="running"
|
||||
stream_hash, file_name, download_directory, data_payment_rate, status="running"
|
||||
)
|
||||
|
||||
def save_published_file(self, stream_hash: str, file_name: str, download_directory: str, data_payment_rate: float,
|
||||
status="finished"):
|
||||
return self.db.execute(
|
||||
"insert into file values (?, ?, ?, ?, ?)",
|
||||
(stream_hash, file_name, download_directory, data_payment_rate, status)
|
||||
)
|
||||
status="finished") -> typing.Awaitable[int]:
|
||||
return self.db.run(store_file, stream_hash, file_name, download_directory, data_payment_rate, status)
|
||||
|
||||
async def get_all_lbry_files(self) -> typing.List[typing.Dict]:
|
||||
def _lbry_file_dict(rowid, stream_hash, file_name, download_dir, data_rate, status, _, sd_hash, stream_key,
|
||||
stream_name, suggested_file_name) -> typing.Dict:
|
||||
return {
|
||||
"row_id": rowid,
|
||||
"stream_hash": stream_hash,
|
||||
"file_name": file_name,
|
||||
"download_directory": download_dir,
|
||||
"blob_data_rate": data_rate,
|
||||
"status": status,
|
||||
"sd_hash": sd_hash,
|
||||
"key": stream_key,
|
||||
"stream_name": stream_name,
|
||||
"suggested_file_name": suggested_file_name
|
||||
}
|
||||
|
||||
def _get_all_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
|
||||
file_infos = list(map(lambda a: _lbry_file_dict(*a), transaction.execute(
|
||||
"select file.rowid, file.*, stream.* "
|
||||
"from file inner join stream on file.stream_hash=stream.stream_hash"
|
||||
).fetchall()))
|
||||
stream_hashes = [file_info['stream_hash'] for file_info in file_infos]
|
||||
claim_infos = get_claims_from_stream_hashes(transaction, stream_hashes)
|
||||
for index in range(len(file_infos)): # pylint: disable=consider-using-enumerate
|
||||
file_infos[index]['claim'] = claim_infos.get(file_infos[index]['stream_hash'])
|
||||
return file_infos
|
||||
|
||||
results = await self.db.run(_get_all_files)
|
||||
if results:
|
||||
return results
|
||||
return []
|
||||
def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]:
|
||||
return self.db.run(get_all_lbry_files)
|
||||
|
||||
def change_file_status(self, stream_hash: str, new_status: str):
|
||||
log.info("update file status %s -> %s", stream_hash, new_status)
|
||||
log.debug("update file status %s -> %s", stream_hash, new_status)
|
||||
return self.db.execute("update file set status=? where stream_hash=?", (new_status, stream_hash))
|
||||
|
||||
def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: str, file_name: str):
|
||||
return self.db.execute("update file set download_directory=?, file_name=? where stream_hash=?", (
|
||||
binascii.hexlify(download_dir.encode()).decode(), binascii.hexlify(file_name.encode()).decode(),
|
||||
stream_hash
|
||||
))
|
||||
|
||||
async def recover_streams(self, descriptors_and_sds: typing.List[typing.Tuple['StreamDescriptor', 'BlobFile']],
|
||||
download_directory: str):
|
||||
def _recover(transaction: sqlite3.Connection):
|
||||
stream_hashes = [d.stream_hash for d, s in descriptors_and_sds]
|
||||
for descriptor, sd_blob in descriptors_and_sds:
|
||||
content_claim = transaction.execute(
|
||||
"select * from content_claim where stream_hash=?", (descriptor.stream_hash, )
|
||||
).fetchone()
|
||||
delete_stream(transaction, descriptor) # this will also delete the content claim
|
||||
store_stream(transaction, sd_blob, descriptor)
|
||||
store_file(transaction, descriptor.stream_hash, os.path.basename(descriptor.suggested_file_name),
|
||||
download_directory, 0.0, 'stopped')
|
||||
if content_claim:
|
||||
transaction.execute("insert or ignore into content_claim values (?, ?)", content_claim)
|
||||
transaction.executemany(
|
||||
"update file set status='stopped' where stream_hash=?",
|
||||
[(stream_hash, ) for stream_hash in stream_hashes]
|
||||
)
|
||||
download_dir = binascii.hexlify(self.conf.download_dir.encode()).decode()
|
||||
transaction.executemany(
|
||||
f"update file set download_directory=? where stream_hash=?",
|
||||
[(download_dir, stream_hash) for stream_hash in stream_hashes]
|
||||
)
|
||||
await self.db.run_with_foreign_keys_disabled(_recover)
|
||||
|
||||
def get_all_stream_hashes(self):
|
||||
return self.run_and_return_list("select stream_hash from stream")
|
||||
|
||||
@@ -682,7 +768,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
if success:
|
||||
return self.db.execute(
|
||||
"insert or replace into reflected_stream values (?, ?, ?)",
|
||||
(sd_hash, reflector_address, self.loop.time())
|
||||
(sd_hash, reflector_address, self.time_getter())
|
||||
)
|
||||
return self.db.execute(
|
||||
"delete from reflected_stream where sd_hash=? and reflector_address=?",
|
||||
@@ -694,5 +780,5 @@ class SQLiteStorage(SQLiteMixin):
|
||||
"select s.sd_hash from stream s "
|
||||
"left outer join reflected_stream r on s.sd_hash=r.sd_hash "
|
||||
"where r.timestamp is null or r.timestamp < ?",
|
||||
self.loop.time() - self.conf.auto_re_reflect_interval
|
||||
int(self.time_getter()) - 86400
|
||||
)
|
||||
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
|
||||
__node_cli__ = 'lbrycrd-cli'
|
||||
__node_bin__ = ''
|
||||
__node_url__ = (
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.12.2.1/lbrycrd-linux.zip'
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.12.4.0/lbrycrd-linux.zip'
|
||||
)
|
||||
__spvserver__ = 'lbrynet.extras.wallet.server.coin.LBCRegTest'
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import json
|
||||
import logging
|
||||
import binascii
|
||||
from hashlib import sha256
|
||||
|
||||
from lbrynet.schema.validator import validate_claim_id
|
||||
from torba.client.baseaccount import BaseAccount
|
||||
@@ -24,6 +25,17 @@ class Account(BaseAccount):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.certificates = {}
|
||||
|
||||
@property
|
||||
def hash(self) -> bytes:
|
||||
h = sha256(json.dumps(self.to_dict(False)).encode())
|
||||
for cert in sorted(self.certificates.keys()):
|
||||
h.update(cert.encode())
|
||||
return h.digest()
|
||||
|
||||
def apply(self, d: dict):
|
||||
super().apply(d)
|
||||
self.certificates.update(d.get('certificates', {}))
|
||||
|
||||
def add_certificate_private_key(self, ref: TXORef, private_key):
|
||||
assert ref.id not in self.certificates, 'Trying to add a duplicate certificate.'
|
||||
self.certificates[ref.id] = private_key
|
||||
@@ -176,9 +188,10 @@ class Account(BaseAccount):
|
||||
account.certificates = d.get('certificates', {})
|
||||
return account
|
||||
|
||||
def to_dict(self):
|
||||
def to_dict(self, include_certificates=True):
|
||||
d = super().to_dict()
|
||||
d['certificates'] = self.certificates
|
||||
if include_certificates:
|
||||
d['certificates'] = self.certificates
|
||||
return d
|
||||
|
||||
async def get_details(self, **kwargs):
|
||||
|
||||
@@ -74,9 +74,10 @@ def verify_proof(proof, root_hash, name):
|
||||
if 'txhash' in proof and 'nOut' in proof:
|
||||
if not verified_value:
|
||||
raise InvalidProofError("mismatch between proof claim and outcome")
|
||||
target = reverse_computed_name[::-1].encode('ISO-8859-1').decode()
|
||||
if 'txhash' in proof and 'nOut' in proof:
|
||||
if name != reverse_computed_name[::-1]:
|
||||
if name != target:
|
||||
raise InvalidProofError("name did not match proof")
|
||||
if not name.startswith(reverse_computed_name[::-1]):
|
||||
if not name.startswith(target):
|
||||
raise InvalidProofError("name fragment does not match proof")
|
||||
return True
|
||||
@@ -260,7 +260,6 @@ class LbryWalletManager(BaseWalletManager):
|
||||
async def resolve(self, *uris, **kwargs):
|
||||
page = kwargs.get('page', 0)
|
||||
page_size = kwargs.get('page_size', 10)
|
||||
check_cache = kwargs.get('check_cache', False) # TODO: put caching back (was force_refresh parameter)
|
||||
ledger: MainNetLedger = self.default_account.ledger
|
||||
results = await ledger.resolve(page, page_size, *uris)
|
||||
if 'error' not in results:
|
||||
|
||||
@@ -369,8 +369,14 @@ def _verify_proof(name, claim_trie_root, result, height, depth, transaction_clas
|
||||
return {'error': 'name is not claimed'}
|
||||
|
||||
if 'proof' in result:
|
||||
proof_name = name
|
||||
if 'name' in result:
|
||||
proof_name = result['name']
|
||||
name = result['name']
|
||||
if 'normalized_name' in result:
|
||||
proof_name = result['normalized_name']
|
||||
try:
|
||||
verify_proof(result['proof'], claim_trie_root, name)
|
||||
verify_proof(result['proof'], claim_trie_root, proof_name)
|
||||
except InvalidProofError:
|
||||
return {'error': "Proof was invalid"}
|
||||
return _parse_proof_result(name, result)
|
||||
@@ -421,7 +427,8 @@ def _decode_claim_result(claim):
|
||||
log.warning('Got an invalid claim while parsing, please report: %s', claim)
|
||||
claim['hex'] = None
|
||||
claim['value'] = None
|
||||
claim['error'] = "Failed to parse: missing value"
|
||||
backend_message = ' SDK message: ' + claim['error'] if 'error' in claim else ''
|
||||
claim['error'] = "Failed to parse: missing value." + backend_message
|
||||
return claim
|
||||
try:
|
||||
decoded = smart_decode(claim['value'])
|
||||
@@ -445,11 +452,11 @@ def _handle_claim_result(results):
|
||||
if results['error'] in ['name is not claimed', 'claim not found']:
|
||||
if 'claim_id' in results:
|
||||
raise UnknownClaimID(results['claim_id'])
|
||||
elif 'name' in results:
|
||||
if 'name' in results:
|
||||
raise UnknownNameError(results['name'])
|
||||
elif 'uri' in results:
|
||||
if 'uri' in results:
|
||||
raise UnknownURI(results['uri'])
|
||||
elif 'outpoint' in results:
|
||||
if 'outpoint' in results:
|
||||
raise UnknownOutpoint(results['outpoint'])
|
||||
raise Exception(results['error'])
|
||||
|
||||
@@ -486,6 +493,7 @@ def pick_winner_from_channel_path_collision(claims_in_channel):
|
||||
continue
|
||||
if winner is None:
|
||||
winner = claim
|
||||
elif claim['claim_sequence'] < winner['claim_sequence']:
|
||||
elif claim['height'] < winner['height'] or \
|
||||
(claim['height'] == winner['height'] and claim['nout'] < winner['nout']):
|
||||
winner = claim
|
||||
return winner
|
||||
@@ -80,7 +80,6 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
if claim_info.cert_id:
|
||||
self.db.put_claim_id_signed_by_cert_id(claim_info.cert_id, claim_id)
|
||||
self.db.put_claim_info(claim_id, claim_info)
|
||||
self.db.put_claim_for_name(claim_info.name, claim_id)
|
||||
self.db.put_claim_id_for_outpoint(txid, nout, claim_id)
|
||||
return claim_id, None
|
||||
|
||||
@@ -116,7 +115,6 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
if undo_claim_info.cert_id:
|
||||
cert_id = self._checksig(undo_claim_info.name, undo_claim_info.value, undo_claim_info.address)
|
||||
self.db.put_claim_id_signed_by_cert_id(cert_id, claim_id)
|
||||
self.db.put_claim_for_name(undo_claim_info.name, claim_id)
|
||||
self.db.put_claim_id_for_outpoint(undo_claim_info.txid, undo_claim_info.nout, claim_id)
|
||||
|
||||
def backup_txs(self, txs):
|
||||
|
||||
@@ -13,10 +13,9 @@ class LBRYDB(DB):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.claim_cache = {}
|
||||
self.claims_for_name_cache = {}
|
||||
self.claims_signed_by_cert_cache = {}
|
||||
self.outpoint_to_claim_id_cache = {}
|
||||
self.claims_db = self.names_db = self.signatures_db = self.outpoint_to_claim_id_db = self.claim_undo_db = None
|
||||
self.claims_db = self.signatures_db = self.outpoint_to_claim_id_db = self.claim_undo_db = None
|
||||
# stores deletes not yet flushed to disk
|
||||
self.pending_abandons = {}
|
||||
super().__init__(*args, **kwargs)
|
||||
@@ -24,7 +23,6 @@ class LBRYDB(DB):
|
||||
def close(self):
|
||||
self.batched_flush_claims()
|
||||
self.claims_db.close()
|
||||
self.names_db.close()
|
||||
self.signatures_db.close()
|
||||
self.outpoint_to_claim_id_db.close()
|
||||
self.claim_undo_db.close()
|
||||
@@ -42,12 +40,10 @@ class LBRYDB(DB):
|
||||
return
|
||||
log_reason('closing claim DBs to re-open', for_sync)
|
||||
self.claims_db.close()
|
||||
self.names_db.close()
|
||||
self.signatures_db.close()
|
||||
self.outpoint_to_claim_id_db.close()
|
||||
self.claim_undo_db.close()
|
||||
self.claims_db = self.db_class('claims', for_sync)
|
||||
self.names_db = self.db_class('names', for_sync)
|
||||
self.signatures_db = self.db_class('signatures', for_sync)
|
||||
self.outpoint_to_claim_id_db = self.db_class('outpoint_claim_id', for_sync)
|
||||
self.claim_undo_db = self.db_class('claim_undo', for_sync)
|
||||
@@ -60,21 +56,18 @@ class LBRYDB(DB):
|
||||
|
||||
def batched_flush_claims(self):
|
||||
with self.claims_db.write_batch() as claims_batch:
|
||||
with self.names_db.write_batch() as names_batch:
|
||||
with self.signatures_db.write_batch() as signed_claims_batch:
|
||||
with self.outpoint_to_claim_id_db.write_batch() as outpoint_batch:
|
||||
self.flush_claims(claims_batch, names_batch, signed_claims_batch,
|
||||
outpoint_batch)
|
||||
with self.signatures_db.write_batch() as signed_claims_batch:
|
||||
with self.outpoint_to_claim_id_db.write_batch() as outpoint_batch:
|
||||
self.flush_claims(claims_batch, signed_claims_batch, outpoint_batch)
|
||||
|
||||
def flush_claims(self, batch, names_batch, signed_claims_batch, outpoint_batch):
|
||||
def flush_claims(self, batch, signed_claims_batch, outpoint_batch):
|
||||
flush_start = time.time()
|
||||
write_claim, write_name, write_cert = batch.put, names_batch.put, signed_claims_batch.put
|
||||
write_claim, write_cert = batch.put, signed_claims_batch.put
|
||||
write_outpoint = outpoint_batch.put
|
||||
delete_claim, delete_outpoint, delete_name = batch.delete, outpoint_batch.delete, names_batch.delete
|
||||
delete_claim, delete_outpoint = batch.delete, outpoint_batch.delete
|
||||
delete_cert = signed_claims_batch.delete
|
||||
for claim_id, outpoints in self.pending_abandons.items():
|
||||
claim = self.get_claim_info(claim_id)
|
||||
self.remove_claim_for_name(claim.name, claim_id)
|
||||
if claim.cert_id:
|
||||
self.remove_claim_from_certificate_claims(claim.cert_id, claim_id)
|
||||
self.remove_certificate(claim_id)
|
||||
@@ -86,11 +79,6 @@ class LBRYDB(DB):
|
||||
write_claim(key, claim)
|
||||
else:
|
||||
delete_claim(key)
|
||||
for name, claims in self.claims_for_name_cache.items():
|
||||
if not claims:
|
||||
delete_name(name)
|
||||
else:
|
||||
write_name(name, msgpack.dumps(claims))
|
||||
for cert_id, claims in self.claims_signed_by_cert_cache.items():
|
||||
if not claims:
|
||||
delete_cert(cert_id)
|
||||
@@ -101,15 +89,13 @@ class LBRYDB(DB):
|
||||
write_outpoint(key, claim_id)
|
||||
else:
|
||||
delete_outpoint(key)
|
||||
self.logger.info('flushed at height {:,d} with {:,d} claims, {:,d} outpoints, {:,d} names '
|
||||
self.logger.info('flushed at height {:,d} with {:,d} claims, {:,d} outpoints '
|
||||
'and {:,d} certificates added while {:,d} were abandoned in {:.1f}s, committing...'
|
||||
.format(self.db_height,
|
||||
len(self.claim_cache), len(self.outpoint_to_claim_id_cache),
|
||||
len(self.claims_for_name_cache),
|
||||
len(self.claims_signed_by_cert_cache), len(self.pending_abandons),
|
||||
time.time() - flush_start))
|
||||
self.claim_cache = {}
|
||||
self.claims_for_name_cache = {}
|
||||
self.claims_signed_by_cert_cache = {}
|
||||
self.outpoint_to_claim_id_cache = {}
|
||||
self.pending_abandons = {}
|
||||
@@ -117,7 +103,6 @@ class LBRYDB(DB):
|
||||
def assert_flushed(self, flush_data):
|
||||
super().assert_flushed(flush_data)
|
||||
assert not self.claim_cache
|
||||
assert not self.claims_for_name_cache
|
||||
assert not self.claims_signed_by_cert_cache
|
||||
assert not self.outpoint_to_claim_id_cache
|
||||
assert not self.pending_abandons
|
||||
@@ -142,27 +127,6 @@ class LBRYDB(DB):
|
||||
key = tx_hash + struct.pack('>I', tx_idx)
|
||||
return self.outpoint_to_claim_id_cache.get(key) or self.outpoint_to_claim_id_db.get(key)
|
||||
|
||||
def get_claims_for_name(self, name):
|
||||
if name in self.claims_for_name_cache:
|
||||
return self.claims_for_name_cache[name]
|
||||
db_claims = self.names_db.get(name)
|
||||
return msgpack.loads(db_claims) if db_claims else {}
|
||||
|
||||
def put_claim_for_name(self, name, claim_id):
|
||||
self.logger.info("[+] Adding claim {} for name {}.".format(hash_to_hex_str(claim_id), name))
|
||||
claims = self.get_claims_for_name(name)
|
||||
claims.setdefault(claim_id, max(claims.values() or [0]) + 1)
|
||||
self.claims_for_name_cache[name] = claims
|
||||
|
||||
def remove_claim_for_name(self, name, claim_id):
|
||||
self.logger.info("[-] Removing claim from name: {} - {}".format(hash_to_hex_str(claim_id), name))
|
||||
claims = self.get_claims_for_name(name)
|
||||
claim_n = claims.pop(claim_id)
|
||||
for _claim_id, number in claims.items():
|
||||
if number > claim_n:
|
||||
claims[_claim_id] = number - 1
|
||||
self.claims_for_name_cache[name] = claims
|
||||
|
||||
def get_signed_claim_ids_by_cert_id(self, cert_id):
|
||||
if cert_id in self.claims_signed_by_cert_cache:
|
||||
return self.claims_signed_by_cert_cache[cert_id]
|
||||
@@ -170,17 +134,20 @@ class LBRYDB(DB):
|
||||
return msgpack.loads(db_claims, use_list=True) if db_claims else []
|
||||
|
||||
def put_claim_id_signed_by_cert_id(self, cert_id, claim_id):
|
||||
self.logger.info("[+] Adding signature: {} - {}".format(hash_to_hex_str(claim_id), hash_to_hex_str(cert_id)))
|
||||
msg = "[+] Adding signature: {} - {}".format(hash_to_hex_str(claim_id), hash_to_hex_str(cert_id))
|
||||
self.logger.info(msg)
|
||||
certs = self.get_signed_claim_ids_by_cert_id(cert_id)
|
||||
certs.append(claim_id)
|
||||
self.claims_signed_by_cert_cache[cert_id] = certs
|
||||
|
||||
def remove_certificate(self, cert_id):
|
||||
self.logger.info("[-] Removing certificate: {}".format(hash_to_hex_str(cert_id)))
|
||||
msg = "[-] Removing certificate: {}".format(hash_to_hex_str(cert_id))
|
||||
self.logger.info(msg)
|
||||
self.claims_signed_by_cert_cache[cert_id] = []
|
||||
|
||||
def remove_claim_from_certificate_claims(self, cert_id, claim_id):
|
||||
self.logger.info("[-] Removing signature: {} - {}".format(hash_to_hex_str(claim_id), hash_to_hex_str(cert_id)))
|
||||
msg = "[-] Removing signature: {} - {}".format(hash_to_hex_str(claim_id), hash_to_hex_str(cert_id))
|
||||
self.logger.info(msg)
|
||||
certs = self.get_signed_claim_ids_by_cert_id(cert_id)
|
||||
if claim_id in certs:
|
||||
certs.remove(claim_id)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import math
|
||||
import unicodedata as uda
|
||||
from binascii import unhexlify, hexlify
|
||||
|
||||
from torba.rpc.jsonrpc import RPCError
|
||||
@@ -6,7 +7,7 @@ from torba.server.hash import hash_to_hex_str
|
||||
from torba.server.session import ElectrumX
|
||||
from torba.server import util
|
||||
|
||||
from lbrynet.schema.uri import parse_lbry_uri
|
||||
from lbrynet.schema.uri import parse_lbry_uri, CLAIM_ID_MAX_LENGTH
|
||||
from lbrynet.schema.error import URIParseError
|
||||
from lbrynet.extras.wallet.server.block_processor import LBRYBlockProcessor
|
||||
from lbrynet.extras.wallet.server.db import LBRYDB
|
||||
@@ -131,22 +132,29 @@ class LBRYElectrumX(ElectrumX):
|
||||
claim_ids = self.get_claim_ids_signed_by(certificate_id)
|
||||
return await self.batched_formatted_claims_from_daemon(claim_ids)
|
||||
|
||||
def claimtrie_getclaimssignedbyidminimal(self, certificate_id):
|
||||
claim_ids = self.get_claim_ids_signed_by(certificate_id)
|
||||
ret = []
|
||||
for claim_id in claim_ids:
|
||||
raw_claim_id = unhexlify(claim_id)[::-1]
|
||||
info = self.db.get_claim_info(raw_claim_id)
|
||||
if info:
|
||||
ret.append({
|
||||
'claim_id': claim_id,
|
||||
'height': info.height,
|
||||
'name': info.name.decode()
|
||||
})
|
||||
return ret
|
||||
|
||||
def get_claim_ids_signed_by(self, certificate_id):
|
||||
raw_certificate_id = unhexlify(certificate_id)[::-1]
|
||||
raw_claim_ids = self.db.get_signed_claim_ids_by_cert_id(raw_certificate_id)
|
||||
return list(map(hash_to_hex_str, raw_claim_ids))
|
||||
|
||||
def get_signed_claims_with_name_for_channel(self, channel_id, name):
|
||||
claim_ids_for_name = list(self.db.get_claims_for_name(name.encode('ISO-8859-1')).keys())
|
||||
claim_ids_for_name = set(map(hash_to_hex_str, claim_ids_for_name))
|
||||
channel_claim_ids = set(self.get_claim_ids_signed_by(channel_id))
|
||||
return claim_ids_for_name.intersection(channel_claim_ids)
|
||||
|
||||
async def claimtrie_getclaimssignedbynthtoname(self, name, n):
|
||||
n = int(n)
|
||||
for claim_id, sequence in self.db.get_claims_for_name(name.encode('ISO-8859-1')).items():
|
||||
if n == sequence:
|
||||
return await self.claimtrie_getclaimssignedbyid(hash_to_hex_str(claim_id))
|
||||
claim = self.claimtrie_getnthclaimforname(name, n)
|
||||
if claim and 'claim_id' in claim:
|
||||
return await self.claimtrie_getclaimssignedbyid(hash_to_hex_str(claim['claim_id']))
|
||||
|
||||
async def claimtrie_getclaimsintx(self, txid):
|
||||
# TODO: this needs further discussion.
|
||||
@@ -161,28 +169,27 @@ class LBRYElectrumX(ElectrumX):
|
||||
if proof_has_winning_claim(proof):
|
||||
tx_hash, nout = proof['txhash'], int(proof['nOut'])
|
||||
transaction_info = await self.daemon.getrawtransaction(tx_hash, True)
|
||||
result['transaction'] = transaction_info['hex']
|
||||
result['height'] = (self.db.db_height - transaction_info['confirmations']) + 1
|
||||
result['transaction'] = transaction_info['hex'] # should have never included this (or the call to get it)
|
||||
raw_claim_id = self.db.get_claim_id_from_outpoint(unhexlify(tx_hash)[::-1], nout)
|
||||
sequence = self.db.get_claims_for_name(name.encode('ISO-8859-1')).get(raw_claim_id)
|
||||
if sequence:
|
||||
claim_id = hexlify(raw_claim_id[::-1]).decode()
|
||||
claim_info = await self.daemon.getclaimbyid(claim_id)
|
||||
if not claim_info or not claim_info.get('value'):
|
||||
claim_info = await self.slow_get_claim_by_id_using_name(claim_id)
|
||||
result['claim_sequence'] = sequence
|
||||
result['claim_id'] = claim_id
|
||||
supports = self.format_supports_from_daemon(claim_info.get('supports', [])) # fixme: lbrycrd#124
|
||||
result['supports'] = supports
|
||||
else:
|
||||
self.logger.warning('tx has no claims in db: %s %s', tx_hash, nout)
|
||||
claim_id = hexlify(raw_claim_id[::-1]).decode()
|
||||
claim = await self.claimtrie_getclaimbyid(claim_id)
|
||||
result.update(claim)
|
||||
|
||||
return result
|
||||
|
||||
async def claimtrie_getnthclaimforname(self, name, n):
|
||||
n = int(n)
|
||||
for claim_id, sequence in self.db.get_claims_for_name(name.encode('ISO-8859-1')).items():
|
||||
if n == sequence:
|
||||
return await self.claimtrie_getclaimbyid(hash_to_hex_str(claim_id))
|
||||
result = await self.claimtrie_getclaimsforname(name)
|
||||
if 'claims' in result and len(result['claims']) > n >= 0:
|
||||
# TODO: revist this after lbrycrd_#209 to see if we can sort by claim_sequence at this point
|
||||
result['claims'].sort(key=lambda c: (int(c['height']), int(c['nout'])))
|
||||
result['claims'][n]['claim_sequence'] = n
|
||||
return result['claims'][n]
|
||||
|
||||
async def claimtrie_getpartialmatch(self, name, part):
|
||||
result = await self.claimtrie_getclaimsforname(name)
|
||||
if 'claims' in result:
|
||||
return next(filter(lambda x: x['claim_id'].starts_with(part), result['claims']), None)
|
||||
|
||||
async def claimtrie_getclaimsforname(self, name):
|
||||
claims = await self.daemon.getclaimsforname(name)
|
||||
@@ -198,37 +205,38 @@ class LBRYElectrumX(ElectrumX):
|
||||
async def batched_formatted_claims_from_daemon(self, claim_ids):
|
||||
claims = await self.daemon.getclaimsbyids(claim_ids)
|
||||
result = []
|
||||
for claim, claim_id in zip(claims, claim_ids):
|
||||
for claim in claims:
|
||||
if claim and claim.get('value'):
|
||||
result.append(self.format_claim_from_daemon(claim))
|
||||
else:
|
||||
recovered_claim = await self.slow_get_claim_by_id_using_name(claim_id)
|
||||
if recovered_claim:
|
||||
result.append(self.format_claim_from_daemon(recovered_claim))
|
||||
return result
|
||||
|
||||
def format_claim_from_daemon(self, claim, name=None):
|
||||
'''Changes the returned claim data to the format expected by lbrynet and adds missing fields.'''
|
||||
"""Changes the returned claim data to the format expected by lbrynet and adds missing fields."""
|
||||
|
||||
if not claim:
|
||||
return {}
|
||||
name = name or claim['name']
|
||||
|
||||
# this ISO-8859 nonsense stems from a nasty form of encoding extended characters in lbrycrd
|
||||
# it will be fixed after the lbrycrd upstream merge to v17 is done
|
||||
# it originated as a fear of terminals not supporting unicode. alas, they all do
|
||||
|
||||
if 'name' in claim:
|
||||
name = claim['name'].encode('ISO-8859-1').decode()
|
||||
claim_id = claim['claimId']
|
||||
raw_claim_id = unhexlify(claim_id)[::-1]
|
||||
if not self.db.get_claim_info(raw_claim_id):
|
||||
#raise RPCError("Lbrycrd has {} but not lbryumx, please submit a bug report.".format(claim_id))
|
||||
info = self.db.get_claim_info(raw_claim_id)
|
||||
if not info:
|
||||
# raise RPCError("Lbrycrd has {} but not lbryumx, please submit a bug report.".format(claim_id))
|
||||
return {}
|
||||
address = self.db.get_claim_info(raw_claim_id).address.decode()
|
||||
sequence = self.db.get_claims_for_name(name.encode('ISO-8859-1')).get(raw_claim_id)
|
||||
if not sequence:
|
||||
return {}
|
||||
supports = self.format_supports_from_daemon(claim.get('supports', [])) # fixme: lbrycrd#124
|
||||
address = info.address.decode()
|
||||
supports = self.format_supports_from_daemon(claim.get('supports', []))
|
||||
|
||||
amount = get_from_possible_keys(claim, 'amount', 'nAmount')
|
||||
height = get_from_possible_keys(claim, 'height', 'nHeight')
|
||||
effective_amount = get_from_possible_keys(claim, 'effective amount', 'nEffectiveAmount')
|
||||
valid_at_height = get_from_possible_keys(claim, 'valid at height', 'nValidAtHeight')
|
||||
|
||||
return {
|
||||
result = {
|
||||
"name": name,
|
||||
"claim_id": claim['claimId'],
|
||||
"txid": claim['txid'],
|
||||
@@ -237,12 +245,19 @@ class LBRYElectrumX(ElectrumX):
|
||||
"depth": self.db.db_height - height,
|
||||
"height": height,
|
||||
"value": hexlify(claim['value'].encode('ISO-8859-1')).decode(),
|
||||
"claim_sequence": sequence, # from index
|
||||
"address": address, # from index
|
||||
"supports": supports, # fixme: to be included in lbrycrd#124
|
||||
"supports": supports,
|
||||
"effective_amount": effective_amount,
|
||||
"valid_at_height": valid_at_height # TODO PR lbrycrd to include it
|
||||
"valid_at_height": valid_at_height
|
||||
}
|
||||
if 'claim_sequence' in claim:
|
||||
# TODO: ensure that lbrycrd #209 fills in this value
|
||||
result['claim_sequence'] = claim['claim_sequence']
|
||||
else:
|
||||
result['claim_sequence'] = -1
|
||||
if 'normalized_name' in claim:
|
||||
result['normalized_name'] = claim['normalized_name'].encode('ISO-8859-1').decode()
|
||||
return result
|
||||
|
||||
def format_supports_from_daemon(self, supports):
|
||||
return [[support['txid'], support['n'], get_from_possible_keys(support, 'amount', 'nAmount')] for
|
||||
@@ -251,8 +266,6 @@ class LBRYElectrumX(ElectrumX):
|
||||
async def claimtrie_getclaimbyid(self, claim_id):
|
||||
self.assert_claim_id(claim_id)
|
||||
claim = await self.daemon.getclaimbyid(claim_id)
|
||||
if not claim or not claim.get('value'):
|
||||
claim = await self.slow_get_claim_by_id_using_name(claim_id)
|
||||
return self.format_claim_from_daemon(claim)
|
||||
|
||||
async def claimtrie_getclaimsbyids(self, *claim_ids):
|
||||
@@ -279,20 +292,16 @@ class LBRYElectrumX(ElectrumX):
|
||||
pass
|
||||
raise RPCError(1, f'{value} should be a claim id hash')
|
||||
|
||||
async def slow_get_claim_by_id_using_name(self, claim_id):
|
||||
# TODO: temporary workaround for a lbrycrd bug on indexing. Should be removed when it gets stable
|
||||
raw_claim_id = unhexlify(claim_id)[::-1]
|
||||
claim = self.db.get_claim_info(raw_claim_id)
|
||||
if claim:
|
||||
name = claim.name.decode('ISO-8859-1')
|
||||
claims = await self.daemon.getclaimsforname(name)
|
||||
for claim in claims['claims']:
|
||||
if claim['claimId'] == claim_id:
|
||||
claim['name'] = name
|
||||
self.logger.warning(
|
||||
'Recovered a claim missing from lbrycrd index: %s %s', name, claim_id
|
||||
)
|
||||
return claim
|
||||
def normalize_name(self, name):
|
||||
# this is designed to match lbrycrd; change it here if it changes there
|
||||
return uda.normalize('NFD', name).casefold()
|
||||
|
||||
def claim_matches_name(self, claim, name):
|
||||
if not name:
|
||||
return False
|
||||
if 'normalized_name' in claim:
|
||||
return self.normalize_name(name) == claim['normalized_name']
|
||||
return name == claim['name']
|
||||
|
||||
async def claimtrie_getvalueforuri(self, block_hash, uri, known_certificates=None):
|
||||
# TODO: this thing is huge, refactor
|
||||
@@ -312,8 +321,11 @@ class LBRYElectrumX(ElectrumX):
|
||||
|
||||
# TODO: this is also done on the else, refactor
|
||||
if parsed_uri.claim_id:
|
||||
certificate_info = await self.claimtrie_getclaimbyid(parsed_uri.claim_id)
|
||||
if certificate_info and certificate_info['name'] == parsed_uri.name:
|
||||
if len(parsed_uri.claim_id) < CLAIM_ID_MAX_LENGTH:
|
||||
certificate_info = self.claimtrie_getpartialmatch(parsed_uri.name, parsed_uri.claim_id)
|
||||
else:
|
||||
certificate_info = await self.claimtrie_getclaimbyid(parsed_uri.claim_id)
|
||||
if certificate_info and self.claim_matches_name(certificate_info, parsed_uri.name):
|
||||
certificate = {'resolution_type': CLAIM_ID, 'result': certificate_info}
|
||||
elif parsed_uri.claim_sequence:
|
||||
certificate_info = await self.claimtrie_getnthclaimforname(parsed_uri.name, parsed_uri.claim_sequence)
|
||||
@@ -327,26 +339,28 @@ class LBRYElectrumX(ElectrumX):
|
||||
if certificate and 'claim_id' not in certificate['result']:
|
||||
return result
|
||||
|
||||
if certificate and not parsed_uri.path:
|
||||
if certificate:
|
||||
result['certificate'] = certificate
|
||||
channel_id = certificate['result']['claim_id']
|
||||
claims_in_channel = await self.claimtrie_getclaimssignedbyid(channel_id)
|
||||
result['unverified_claims_in_channel'] = {claim['claim_id']: (claim['name'], claim['height'])
|
||||
for claim in claims_in_channel if claim}
|
||||
elif certificate:
|
||||
result['certificate'] = certificate
|
||||
channel_id = certificate['result']['claim_id']
|
||||
claim_ids_matching_name = self.get_signed_claims_with_name_for_channel(channel_id, parsed_uri.path)
|
||||
claims = await self.batched_formatted_claims_from_daemon(claim_ids_matching_name)
|
||||
claims_in_channel = self.claimtrie_getclaimssignedbyidminimal(channel_id)
|
||||
if not parsed_uri.path:
|
||||
result['unverified_claims_in_channel'] = {claim['claim_id']: (claim['name'], claim['height'])
|
||||
for claim in claims_in_channel}
|
||||
else:
|
||||
# making an assumption that there aren't case conflicts on an existing channel
|
||||
norm_path = self.normalize_name(parsed_uri.path)
|
||||
result['unverified_claims_for_name'] = {claim['claim_id']: (claim['name'], claim['height'])
|
||||
for claim in claims_in_channel
|
||||
if self.normalize_name(claim['name']) == norm_path}
|
||||
|
||||
claims_in_channel = {claim['claim_id']: (claim['name'], claim['height'])
|
||||
for claim in claims}
|
||||
result['unverified_claims_for_name'] = claims_in_channel
|
||||
else:
|
||||
claim = None
|
||||
if parsed_uri.claim_id:
|
||||
claim_info = await self.claimtrie_getclaimbyid(parsed_uri.claim_id)
|
||||
if claim_info and claim_info['name'] == parsed_uri.name:
|
||||
if len(parsed_uri.claim_id) < CLAIM_ID_MAX_LENGTH:
|
||||
claim_info = self.claimtrie_getpartialmatch(parsed_uri.name, parsed_uri.claim_id)
|
||||
else:
|
||||
claim_info = await self.claimtrie_getclaimbyid(parsed_uri.claim_id)
|
||||
if claim_info and self.claim_matches_name(claim_info, parsed_uri.name):
|
||||
claim = {'resolution_type': CLAIM_ID, 'result': claim_info}
|
||||
elif parsed_uri.claim_sequence:
|
||||
claim_info = await self.claimtrie_getnthclaimforname(parsed_uri.name, parsed_uri.claim_sequence)
|
||||
@@ -369,6 +383,7 @@ class LBRYElectrumX(ElectrumX):
|
||||
'result': certificate}
|
||||
result['certificate'] = certificate
|
||||
result['claim'] = claim
|
||||
|
||||
return result
|
||||
|
||||
async def claimtrie_getvalueforuris(self, block_hash, *uris):
|
||||
|
||||
@@ -139,7 +139,7 @@ def get_schema_regex():
|
||||
protocol = _named("protocol", re.escape(PROTOCOL))
|
||||
|
||||
# Define basic building blocks
|
||||
valid_name_char = "[a-zA-Z0-9\-]" # these characters are the only valid name characters
|
||||
valid_name_char = "[^=&#:$@%?/]" # from the grammar section of https://spec.lbry.io/
|
||||
name_content = valid_name_char + '+'
|
||||
name_min_channel_length = valid_name_char + '{' + str(CHANNEL_NAME_MIN_LENGTH) + ',}'
|
||||
|
||||
|
||||
+68
-33
@@ -15,13 +15,13 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_next_available_file_name(download_directory: str, file_name: str) -> str:
|
||||
base_name, ext = os.path.splitext(file_name)
|
||||
base_name, ext = os.path.splitext(os.path.basename(file_name))
|
||||
i = 0
|
||||
while os.path.isfile(os.path.join(download_directory, file_name)):
|
||||
i += 1
|
||||
file_name = "%s_%i%s" % (base_name, i, ext)
|
||||
|
||||
return os.path.join(download_directory, file_name)
|
||||
return file_name
|
||||
|
||||
|
||||
async def get_next_available_file_name(loop: asyncio.BaseEventLoop, download_directory: str, file_name: str) -> str:
|
||||
@@ -29,7 +29,9 @@ async def get_next_available_file_name(loop: asyncio.BaseEventLoop, download_dir
|
||||
|
||||
|
||||
class StreamAssembler:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', sd_hash: str):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', sd_hash: str,
|
||||
output_file_name: typing.Optional[str] = None):
|
||||
self.output_file_name = output_file_name
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.sd_hash = sd_hash
|
||||
@@ -43,11 +45,11 @@ class StreamAssembler:
|
||||
self.written_bytes: int = 0
|
||||
|
||||
async def _decrypt_blob(self, blob: 'BlobFile', blob_info: 'BlobInfo', key: str):
|
||||
offset = blob_info.blob_num * (MAX_BLOB_SIZE - 1)
|
||||
if not blob or not self.stream_handle or self.stream_handle.closed:
|
||||
return False
|
||||
|
||||
def _decrypt_and_write():
|
||||
if self.stream_handle.closed:
|
||||
return False
|
||||
offset = blob_info.blob_num * (MAX_BLOB_SIZE - 1)
|
||||
self.stream_handle.seek(offset)
|
||||
_decrypted = blob.decrypt(
|
||||
binascii.unhexlify(key), binascii.unhexlify(blob_info.iv.encode())
|
||||
@@ -55,47 +57,80 @@ class StreamAssembler:
|
||||
self.stream_handle.write(_decrypted)
|
||||
self.stream_handle.flush()
|
||||
self.written_bytes += len(_decrypted)
|
||||
return True
|
||||
log.debug("decrypted %s", blob.blob_hash[:8])
|
||||
|
||||
decrypted = await self.loop.run_in_executor(None, _decrypt_and_write)
|
||||
if decrypted:
|
||||
log.info("decrypted %s", blob.blob_hash[:8])
|
||||
return
|
||||
await self.loop.run_in_executor(None, _decrypt_and_write)
|
||||
return True
|
||||
|
||||
async def setup(self):
|
||||
pass
|
||||
|
||||
async def after_got_descriptor(self):
|
||||
pass
|
||||
|
||||
async def after_finished(self):
|
||||
pass
|
||||
|
||||
async def assemble_decrypted_stream(self, output_dir: str, output_file_name: typing.Optional[str] = None):
|
||||
if not os.path.isdir(output_dir):
|
||||
raise OSError(f"output directory does not exist: '{output_dir}' '{output_file_name}'")
|
||||
await self.setup()
|
||||
self.sd_blob = await self.get_blob(self.sd_hash)
|
||||
await self.blob_manager.blob_completed(self.sd_blob)
|
||||
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(self.loop, self.blob_manager.blob_dir,
|
||||
self.sd_blob)
|
||||
await self.after_got_descriptor()
|
||||
self.output_file_name = output_file_name or self.descriptor.suggested_file_name
|
||||
self.output_file_name = await get_next_available_file_name(self.loop, output_dir, self.output_file_name)
|
||||
self.output_path = os.path.join(output_dir, self.output_file_name)
|
||||
if not self.got_descriptor.is_set():
|
||||
self.got_descriptor.set()
|
||||
self.output_path = await get_next_available_file_name(self.loop, output_dir,
|
||||
output_file_name or self.descriptor.suggested_file_name)
|
||||
|
||||
self.stream_handle = open(self.output_path, 'wb')
|
||||
await self.blob_manager.storage.store_stream(
|
||||
self.sd_blob, self.descriptor
|
||||
)
|
||||
await self.blob_manager.blob_completed(self.sd_blob)
|
||||
written_blobs = None
|
||||
save_tasks = []
|
||||
try:
|
||||
for blob_info in self.descriptor.blobs[:-1]:
|
||||
while True:
|
||||
try:
|
||||
blob = await self.get_blob(blob_info.blob_hash, blob_info.length)
|
||||
await self._decrypt_blob(blob, blob_info, self.descriptor.key)
|
||||
break
|
||||
except ValueError as err:
|
||||
log.error("failed to decrypt blob %s for stream %s - %s", blob_info.blob_hash,
|
||||
self.descriptor.sd_hash, str(err))
|
||||
continue
|
||||
if not self.wrote_bytes_event.is_set():
|
||||
self.wrote_bytes_event.set()
|
||||
self.stream_finished_event.set()
|
||||
with open(self.output_path, 'wb') as stream_handle:
|
||||
self.stream_handle = stream_handle
|
||||
for i, blob_info in enumerate(self.descriptor.blobs[:-1]):
|
||||
if blob_info.blob_num != i:
|
||||
log.error("sd blob %s is invalid, cannot assemble stream", self.descriptor.sd_hash)
|
||||
return
|
||||
while self.stream_handle and not self.stream_handle.closed:
|
||||
try:
|
||||
blob = await self.get_blob(blob_info.blob_hash, blob_info.length)
|
||||
if blob and blob.length != blob_info.length:
|
||||
log.warning("Found incomplete, deleting: %s", blob_info.blob_hash)
|
||||
await self.blob_manager.delete_blobs([blob_info.blob_hash])
|
||||
continue
|
||||
if await self._decrypt_blob(blob, blob_info, self.descriptor.key):
|
||||
save_tasks.append(asyncio.ensure_future(self.blob_manager.blob_completed(blob)))
|
||||
written_blobs = i
|
||||
if not self.wrote_bytes_event.is_set():
|
||||
self.wrote_bytes_event.set()
|
||||
log.debug("written %i/%i", written_blobs, len(self.descriptor.blobs) - 2)
|
||||
break
|
||||
except FileNotFoundError:
|
||||
log.debug("stream assembler stopped")
|
||||
return
|
||||
except (ValueError, IOError, OSError):
|
||||
log.warning("failed to decrypt blob %s for stream %s", blob_info.blob_hash,
|
||||
self.descriptor.sd_hash)
|
||||
continue
|
||||
finally:
|
||||
self.stream_handle.close()
|
||||
if written_blobs == len(self.descriptor.blobs) - 2:
|
||||
log.debug("finished decrypting and assembling stream")
|
||||
if save_tasks:
|
||||
await asyncio.wait(save_tasks)
|
||||
await self.after_finished()
|
||||
self.stream_finished_event.set()
|
||||
else:
|
||||
log.debug("stream decryption and assembly did not finish (%i/%i blobs are done)", written_blobs or 0,
|
||||
len(self.descriptor.blobs) - 2)
|
||||
if self.output_path and os.path.isfile(self.output_path):
|
||||
log.debug("erasing incomplete file assembly: %s", self.output_path)
|
||||
os.unlink(self.output_path)
|
||||
|
||||
async def get_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
|
||||
f = asyncio.Future(loop=self.loop)
|
||||
f.set_result(self.blob_manager.get_blob(blob_hash, length))
|
||||
return await f
|
||||
return self.blob_manager.get_blob(blob_hash, length)
|
||||
@@ -4,6 +4,7 @@ import binascii
|
||||
import logging
|
||||
import typing
|
||||
import asyncio
|
||||
from collections import OrderedDict
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from lbrynet.blob import MAX_BLOB_SIZE
|
||||
from lbrynet.blob.blob_info import BlobInfo
|
||||
@@ -58,6 +59,10 @@ class StreamDescriptor:
|
||||
self.stream_hash = stream_hash or self.get_stream_hash()
|
||||
self.sd_hash = sd_hash
|
||||
|
||||
@property
|
||||
def length(self):
|
||||
return len(self.as_json())
|
||||
|
||||
def get_stream_hash(self) -> str:
|
||||
return self.calculate_stream_hash(
|
||||
binascii.hexlify(self.stream_name.encode()), self.key.encode(),
|
||||
@@ -78,15 +83,46 @@ class StreamDescriptor:
|
||||
[blob_info.as_dict() for blob_info in self.blobs]), sort_keys=True
|
||||
).encode()
|
||||
|
||||
async def make_sd_blob(self):
|
||||
sd_hash = self.calculate_sd_hash()
|
||||
sd_data = self.as_json()
|
||||
sd_blob = BlobFile(self.loop, self.blob_dir, sd_hash, len(sd_data))
|
||||
def old_sort_json(self) -> bytes:
|
||||
blobs = []
|
||||
for b in self.blobs:
|
||||
blobs.append(OrderedDict(
|
||||
[('length', b.length), ('blob_num', b.blob_num), ('iv', b.iv)] if not b.blob_hash else
|
||||
[('length', b.length), ('blob_num', b.blob_num), ('blob_hash', b.blob_hash), ('iv', b.iv)]
|
||||
))
|
||||
if not b.blob_hash:
|
||||
break
|
||||
return json.dumps(
|
||||
OrderedDict([
|
||||
('stream_name', binascii.hexlify(self.stream_name.encode()).decode()),
|
||||
('blobs', blobs),
|
||||
('stream_type', 'lbryfile'),
|
||||
('key', self.key),
|
||||
('suggested_file_name', binascii.hexlify(self.suggested_file_name.encode()).decode()),
|
||||
('stream_hash', self.stream_hash),
|
||||
])
|
||||
).encode()
|
||||
|
||||
def calculate_old_sort_sd_hash(self):
|
||||
h = get_lbry_hash_obj()
|
||||
h.update(self.old_sort_json())
|
||||
return h.hexdigest()
|
||||
|
||||
async def make_sd_blob(self, blob_file_obj: typing.Optional[BlobFile] = None,
|
||||
old_sort: typing.Optional[bool] = False):
|
||||
sd_hash = self.calculate_sd_hash() if not old_sort else self.calculate_old_sort_sd_hash()
|
||||
if not old_sort:
|
||||
sd_data = self.as_json()
|
||||
else:
|
||||
sd_data = self.old_sort_json()
|
||||
sd_blob = blob_file_obj or BlobFile(self.loop, self.blob_dir, sd_hash, len(sd_data))
|
||||
if blob_file_obj:
|
||||
blob_file_obj.set_length(len(sd_data))
|
||||
if not sd_blob.get_is_verified():
|
||||
writer = sd_blob.open_for_writing()
|
||||
writer.write(sd_data)
|
||||
await sd_blob.verified.wait()
|
||||
await sd_blob.close()
|
||||
sd_blob.close()
|
||||
return sd_blob
|
||||
|
||||
@classmethod
|
||||
@@ -95,13 +131,19 @@ class StreamDescriptor:
|
||||
assert os.path.isfile(blob.file_path)
|
||||
with open(blob.file_path, 'rb') as f:
|
||||
json_bytes = f.read()
|
||||
decoded = json.loads(json_bytes.decode())
|
||||
try:
|
||||
decoded = json.loads(json_bytes.decode())
|
||||
except json.JSONDecodeError:
|
||||
blob.delete()
|
||||
raise InvalidStreamDescriptorError("Does not decode as valid JSON")
|
||||
if decoded['blobs'][-1]['length'] != 0:
|
||||
raise InvalidStreamDescriptorError("Does not end with a zero-length blob.")
|
||||
if any([blob_info['length'] == 0 for blob_info in decoded['blobs'][:-1]]):
|
||||
raise InvalidStreamDescriptorError("Contains zero-length data blob")
|
||||
if 'blob_hash' in decoded['blobs'][-1]:
|
||||
raise InvalidStreamDescriptorError("Stream terminator blob should not have a hash")
|
||||
if any([i != blob_info['blob_num'] for i, blob_info in enumerate(decoded['blobs'])]):
|
||||
raise InvalidStreamDescriptorError("Stream contains out of order or skipped blobs")
|
||||
descriptor = cls(
|
||||
loop, blob_dir,
|
||||
binascii.unhexlify(decoded['stream_name']).decode(),
|
||||
@@ -154,8 +196,8 @@ class StreamDescriptor:
|
||||
@classmethod
|
||||
async def create_stream(cls, loop: asyncio.BaseEventLoop, blob_dir: str,
|
||||
file_path: str, key: typing.Optional[bytes] = None,
|
||||
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None
|
||||
) -> 'StreamDescriptor':
|
||||
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None,
|
||||
old_sort: bool = False) -> 'StreamDescriptor':
|
||||
|
||||
blobs: typing.List[BlobInfo] = []
|
||||
|
||||
@@ -174,7 +216,7 @@ class StreamDescriptor:
|
||||
loop, blob_dir, os.path.basename(file_path), binascii.hexlify(key).decode(), os.path.basename(file_path),
|
||||
blobs
|
||||
)
|
||||
sd_blob = await descriptor.make_sd_blob()
|
||||
sd_blob = await descriptor.make_sd_blob(old_sort=old_sort)
|
||||
descriptor.sd_hash = sd_blob.blob_hash
|
||||
return descriptor
|
||||
|
||||
@@ -184,3 +226,19 @@ class StreamDescriptor:
|
||||
|
||||
def upper_bound_decrypted_length(self) -> int:
|
||||
return self.lower_bound_decrypted_length() + (AES.block_size // 8)
|
||||
|
||||
@classmethod
|
||||
async def recover(cls, blob_dir: str, sd_blob: 'BlobFile', stream_hash: str, stream_name: str,
|
||||
suggested_file_name: str, key: str,
|
||||
blobs: typing.List['BlobInfo']) -> typing.Optional['StreamDescriptor']:
|
||||
descriptor = cls(asyncio.get_event_loop(), blob_dir, stream_name, key, suggested_file_name,
|
||||
blobs, stream_hash, sd_blob.blob_hash)
|
||||
|
||||
if descriptor.calculate_sd_hash() == sd_blob.blob_hash: # first check for a normal valid sd
|
||||
old_sort = False
|
||||
elif descriptor.calculate_old_sort_sd_hash() == sd_blob.blob_hash: # check if old field sorting works
|
||||
old_sort = True
|
||||
else:
|
||||
return
|
||||
await descriptor.make_sd_blob(sd_blob, old_sort)
|
||||
return descriptor
|
||||
+78
-205
@@ -2,12 +2,14 @@ import os
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet.utils import drain_tasks, cancel_task
|
||||
from lbrynet.utils import resolve_host
|
||||
from lbrynet.stream.assembler import StreamAssembler
|
||||
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.blob_exchange.downloader import BlobDownloader
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
|
||||
@@ -19,217 +21,88 @@ def drain_into(a: list, b: list):
|
||||
b.append(a.pop())
|
||||
|
||||
|
||||
class StreamDownloader(StreamAssembler): # TODO: reduce duplication, refactor to inherit BlobDownloader
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', sd_hash: str,
|
||||
peer_timeout: float, peer_connect_timeout: float, output_dir: typing.Optional[str] = None,
|
||||
output_file_name: typing.Optional[str] = None,
|
||||
fixed_peers: typing.Optional[typing.List['KademliaPeer']] = None,
|
||||
max_connections_per_stream: typing.Optional[int] = 8):
|
||||
super().__init__(loop, blob_manager, sd_hash)
|
||||
self.peer_timeout = peer_timeout
|
||||
self.peer_connect_timeout = peer_connect_timeout
|
||||
self.current_blob: 'BlobFile' = None
|
||||
|
||||
self.download_task: asyncio.Task = None
|
||||
self.accumulate_connections_task: asyncio.Task = None
|
||||
self.new_peer_event = asyncio.Event(loop=self.loop)
|
||||
self.active_connections: typing.Dict['KademliaPeer', BlobExchangeClientProtocol] = {}
|
||||
self.running_download_requests: typing.List[asyncio.Task] = []
|
||||
self.requested_from: typing.Dict[str, typing.Dict['KademliaPeer', asyncio.Task]] = {}
|
||||
self.output_dir = output_dir or os.getcwd()
|
||||
class StreamDownloader(StreamAssembler):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager', sd_hash: str,
|
||||
output_dir: typing.Optional[str] = None, output_file_name: typing.Optional[str] = None):
|
||||
super().__init__(loop, blob_manager, sd_hash, output_file_name)
|
||||
self.config = config
|
||||
self.output_dir = output_dir or self.config.download_dir
|
||||
self.output_file_name = output_file_name
|
||||
self._lock = asyncio.Lock(loop=self.loop)
|
||||
self.max_connections_per_stream = max_connections_per_stream
|
||||
self.fixed_peers = fixed_peers or []
|
||||
self.blob_downloader: typing.Optional[BlobDownloader] = None
|
||||
self.search_queue = asyncio.Queue(loop=loop)
|
||||
self.peer_queue = asyncio.Queue(loop=loop)
|
||||
self.accumulate_task: typing.Optional[asyncio.Task] = None
|
||||
self.descriptor: typing.Optional[StreamDescriptor]
|
||||
self.node: typing.Optional['Node'] = None
|
||||
self.assemble_task: typing.Optional[asyncio.Task] = None
|
||||
self.fixed_peers_handle: typing.Optional[asyncio.Handle] = None
|
||||
self.fixed_peers_delay: typing.Optional[float] = None
|
||||
self.added_fixed_peers = False
|
||||
|
||||
async def _update_current_blob(self, blob: 'BlobFile'):
|
||||
async with self._lock:
|
||||
drain_tasks(self.running_download_requests)
|
||||
self.current_blob = blob
|
||||
if not blob.get_is_verified():
|
||||
self._update_requests()
|
||||
async def setup(self): # start the peer accumulator and initialize the downloader
|
||||
if self.blob_downloader:
|
||||
raise Exception("downloader is already set up")
|
||||
if self.node:
|
||||
_, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue)
|
||||
self.blob_downloader = BlobDownloader(self.loop, self.config, self.blob_manager, self.peer_queue)
|
||||
self.search_queue.put_nowait(self.sd_hash)
|
||||
|
||||
async def _request_blob(self, peer: 'KademliaPeer'):
|
||||
if self.current_blob.get_is_verified():
|
||||
log.info("already verified")
|
||||
return
|
||||
if peer not in self.active_connections:
|
||||
log.warning("not active, adding: %s", str(peer))
|
||||
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.peer_timeout)
|
||||
protocol = self.active_connections[peer]
|
||||
success, keep_connection = await request_blob(self.loop, self.current_blob, protocol,
|
||||
peer.address, peer.tcp_port, self.peer_connect_timeout)
|
||||
await protocol.close()
|
||||
if not keep_connection:
|
||||
log.info("drop peer %s:%i", peer.address, peer.tcp_port)
|
||||
if peer in self.active_connections:
|
||||
async with self._lock:
|
||||
del self.active_connections[peer]
|
||||
return
|
||||
log.info("keep peer %s:%i", peer.address, peer.tcp_port)
|
||||
async def after_got_descriptor(self):
|
||||
self.search_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
|
||||
log.info("added head blob to search")
|
||||
|
||||
def _update_requests(self):
|
||||
self.new_peer_event.clear()
|
||||
if self.current_blob.blob_hash not in self.requested_from:
|
||||
self.requested_from[self.current_blob.blob_hash] = {}
|
||||
to_add = []
|
||||
for peer in self.active_connections.keys():
|
||||
if peer not in self.requested_from[self.current_blob.blob_hash] and peer not in to_add:
|
||||
to_add.append(peer)
|
||||
if to_add or self.running_download_requests:
|
||||
log.info("adding download probes for %i peers to %i already active",
|
||||
min(len(to_add), 8 - len(self.running_download_requests)),
|
||||
len(self.running_download_requests))
|
||||
else:
|
||||
log.info("downloader idle...")
|
||||
for peer in to_add:
|
||||
if len(self.running_download_requests) >= 8:
|
||||
break
|
||||
task = self.loop.create_task(self._request_blob(peer))
|
||||
self.requested_from[self.current_blob.blob_hash][peer] = task
|
||||
self.running_download_requests.append(task)
|
||||
async def after_finished(self):
|
||||
log.info("downloaded stream %s -> %s", self.sd_hash, self.output_path)
|
||||
await self.blob_manager.storage.change_file_status(self.descriptor.stream_hash, 'finished')
|
||||
self.blob_downloader.close()
|
||||
|
||||
async def wait_for_download_or_new_peer(self) -> typing.Optional['BlobFile']:
|
||||
async with self._lock:
|
||||
if len(self.running_download_requests) < self.max_connections_per_stream:
|
||||
# update the running download requests
|
||||
self._update_requests()
|
||||
|
||||
# drain the tasks into a temporary list
|
||||
download_tasks = []
|
||||
drain_into(self.running_download_requests, download_tasks)
|
||||
|
||||
got_new_peer = self.loop.create_task(self.new_peer_event.wait())
|
||||
|
||||
# wait for a new peer to be added or for a download attempt to finish
|
||||
await asyncio.wait([got_new_peer] + download_tasks, return_when='FIRST_COMPLETED',
|
||||
loop=self.loop)
|
||||
if got_new_peer and not got_new_peer.done():
|
||||
got_new_peer.cancel()
|
||||
async with self._lock:
|
||||
if self.current_blob.get_is_verified():
|
||||
if got_new_peer and not got_new_peer.done():
|
||||
got_new_peer.cancel()
|
||||
drain_tasks(download_tasks)
|
||||
return self.current_blob
|
||||
else:
|
||||
for task in download_tasks:
|
||||
if task and not task.done():
|
||||
self.running_download_requests.append(task)
|
||||
return
|
||||
def stop(self):
|
||||
if self.accumulate_task:
|
||||
self.accumulate_task.cancel()
|
||||
self.accumulate_task = None
|
||||
if self.assemble_task:
|
||||
self.assemble_task.cancel()
|
||||
self.assemble_task = None
|
||||
if self.fixed_peers_handle:
|
||||
self.fixed_peers_handle.cancel()
|
||||
self.fixed_peers_handle = None
|
||||
self.blob_downloader = None
|
||||
if self.stream_handle:
|
||||
if not self.stream_handle.closed:
|
||||
self.stream_handle.close()
|
||||
self.stream_handle = None
|
||||
if not self.stream_finished_event.is_set() and self.wrote_bytes_event.is_set() and \
|
||||
self.output_path and os.path.isfile(self.output_path):
|
||||
os.remove(self.output_path)
|
||||
|
||||
async def get_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
|
||||
blob = self.blob_manager.get_blob(blob_hash, length)
|
||||
await self._update_current_blob(blob)
|
||||
if blob.get_is_verified():
|
||||
return blob
|
||||
return await self.blob_downloader.download_blob(blob_hash, length)
|
||||
|
||||
# the blob must be downloaded
|
||||
try:
|
||||
while not self.current_blob.get_is_verified():
|
||||
if not self.active_connections: # wait for a new connection
|
||||
await self.new_peer_event.wait()
|
||||
continue
|
||||
blob = await self.wait_for_download_or_new_peer()
|
||||
if blob:
|
||||
drain_tasks(self.running_download_requests)
|
||||
return blob
|
||||
return blob
|
||||
except asyncio.CancelledError:
|
||||
drain_tasks(self.running_download_requests)
|
||||
raise
|
||||
def add_fixed_peers(self):
|
||||
async def _add_fixed_peers():
|
||||
addresses = [
|
||||
(await resolve_host(url, port + 1, proto='tcp'), port)
|
||||
for url, port in self.config.reflector_servers
|
||||
]
|
||||
|
||||
def _add_peer_protocols(self, peers: typing.List['KademliaPeer']):
|
||||
added = 0
|
||||
for peer in peers:
|
||||
if peer not in self.active_connections:
|
||||
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.peer_timeout)
|
||||
added += 1
|
||||
if added:
|
||||
if not self.new_peer_event.is_set():
|
||||
log.info("added %i new peers", len(peers))
|
||||
self.new_peer_event.set()
|
||||
def _delayed_add_fixed_peers():
|
||||
self.added_fixed_peers = True
|
||||
self.peer_queue.put_nowait([
|
||||
KademliaPeer(self.loop, address=address, tcp_port=port + 1)
|
||||
for address, port in addresses
|
||||
])
|
||||
|
||||
async def _accumulate_connections(self, node: 'Node'):
|
||||
blob_queue = asyncio.Queue(loop=self.loop)
|
||||
blob_queue.put_nowait(self.sd_hash)
|
||||
task = asyncio.create_task(self.got_descriptor.wait())
|
||||
added_peers = asyncio.Event(loop=self.loop)
|
||||
add_fixed_peers_timer: typing.Optional[asyncio.Handle] = None
|
||||
|
||||
if self.fixed_peers:
|
||||
def check_added_peers():
|
||||
if not added_peers.is_set():
|
||||
self._add_peer_protocols(self.fixed_peers)
|
||||
log.info("no dht peers for download yet, adding fixed peer")
|
||||
added_peers.set()
|
||||
|
||||
add_fixed_peers_timer = self.loop.call_later(2, check_added_peers)
|
||||
|
||||
def got_descriptor(f):
|
||||
try:
|
||||
f.result()
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
log.info("add head blob hash to peer search")
|
||||
blob_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
|
||||
|
||||
task.add_done_callback(got_descriptor)
|
||||
try:
|
||||
async with node.stream_peer_search_junction(blob_queue) as search_junction:
|
||||
async for peers in search_junction:
|
||||
if not isinstance(peers, list): # TODO: what's up with this?
|
||||
log.error("not a list: %s %s", peers, str(type(peers)))
|
||||
else:
|
||||
self._add_peer_protocols(peers)
|
||||
if not added_peers.is_set():
|
||||
added_peers.set()
|
||||
self.fixed_peers_handle = self.loop.call_later(self.fixed_peers_delay, _delayed_add_fixed_peers)
|
||||
if not self.config.reflector_servers:
|
||||
return
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
finally:
|
||||
if task and not task.done():
|
||||
task.cancel()
|
||||
log.info("cancelled head blob task")
|
||||
if add_fixed_peers_timer and not add_fixed_peers_timer.cancelled():
|
||||
add_fixed_peers_timer.cancel()
|
||||
if 'dht' in self.config.components_to_skip or not self.node or not \
|
||||
len(self.node.protocol.routing_table.get_peers()):
|
||||
self.fixed_peers_delay = 0.0
|
||||
else:
|
||||
self.fixed_peers_delay = self.config.fixed_peer_delay
|
||||
self.loop.create_task(_add_fixed_peers())
|
||||
|
||||
async def stop(self):
|
||||
cancel_task(self.accumulate_connections_task)
|
||||
self.accumulate_connections_task = None
|
||||
drain_tasks(self.running_download_requests)
|
||||
|
||||
while self.requested_from:
|
||||
_, peer_task_dict = self.requested_from.popitem()
|
||||
while peer_task_dict:
|
||||
peer, task = peer_task_dict.popitem()
|
||||
try:
|
||||
cancel_task(task)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
while self.active_connections:
|
||||
_, client = self.active_connections.popitem()
|
||||
if client:
|
||||
await client.close()
|
||||
log.info("stopped downloader")
|
||||
|
||||
async def _download(self):
|
||||
try:
|
||||
|
||||
log.info("download and decrypt stream")
|
||||
await self.assemble_decrypted_stream(self.output_dir, self.output_file_name)
|
||||
log.info(
|
||||
"downloaded stream %s -> %s", self.sd_hash, self.output_path
|
||||
)
|
||||
await self.blob_manager.storage.change_file_status(
|
||||
self.descriptor.stream_hash, 'finished'
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
finally:
|
||||
await self.stop()
|
||||
|
||||
def download(self, node: 'Node'):
|
||||
self.accumulate_connections_task = self.loop.create_task(self._accumulate_connections(node))
|
||||
self.download_task = self.loop.create_task(self._download())
|
||||
def download(self, node: typing.Optional['Node'] = None):
|
||||
self.node = node
|
||||
self.assemble_task = self.loop.create_task(self.assemble_decrypted_stream(self.config.download_dir))
|
||||
self.add_fixed_peers()
|
||||
@@ -2,13 +2,17 @@ import os
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
import binascii
|
||||
from lbrynet.utils import generate_id
|
||||
from lbrynet.extras.daemon.mime_types import guess_media_type
|
||||
from lbrynet.stream.downloader import StreamDownloader
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.stream.reflector.client import StreamReflectorClient
|
||||
from lbrynet.extras.daemon.storage import StoredStreamClaim
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.extras.daemon.storage import StoredStreamClaim
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.dht.node import Node
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -18,20 +22,29 @@ class ManagedStream:
|
||||
STATUS_STOPPED = "stopped"
|
||||
STATUS_FINISHED = "finished"
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', descriptor: 'StreamDescriptor',
|
||||
download_directory: str, file_name: str, downloader: typing.Optional[StreamDownloader] = None,
|
||||
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional['StoredStreamClaim'] = None):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', rowid: int,
|
||||
descriptor: 'StreamDescriptor', download_directory: str, file_name: typing.Optional[str],
|
||||
downloader: typing.Optional[StreamDownloader] = None,
|
||||
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional[StoredStreamClaim] = None,
|
||||
download_id: typing.Optional[str] = None):
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.rowid = rowid
|
||||
self.download_directory = download_directory
|
||||
self.file_name = file_name
|
||||
self._file_name = file_name
|
||||
self.descriptor = descriptor
|
||||
self.downloader = downloader
|
||||
self.stream_hash = descriptor.stream_hash
|
||||
self.stream_claim_info = claim
|
||||
self._status = status
|
||||
self._store_after_finished: asyncio.Task = None
|
||||
|
||||
self.fully_reflected = asyncio.Event(loop=self.loop)
|
||||
self.tx = None
|
||||
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
|
||||
|
||||
@property
|
||||
def file_name(self) -> typing.Optional[str]:
|
||||
return self.downloader.output_file_name if self.downloader else self._file_name
|
||||
|
||||
@property
|
||||
def status(self) -> str:
|
||||
@@ -98,13 +111,23 @@ class ManagedStream:
|
||||
def sd_hash(self):
|
||||
return self.descriptor.sd_hash
|
||||
|
||||
def as_dict(self) -> typing.Dict:
|
||||
full_path = os.path.join(self.download_directory, self.file_name)
|
||||
if not os.path.exists(full_path):
|
||||
full_path = None
|
||||
mime_type = guess_media_type(os.path.basename(self.file_name))
|
||||
@property
|
||||
def blobs_remaining(self) -> int:
|
||||
return self.blobs_in_stream - self.blobs_completed
|
||||
|
||||
if self.downloader:
|
||||
@property
|
||||
def full_path(self) -> typing.Optional[str]:
|
||||
return os.path.join(self.download_directory, os.path.basename(self.file_name)) if self.file_name else None
|
||||
|
||||
@property
|
||||
def output_file_exists(self):
|
||||
return os.path.isfile(self.full_path) if self.full_path else False
|
||||
|
||||
def as_dict(self) -> typing.Dict:
|
||||
full_path = self.full_path if self.output_file_exists else None
|
||||
mime_type = guess_media_type(os.path.basename(self.descriptor.suggested_file_name))
|
||||
|
||||
if self.downloader and self.downloader.written_bytes:
|
||||
written_bytes = self.downloader.written_bytes
|
||||
elif full_path:
|
||||
written_bytes = os.stat(full_path).st_size
|
||||
@@ -115,6 +138,7 @@ class ManagedStream:
|
||||
'file_name': self.file_name,
|
||||
'download_directory': self.download_directory,
|
||||
'points_paid': 0.0,
|
||||
'tx': self.tx,
|
||||
'stopped': not self.running,
|
||||
'stream_hash': self.stream_hash,
|
||||
'stream_name': self.descriptor.stream_name,
|
||||
@@ -128,6 +152,7 @@ class ManagedStream:
|
||||
'written_bytes': written_bytes,
|
||||
'blobs_completed': self.blobs_completed,
|
||||
'blobs_in_stream': self.blobs_in_stream,
|
||||
'blobs_remaining': self.blobs_remaining,
|
||||
'status': self.status,
|
||||
'claim_id': self.claim_id,
|
||||
'txid': self.txid,
|
||||
@@ -147,50 +172,60 @@ class ManagedStream:
|
||||
loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator
|
||||
)
|
||||
sd_blob = blob_manager.get_blob(descriptor.sd_hash)
|
||||
await blob_manager.blob_completed(sd_blob)
|
||||
await blob_manager.storage.store_stream(
|
||||
blob_manager.get_blob(descriptor.sd_hash), descriptor
|
||||
)
|
||||
return cls(loop, blob_manager, descriptor, os.path.dirname(file_path), os.path.basename(file_path),
|
||||
await blob_manager.blob_completed(sd_blob)
|
||||
for blob in descriptor.blobs[:-1]:
|
||||
await blob_manager.blob_completed(blob_manager.get_blob(blob.blob_hash, blob.length))
|
||||
row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path),
|
||||
os.path.dirname(file_path), 0)
|
||||
return cls(loop, blob_manager, row_id, descriptor, os.path.dirname(file_path), os.path.basename(file_path),
|
||||
status=cls.STATUS_FINISHED)
|
||||
|
||||
async def stop_download(self):
|
||||
def start_download(self, node: typing.Optional['Node']):
|
||||
self.downloader.download(node)
|
||||
self.update_status(self.STATUS_RUNNING)
|
||||
|
||||
def stop_download(self):
|
||||
if self.downloader:
|
||||
await self.downloader.stop()
|
||||
if not self.finished:
|
||||
self.update_status(self.STATUS_STOPPED)
|
||||
self.downloader.stop()
|
||||
self.downloader = None
|
||||
|
||||
async def upload_to_reflector(self, host: str, port: int) -> typing.List[str]:
|
||||
sent = []
|
||||
protocol = StreamReflectorClient(self.blob_manager, self.descriptor)
|
||||
try:
|
||||
await self.loop.create_connection(lambda: protocol, host, port)
|
||||
except ConnectionRefusedError:
|
||||
return sent
|
||||
try:
|
||||
await protocol.send_handshake()
|
||||
except (asyncio.CancelledError, asyncio.TimeoutError, ValueError):
|
||||
if protocol.transport:
|
||||
protocol.transport.close()
|
||||
return sent
|
||||
try:
|
||||
sent_sd, needed = await protocol.send_descriptor()
|
||||
if sent_sd:
|
||||
sent.append(self.sd_hash)
|
||||
except (asyncio.CancelledError, asyncio.TimeoutError, ValueError):
|
||||
if protocol.transport:
|
||||
protocol.transport.close()
|
||||
return sent
|
||||
for blob_hash in needed:
|
||||
try:
|
||||
if not sent_sd and not needed:
|
||||
if not self.fully_reflected.is_set():
|
||||
self.fully_reflected.set()
|
||||
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
|
||||
return []
|
||||
we_have = [blob_hash for blob_hash in needed if blob_hash in self.blob_manager.completed_blob_hashes]
|
||||
for blob_hash in we_have:
|
||||
await protocol.send_blob(blob_hash)
|
||||
sent.append(blob_hash)
|
||||
except (asyncio.CancelledError, asyncio.TimeoutError, ValueError):
|
||||
if protocol.transport:
|
||||
protocol.transport.close()
|
||||
return sent
|
||||
if protocol.transport:
|
||||
protocol.transport.close()
|
||||
except (asyncio.TimeoutError, ValueError):
|
||||
return sent
|
||||
except ConnectionRefusedError:
|
||||
return sent
|
||||
finally:
|
||||
if protocol.transport:
|
||||
protocol.transport.close()
|
||||
if not self.fully_reflected.is_set():
|
||||
self.fully_reflected.set()
|
||||
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
|
||||
return sent
|
||||
|
||||
def set_claim(self, claim_info: typing.Dict, claim: 'ClaimDict'):
|
||||
self.stream_claim_info = StoredStreamClaim(
|
||||
self.stream_hash, f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
|
||||
claim_info['name'], claim_info['amount'], claim_info['height'],
|
||||
binascii.hexlify(claim.serialized).decode(), claim.certificate_id, claim_info['address'],
|
||||
claim_info['claim_sequence'], claim_info.get('channel_name')
|
||||
)
|
||||
@@ -15,6 +15,7 @@ log = logging.getLogger(__name__)
|
||||
|
||||
class StreamReflectorClient(asyncio.Protocol):
|
||||
def __init__(self, blob_manager: 'BlobFileManager', descriptor: 'StreamDescriptor'):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.transport: asyncio.StreamWriter = None
|
||||
self.blob_manager = blob_manager
|
||||
self.descriptor = descriptor
|
||||
@@ -26,12 +27,14 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
|
||||
def connection_made(self, transport):
|
||||
self.transport = transport
|
||||
log.info("Connected to reflector")
|
||||
log.debug("Connected to reflector")
|
||||
self.connected.set()
|
||||
|
||||
def connection_lost(self, exc: typing.Optional[Exception]):
|
||||
self.transport = None
|
||||
self.connected.clear()
|
||||
if self.reflected_blobs:
|
||||
log.info("Finished sending reflector %i blobs", len(self.reflected_blobs))
|
||||
|
||||
def data_received(self, data):
|
||||
try:
|
||||
@@ -45,7 +48,7 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
msg = json.dumps(request_dict)
|
||||
self.transport.write(msg.encode())
|
||||
try:
|
||||
self.pending_request = asyncio.get_event_loop().create_task(self.response_queue.get())
|
||||
self.pending_request = self.loop.create_task(self.response_queue.get())
|
||||
return await self.pending_request
|
||||
finally:
|
||||
self.pending_request = None
|
||||
@@ -80,8 +83,11 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
if self.blob_manager.get_blob(blob.blob_hash, blob.length).get_is_verified():
|
||||
needed.append(blob.blob_hash)
|
||||
log.info("Sent reflector descriptor %s", sd_blob.blob_hash[:8])
|
||||
self.reflected_blobs.append(sd_blob.blob_hash)
|
||||
else:
|
||||
log.warning("Reflector failed to receive descriptor %s", sd_blob.blob_hash[:8])
|
||||
if needed:
|
||||
log.info("Reflector needs %i blobs for %s", len(needed), sd_blob.blob_hash[:8])
|
||||
return sent_sd, needed
|
||||
|
||||
async def send_blob(self, blob_hash: str):
|
||||
|
||||
@@ -37,7 +37,8 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
try:
|
||||
self.writer.write(data)
|
||||
except IOError as err:
|
||||
log.error("error downloading blob: %s", err)
|
||||
log.error("error receiving blob: %s", err)
|
||||
self.transport.close()
|
||||
return
|
||||
try:
|
||||
request = json.loads(data.decode())
|
||||
@@ -67,28 +68,30 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
self.send_response({"send_sd_blob": True})
|
||||
try:
|
||||
await asyncio.wait_for(self.sd_blob.finished_writing.wait(), 30, loop=self.loop)
|
||||
self.send_response({"received_sd_blob": True})
|
||||
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
|
||||
self.loop, self.blob_manager.blob_dir, self.sd_blob
|
||||
)
|
||||
self.incoming.clear()
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||||
self.send_response({"received_sd_blob": True})
|
||||
except asyncio.TimeoutError:
|
||||
self.send_response({"received_sd_blob": False})
|
||||
self.transport.close()
|
||||
finally:
|
||||
self.incoming.clear()
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
self.transport.close()
|
||||
return
|
||||
else:
|
||||
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
|
||||
self.loop, self.blob_manager.blob_dir, self.sd_blob
|
||||
)
|
||||
self.incoming.clear()
|
||||
if self.writer:
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
self.send_response({"send_sd_blob": False, 'needed': [
|
||||
blob.blob_hash for blob in self.descriptor.blobs[:-1]
|
||||
if not self.blob_manager.get_blob(blob.blob_hash).get_is_verified()
|
||||
]})
|
||||
return
|
||||
return
|
||||
elif self.descriptor:
|
||||
if 'blob_hash' not in request:
|
||||
@@ -105,7 +108,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
try:
|
||||
await asyncio.wait_for(blob.finished_writing.wait(), 30, loop=self.loop)
|
||||
self.send_response({"received_blob": True})
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||||
except asyncio.TimeoutError:
|
||||
self.send_response({"received_blob": False})
|
||||
self.incoming.clear()
|
||||
self.writer.close_handle()
|
||||
|
||||
+400
-169
@@ -4,21 +4,29 @@ import typing
|
||||
import binascii
|
||||
import logging
|
||||
import random
|
||||
from lbrynet.error import ResolveError, InvalidStreamDescriptorError, KeyFeeAboveMaxAllowed, InsufficientFundsError, \
|
||||
DownloadDataTimeout, DownloadSDTimeout
|
||||
from lbrynet.utils import generate_id
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.stream.downloader import StreamDownloader
|
||||
from lbrynet.stream.managed_stream import ManagedStream
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.extras.daemon.storage import StoredStreamClaim, lbc_to_dewies
|
||||
from lbrynet.schema.uri import parse_lbry_uri
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
from lbrynet.extras.daemon.storage import lbc_to_dewies
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.extras.daemon.analytics import AnalyticsManager
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage, StoredStreamClaim
|
||||
from lbrynet.extras.wallet import LbryWalletManager
|
||||
from lbrynet.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
filter_fields = [
|
||||
'rowid',
|
||||
'status',
|
||||
'file_name',
|
||||
'sd_hash',
|
||||
@@ -31,7 +39,9 @@ filter_fields = [
|
||||
'nout',
|
||||
'channel_claim_id',
|
||||
'channel_name',
|
||||
'full_status'
|
||||
'full_status', # TODO: remove
|
||||
'blobs_remaining',
|
||||
'blobs_in_stream'
|
||||
]
|
||||
|
||||
comparison_operators = {
|
||||
@@ -45,205 +55,231 @@ comparison_operators = {
|
||||
|
||||
|
||||
class StreamManager:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', wallet: 'LbryWalletManager',
|
||||
storage: 'SQLiteStorage', node: typing.Optional['Node'], peer_timeout: float,
|
||||
peer_connect_timeout: float, fixed_peers: typing.Optional[typing.List['KademliaPeer']] = None,
|
||||
reflector_servers: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager',
|
||||
wallet: 'LbryWalletManager', storage: 'SQLiteStorage', node: typing.Optional['Node'],
|
||||
analytics_manager: typing.Optional['AnalyticsManager'] = None):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
self.blob_manager = blob_manager
|
||||
self.wallet = wallet
|
||||
self.storage = storage
|
||||
self.node = node
|
||||
self.peer_timeout = peer_timeout
|
||||
self.peer_connect_timeout = peer_connect_timeout
|
||||
self.analytics_manager = analytics_manager
|
||||
self.streams: typing.Set[ManagedStream] = set()
|
||||
self.starting_streams: typing.Dict[str, asyncio.Future] = {}
|
||||
self.resume_downloading_task: asyncio.Task = None
|
||||
self.re_reflect_task: asyncio.Task = None
|
||||
self.update_stream_finished_futs: typing.List[asyncio.Future] = []
|
||||
self.fixed_peers = fixed_peers
|
||||
self.reflector_servers = reflector_servers
|
||||
self.running_reflector_uploads: typing.List[asyncio.Task] = []
|
||||
|
||||
async def _update_content_claim(self, stream: ManagedStream):
|
||||
claim_info = await self.storage.get_content_claim(stream.stream_hash)
|
||||
stream.set_claim(claim_info, smart_decode(claim_info['value']))
|
||||
|
||||
async def start_stream(self, stream: ManagedStream) -> bool:
|
||||
"""
|
||||
Resume or rebuild a partial or completed stream
|
||||
"""
|
||||
if not stream.running and not stream.output_file_exists:
|
||||
if stream.downloader:
|
||||
stream.downloader.stop()
|
||||
stream.downloader = None
|
||||
|
||||
# the directory is gone, can happen when the folder that contains a published file is deleted
|
||||
# reset the download directory to the default and update the file name
|
||||
if not os.path.isdir(stream.download_directory):
|
||||
stream.download_directory = self.config.download_dir
|
||||
|
||||
stream.downloader = self.make_downloader(
|
||||
stream.sd_hash, stream.download_directory, stream.descriptor.suggested_file_name
|
||||
)
|
||||
if stream.status != ManagedStream.STATUS_FINISHED:
|
||||
await self.storage.change_file_status(stream.stream_hash, 'running')
|
||||
stream.update_status('running')
|
||||
stream.start_download(self.node)
|
||||
try:
|
||||
await asyncio.wait_for(self.loop.create_task(stream.downloader.wrote_bytes_event.wait()),
|
||||
self.config.download_timeout)
|
||||
except asyncio.TimeoutError:
|
||||
await self.stop_stream(stream)
|
||||
if stream in self.streams:
|
||||
self.streams.remove(stream)
|
||||
return False
|
||||
file_name = os.path.basename(stream.downloader.output_path)
|
||||
output_dir = os.path.dirname(stream.downloader.output_path)
|
||||
await self.storage.change_file_download_dir_and_file_name(
|
||||
stream.stream_hash, output_dir, file_name
|
||||
)
|
||||
stream._file_name = file_name
|
||||
stream.download_directory = output_dir
|
||||
self.wait_for_stream_finished(stream)
|
||||
return True
|
||||
return True
|
||||
|
||||
async def stop_stream(self, stream: ManagedStream):
|
||||
stream.stop_download()
|
||||
if not stream.finished and stream.output_file_exists:
|
||||
try:
|
||||
os.remove(stream.full_path)
|
||||
except OSError as err:
|
||||
log.warning("Failed to delete partial download %s from downloads directory: %s", stream.full_path,
|
||||
str(err))
|
||||
if stream.running:
|
||||
stream.update_status(ManagedStream.STATUS_STOPPED)
|
||||
await self.storage.change_file_status(stream.stream_hash, ManagedStream.STATUS_STOPPED)
|
||||
|
||||
def make_downloader(self, sd_hash: str, download_directory: str, file_name: str):
|
||||
return StreamDownloader(
|
||||
self.loop, self.config, self.blob_manager, sd_hash, download_directory, file_name
|
||||
)
|
||||
|
||||
async def recover_streams(self, file_infos: typing.List[typing.Dict]):
|
||||
to_restore = []
|
||||
|
||||
async def recover_stream(sd_hash: str, stream_hash: str, stream_name: str,
|
||||
suggested_file_name: str, key: str) -> typing.Optional[StreamDescriptor]:
|
||||
sd_blob = self.blob_manager.get_blob(sd_hash)
|
||||
blobs = await self.storage.get_blobs_for_stream(stream_hash)
|
||||
descriptor = await StreamDescriptor.recover(
|
||||
self.blob_manager.blob_dir, sd_blob, stream_hash, stream_name, suggested_file_name, key, blobs
|
||||
)
|
||||
if not descriptor:
|
||||
return
|
||||
to_restore.append((descriptor, sd_blob))
|
||||
|
||||
await asyncio.gather(*[
|
||||
recover_stream(
|
||||
file_info['sd_hash'], file_info['stream_hash'], binascii.unhexlify(file_info['stream_name']).decode(),
|
||||
binascii.unhexlify(file_info['suggested_file_name']).decode(), file_info['key']
|
||||
) for file_info in file_infos
|
||||
])
|
||||
|
||||
if to_restore:
|
||||
await self.storage.recover_streams(to_restore, self.config.download_dir)
|
||||
log.info("Recovered %i/%i attempted streams", len(to_restore), len(file_infos))
|
||||
|
||||
async def add_stream(self, rowid: int, sd_hash: str, file_name: str, download_directory: str, status: str,
|
||||
claim: typing.Optional['StoredStreamClaim']):
|
||||
sd_blob = self.blob_manager.get_blob(sd_hash)
|
||||
if not sd_blob.get_is_verified():
|
||||
return
|
||||
try:
|
||||
descriptor = await self.blob_manager.get_stream_descriptor(sd_blob.blob_hash)
|
||||
except InvalidStreamDescriptorError as err:
|
||||
log.warning("Failed to start stream for sd %s - %s", sd_hash, str(err))
|
||||
return
|
||||
if status == ManagedStream.STATUS_RUNNING:
|
||||
downloader = self.make_downloader(descriptor.sd_hash, download_directory, file_name)
|
||||
else:
|
||||
downloader = None
|
||||
stream = ManagedStream(
|
||||
self.loop, self.blob_manager, rowid, descriptor, download_directory, file_name, downloader, status, claim
|
||||
)
|
||||
self.streams.add(stream)
|
||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||
|
||||
async def load_streams_from_database(self):
|
||||
infos = await self.storage.get_all_lbry_files()
|
||||
for file_info in infos:
|
||||
sd_blob = self.blob_manager.get_blob(file_info['sd_hash'])
|
||||
if sd_blob.get_is_verified():
|
||||
descriptor = await self.blob_manager.get_stream_descriptor(sd_blob.blob_hash)
|
||||
downloader = StreamDownloader(
|
||||
self.loop, self.blob_manager, descriptor.sd_hash, self.peer_timeout,
|
||||
self.peer_connect_timeout, binascii.unhexlify(file_info['download_directory']).decode(),
|
||||
binascii.unhexlify(file_info['file_name']).decode(), self.fixed_peers
|
||||
)
|
||||
stream = ManagedStream(
|
||||
self.loop, self.blob_manager, descriptor,
|
||||
binascii.unhexlify(file_info['download_directory']).decode(),
|
||||
binascii.unhexlify(file_info['file_name']).decode(),
|
||||
downloader, file_info['status'], file_info['claim']
|
||||
)
|
||||
self.streams.add(stream)
|
||||
to_recover = []
|
||||
for file_info in await self.storage.get_all_lbry_files():
|
||||
if not self.blob_manager.get_blob(file_info['sd_hash']).get_is_verified():
|
||||
to_recover.append(file_info)
|
||||
|
||||
if to_recover:
|
||||
log.info("Attempting to recover %i streams", len(to_recover))
|
||||
await self.recover_streams(to_recover)
|
||||
|
||||
to_start = []
|
||||
for file_info in await self.storage.get_all_lbry_files():
|
||||
if self.blob_manager.get_blob(file_info['sd_hash']).get_is_verified():
|
||||
to_start.append(file_info)
|
||||
log.info("Initializing %i files", len(to_start))
|
||||
|
||||
await asyncio.gather(*[
|
||||
self.add_stream(
|
||||
file_info['rowid'], file_info['sd_hash'], binascii.unhexlify(file_info['file_name']).decode(),
|
||||
binascii.unhexlify(file_info['download_directory']).decode(), file_info['status'],
|
||||
file_info['claim']
|
||||
) for file_info in to_start
|
||||
])
|
||||
log.info("Started stream manager with %i files", len(self.streams))
|
||||
|
||||
async def resume(self):
|
||||
if not self.node:
|
||||
log.warning("no DHT node given, cannot resume downloads")
|
||||
return
|
||||
await self.node.joined.wait()
|
||||
resumed = 0
|
||||
for stream in self.streams:
|
||||
if stream.status == ManagedStream.STATUS_RUNNING:
|
||||
resumed += 1
|
||||
stream.downloader.download(self.node)
|
||||
self.wait_for_stream_finished(stream)
|
||||
if resumed:
|
||||
log.info("resuming %i downloads", resumed)
|
||||
if self.node:
|
||||
await self.node.joined.wait()
|
||||
else:
|
||||
log.warning("no DHT node given, resuming downloads trusting that we can contact reflector")
|
||||
t = [
|
||||
(stream.start_download(self.node), self.wait_for_stream_finished(stream))
|
||||
for stream in self.streams if stream.status == ManagedStream.STATUS_RUNNING
|
||||
]
|
||||
if t:
|
||||
log.info("resuming %i downloads", len(t))
|
||||
|
||||
async def reflect_streams(self):
|
||||
streams = list(self.streams)
|
||||
batch = []
|
||||
while streams:
|
||||
stream = streams.pop()
|
||||
if not stream.fully_reflected.is_set():
|
||||
host, port = random.choice(self.reflector_servers)
|
||||
batch.append(stream.upload_to_reflector(host, port))
|
||||
if len(batch) >= 10:
|
||||
await asyncio.gather(*batch)
|
||||
while True:
|
||||
if self.config.reflect_streams and self.config.reflector_servers:
|
||||
sd_hashes = await self.storage.get_streams_to_re_reflect()
|
||||
streams = list(filter(lambda s: s.sd_hash in sd_hashes, self.streams))
|
||||
batch = []
|
||||
if batch:
|
||||
await asyncio.gather(*batch)
|
||||
while streams:
|
||||
stream = streams.pop()
|
||||
if not stream.fully_reflected.is_set():
|
||||
host, port = random.choice(self.config.reflector_servers)
|
||||
batch.append(stream.upload_to_reflector(host, port))
|
||||
if len(batch) >= self.config.concurrent_reflector_uploads:
|
||||
await asyncio.gather(*batch)
|
||||
batch = []
|
||||
if batch:
|
||||
await asyncio.gather(*batch)
|
||||
await asyncio.sleep(300, loop=self.loop)
|
||||
|
||||
async def start(self):
|
||||
await self.load_streams_from_database()
|
||||
self.resume_downloading_task = self.loop.create_task(self.resume())
|
||||
self.re_reflect_task = self.loop.create_task(self.reflect_streams())
|
||||
|
||||
async def stop(self):
|
||||
def stop(self):
|
||||
if self.resume_downloading_task and not self.resume_downloading_task.done():
|
||||
self.resume_downloading_task.cancel()
|
||||
if self.re_reflect_task and not self.re_reflect_task.done():
|
||||
self.re_reflect_task.cancel()
|
||||
while self.streams:
|
||||
stream = self.streams.pop()
|
||||
await stream.stop_download()
|
||||
stream.stop_download()
|
||||
while self.update_stream_finished_futs:
|
||||
self.update_stream_finished_futs.pop().cancel()
|
||||
while self.running_reflector_uploads:
|
||||
self.running_reflector_uploads.pop().cancel()
|
||||
|
||||
async def create_stream(self, file_path: str, key: typing.Optional[bytes] = None,
|
||||
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
|
||||
stream = await ManagedStream.create(self.loop, self.blob_manager, file_path, key, iv_generator)
|
||||
self.streams.add(stream)
|
||||
if self.reflector_servers:
|
||||
host, port = random.choice(self.reflector_servers)
|
||||
self.loop.create_task(stream.upload_to_reflector(host, port))
|
||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||
if self.config.reflect_streams and self.config.reflector_servers:
|
||||
host, port = random.choice(self.config.reflector_servers)
|
||||
task = self.loop.create_task(stream.upload_to_reflector(host, port))
|
||||
self.running_reflector_uploads.append(task)
|
||||
task.add_done_callback(
|
||||
lambda _: None
|
||||
if task not in self.running_reflector_uploads else self.running_reflector_uploads.remove(task)
|
||||
)
|
||||
return stream
|
||||
|
||||
async def delete_stream(self, stream: ManagedStream, delete_file: typing.Optional[bool] = False):
|
||||
await stream.stop_download()
|
||||
self.streams.remove(stream)
|
||||
await self.stop_stream(stream)
|
||||
if stream in self.streams:
|
||||
self.streams.remove(stream)
|
||||
blob_hashes = [stream.sd_hash] + [b.blob_hash for b in stream.descriptor.blobs[:-1]]
|
||||
await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False)
|
||||
await self.storage.delete_stream(stream.descriptor)
|
||||
if delete_file and stream.output_file_exists:
|
||||
os.remove(stream.full_path)
|
||||
|
||||
blob_hashes = [stream.sd_hash]
|
||||
for blob_info in stream.descriptor.blobs[:-1]:
|
||||
blob_hashes.append(blob_info.blob_hash)
|
||||
for blob_hash in blob_hashes:
|
||||
blob = self.blob_manager.get_blob(blob_hash)
|
||||
if blob.get_is_verified():
|
||||
await blob.delete()
|
||||
|
||||
if delete_file:
|
||||
path = os.path.join(stream.download_directory, stream.file_name)
|
||||
if os.path.isfile(path):
|
||||
os.remove(path)
|
||||
|
||||
def wait_for_stream_finished(self, stream: ManagedStream):
|
||||
async def _wait_for_stream_finished():
|
||||
if stream.downloader and stream.running:
|
||||
try:
|
||||
await stream.downloader.stream_finished_event.wait()
|
||||
stream.update_status(ManagedStream.STATUS_FINISHED)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
task = self.loop.create_task(_wait_for_stream_finished())
|
||||
self.update_stream_finished_futs.append(task)
|
||||
task.add_done_callback(
|
||||
lambda _: None if task not in self.update_stream_finished_futs else
|
||||
self.update_stream_finished_futs.remove(task)
|
||||
)
|
||||
|
||||
async def _download_stream_from_claim(self, node: 'Node', download_directory: str, claim_info: typing.Dict,
|
||||
file_name: typing.Optional[str] = None, data_rate: typing.Optional[int] = 0,
|
||||
sd_blob_timeout: typing.Optional[float] = 60
|
||||
) -> typing.Optional[ManagedStream]:
|
||||
|
||||
claim = ClaimDict.load_dict(claim_info['value'])
|
||||
downloader = StreamDownloader(self.loop, self.blob_manager, claim.source_hash.decode(), self.peer_timeout,
|
||||
self.peer_connect_timeout, download_directory, file_name, self.fixed_peers)
|
||||
try:
|
||||
downloader.download(node)
|
||||
await asyncio.wait_for(downloader.got_descriptor.wait(), sd_blob_timeout)
|
||||
log.info("got descriptor %s for %s", claim.source_hash.decode(), claim_info['name'])
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||||
log.info("stream timeout")
|
||||
await downloader.stop()
|
||||
log.info("stopped stream")
|
||||
return
|
||||
if not await self.blob_manager.storage.stream_exists(downloader.sd_hash):
|
||||
await self.blob_manager.storage.store_stream(downloader.sd_blob, downloader.descriptor)
|
||||
if not await self.blob_manager.storage.file_exists(downloader.sd_hash):
|
||||
await self.blob_manager.storage.save_downloaded_file(
|
||||
downloader.descriptor.stream_hash, os.path.basename(downloader.output_path), download_directory,
|
||||
data_rate
|
||||
)
|
||||
await self.blob_manager.storage.save_content_claim(
|
||||
downloader.descriptor.stream_hash, f"{claim_info['txid']}:{claim_info['nout']}"
|
||||
)
|
||||
|
||||
stored_claim = StoredStreamClaim(
|
||||
downloader.descriptor.stream_hash, f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
|
||||
claim_info['name'], claim_info['amount'], claim_info['height'], claim_info['hex'],
|
||||
claim.certificate_id, claim_info['address'], claim_info['claim_sequence'],
|
||||
claim_info.get('channel_name')
|
||||
)
|
||||
stream = ManagedStream(self.loop, self.blob_manager, downloader.descriptor, download_directory,
|
||||
os.path.basename(downloader.output_path), downloader, ManagedStream.STATUS_RUNNING,
|
||||
stored_claim)
|
||||
self.streams.add(stream)
|
||||
try:
|
||||
await stream.downloader.wrote_bytes_event.wait()
|
||||
self.wait_for_stream_finished(stream)
|
||||
return stream
|
||||
except asyncio.CancelledError:
|
||||
await downloader.stop()
|
||||
|
||||
async def download_stream_from_claim(self, node: 'Node', download_directory: str, claim_info: typing.Dict,
|
||||
file_name: typing.Optional[str] = None,
|
||||
sd_blob_timeout: typing.Optional[float] = 60,
|
||||
fee_amount: typing.Optional[float] = 0.0,
|
||||
fee_address: typing.Optional[str] = None) -> typing.Optional[ManagedStream]:
|
||||
log.info("get lbry://%s#%s", claim_info['name'], claim_info['claim_id'])
|
||||
claim = ClaimDict.load_dict(claim_info['value'])
|
||||
if fee_address and fee_amount:
|
||||
if fee_amount > await self.wallet.default_account.get_balance():
|
||||
raise Exception("not enough funds")
|
||||
sd_hash = claim.source_hash.decode()
|
||||
if sd_hash in self.starting_streams:
|
||||
return await self.starting_streams[sd_hash]
|
||||
already_started = tuple(filter(lambda s: s.descriptor.sd_hash == sd_hash, self.streams))
|
||||
if already_started:
|
||||
return already_started[0]
|
||||
|
||||
self.starting_streams[sd_hash] = asyncio.Future(loop=self.loop)
|
||||
stream_task = self.loop.create_task(
|
||||
self._download_stream_from_claim(node, download_directory, claim_info, file_name, 0, sd_blob_timeout)
|
||||
)
|
||||
try:
|
||||
await asyncio.wait_for(stream_task, sd_blob_timeout)
|
||||
stream = await stream_task
|
||||
self.starting_streams[sd_hash].set_result(stream)
|
||||
if fee_address and fee_amount:
|
||||
await self.wallet.send_amount_to_address(lbc_to_dewies(str(fee_amount)), fee_address.encode('latin1'))
|
||||
return stream
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||||
return
|
||||
finally:
|
||||
if sd_hash in self.starting_streams:
|
||||
del self.starting_streams[sd_hash]
|
||||
log.info("returned from get lbry://%s#%s", claim_info['name'], claim_info['claim_id'])
|
||||
def get_stream_by_stream_hash(self, stream_hash: str) -> typing.Optional[ManagedStream]:
|
||||
streams = tuple(filter(lambda stream: stream.stream_hash == stream_hash, self.streams))
|
||||
if streams:
|
||||
return streams[0]
|
||||
|
||||
def get_filtered_streams(self, sort_by: typing.Optional[str] = None, reverse: typing.Optional[bool] = False,
|
||||
comparison: typing.Optional[str] = None,
|
||||
@@ -260,6 +296,8 @@ class StreamManager:
|
||||
raise ValueError(f"'{sort_by}' is not a valid field to sort by")
|
||||
if comparison and comparison not in comparison_operators:
|
||||
raise ValueError(f"'{comparison}' is not a valid comparison")
|
||||
if 'full_status' in search_by:
|
||||
del search_by['full_status']
|
||||
for search in search_by.keys():
|
||||
if search not in filter_fields:
|
||||
raise ValueError(f"'{search}' is not a valid search operation")
|
||||
@@ -268,8 +306,6 @@ class StreamManager:
|
||||
streams = []
|
||||
for stream in self.streams:
|
||||
for search, val in search_by.items():
|
||||
if search == 'full_status':
|
||||
continue
|
||||
if comparison_operators[comparison](getattr(stream, search), val):
|
||||
streams.append(stream)
|
||||
break
|
||||
@@ -280,3 +316,198 @@ class StreamManager:
|
||||
if reverse:
|
||||
streams.reverse()
|
||||
return streams
|
||||
|
||||
def wait_for_stream_finished(self, stream: ManagedStream):
|
||||
async def _wait_for_stream_finished():
|
||||
if stream.downloader and stream.running:
|
||||
await stream.downloader.stream_finished_event.wait()
|
||||
stream.update_status(ManagedStream.STATUS_FINISHED)
|
||||
if self.analytics_manager:
|
||||
self.loop.create_task(self.analytics_manager.send_download_finished(
|
||||
stream.download_id, stream.claim_name, stream.sd_hash
|
||||
))
|
||||
|
||||
task = self.loop.create_task(_wait_for_stream_finished())
|
||||
self.update_stream_finished_futs.append(task)
|
||||
task.add_done_callback(
|
||||
lambda _: None if task not in self.update_stream_finished_futs else
|
||||
self.update_stream_finished_futs.remove(task)
|
||||
)
|
||||
|
||||
async def _store_stream(self, downloader: StreamDownloader) -> int:
|
||||
file_name = os.path.basename(downloader.output_path)
|
||||
download_directory = os.path.dirname(downloader.output_path)
|
||||
if not await self.storage.stream_exists(downloader.sd_hash):
|
||||
await self.storage.store_stream(downloader.sd_blob, downloader.descriptor)
|
||||
if not await self.storage.file_exists(downloader.sd_hash):
|
||||
return await self.storage.save_downloaded_file(
|
||||
downloader.descriptor.stream_hash, file_name, download_directory,
|
||||
0.0
|
||||
)
|
||||
else:
|
||||
return await self.storage.rowid_for_stream(downloader.descriptor.stream_hash)
|
||||
|
||||
async def _check_update_or_replace(self, outpoint: str, claim_id: str, claim: ClaimDict) -> typing.Tuple[
|
||||
typing.Optional[ManagedStream], typing.Optional[ManagedStream]]:
|
||||
existing = self.get_filtered_streams(outpoint=outpoint)
|
||||
if existing:
|
||||
await self.start_stream(existing[0])
|
||||
return existing[0], None
|
||||
existing = self.get_filtered_streams(sd_hash=claim.source_hash.decode())
|
||||
if existing and existing[0].claim_id != claim_id:
|
||||
raise ResolveError(f"stream for {existing[0].claim_id} collides with existing "
|
||||
f"download {claim_id}")
|
||||
if existing:
|
||||
log.info("claim contains a metadata only update to a stream we have")
|
||||
await self.storage.save_content_claim(
|
||||
existing[0].stream_hash, outpoint
|
||||
)
|
||||
await self._update_content_claim(existing[0])
|
||||
await self.start_stream(existing[0])
|
||||
return existing[0], None
|
||||
else:
|
||||
existing_for_claim_id = self.get_filtered_streams(claim_id=claim_id)
|
||||
if existing_for_claim_id:
|
||||
log.info("claim contains an update to a stream we have, downloading it")
|
||||
return None, existing_for_claim_id[0]
|
||||
return None, None
|
||||
|
||||
async def start_downloader(self, got_descriptor_time: asyncio.Future, downloader: StreamDownloader,
|
||||
download_id: str, outpoint: str, claim: ClaimDict, resolved: typing.Dict,
|
||||
file_name: typing.Optional[str] = None) -> ManagedStream:
|
||||
start_time = self.loop.time()
|
||||
downloader.download(self.node)
|
||||
await downloader.got_descriptor.wait()
|
||||
got_descriptor_time.set_result(self.loop.time() - start_time)
|
||||
rowid = await self._store_stream(downloader)
|
||||
await self.storage.save_content_claim(
|
||||
downloader.descriptor.stream_hash, outpoint
|
||||
)
|
||||
stream = ManagedStream(self.loop, self.blob_manager, rowid, downloader.descriptor, self.config.download_dir,
|
||||
file_name, downloader, ManagedStream.STATUS_RUNNING, download_id=download_id)
|
||||
stream.set_claim(resolved, claim)
|
||||
await stream.downloader.wrote_bytes_event.wait()
|
||||
self.streams.add(stream)
|
||||
return stream
|
||||
|
||||
async def _download_stream_from_uri(self, uri, timeout: float, exchange_rate_manager: 'ExchangeRateManager',
|
||||
file_name: typing.Optional[str] = None) -> ManagedStream:
|
||||
start_time = self.loop.time()
|
||||
parsed_uri = parse_lbry_uri(uri)
|
||||
if parsed_uri.is_channel:
|
||||
raise ResolveError("cannot download a channel claim, specify a /path")
|
||||
|
||||
# resolve the claim
|
||||
resolved = (await self.wallet.resolve(uri)).get(uri, {})
|
||||
resolved = resolved if 'value' in resolved else resolved.get('claim')
|
||||
|
||||
if not resolved:
|
||||
raise ResolveError(f"Failed to resolve stream at '{uri}'")
|
||||
if 'error' in resolved:
|
||||
raise ResolveError(f"error resolving stream: {resolved['error']}")
|
||||
|
||||
claim = ClaimDict.load_dict(resolved['value'])
|
||||
outpoint = f"{resolved['txid']}:{resolved['nout']}"
|
||||
resolved_time = self.loop.time() - start_time
|
||||
|
||||
# resume or update an existing stream, if the stream changed download it and delete the old one after
|
||||
updated_stream, to_replace = await self._check_update_or_replace(outpoint, resolved['claim_id'], claim)
|
||||
if updated_stream:
|
||||
return updated_stream
|
||||
|
||||
# check that the fee is payable
|
||||
fee_amount, fee_address = None, None
|
||||
if claim.has_fee:
|
||||
fee_amount = round(exchange_rate_manager.convert_currency(
|
||||
claim.source_fee.currency, "LBC", claim.source_fee.amount
|
||||
), 5)
|
||||
max_fee_amount = round(exchange_rate_manager.convert_currency(
|
||||
self.config.max_key_fee['currency'], "LBC", self.config.max_key_fee['amount']
|
||||
), 5)
|
||||
if fee_amount > max_fee_amount:
|
||||
msg = f"fee of {fee_amount} exceeds max configured to allow of {max_fee_amount}"
|
||||
log.warning(msg)
|
||||
raise KeyFeeAboveMaxAllowed(msg)
|
||||
balance = await self.wallet.default_account.get_balance()
|
||||
if lbc_to_dewies(str(fee_amount)) > balance:
|
||||
msg = f"fee of {fee_amount} exceeds max available balance"
|
||||
log.warning(msg)
|
||||
raise InsufficientFundsError(msg)
|
||||
fee_address = claim.source_fee.address.decode()
|
||||
|
||||
# download the stream
|
||||
download_id = binascii.hexlify(generate_id()).decode()
|
||||
downloader = StreamDownloader(self.loop, self.config, self.blob_manager, claim.source_hash.decode(),
|
||||
self.config.download_dir, file_name)
|
||||
|
||||
stream = None
|
||||
descriptor_time_fut = self.loop.create_future()
|
||||
start_download_time = self.loop.time()
|
||||
time_to_descriptor = None
|
||||
time_to_first_bytes = None
|
||||
error = None
|
||||
try:
|
||||
stream = await asyncio.wait_for(
|
||||
asyncio.ensure_future(
|
||||
self.start_downloader(descriptor_time_fut, downloader, download_id, outpoint, claim, resolved,
|
||||
file_name)
|
||||
), timeout
|
||||
)
|
||||
time_to_descriptor = await descriptor_time_fut
|
||||
time_to_first_bytes = self.loop.time() - start_download_time - time_to_descriptor
|
||||
self.wait_for_stream_finished(stream)
|
||||
if fee_address and fee_amount and not to_replace:
|
||||
stream.tx = await self.wallet.send_amount_to_address(
|
||||
lbc_to_dewies(str(fee_amount)), fee_address.encode('latin1'))
|
||||
elif to_replace: # delete old stream now that the replacement has started downloading
|
||||
await self.delete_stream(to_replace)
|
||||
except asyncio.TimeoutError:
|
||||
if descriptor_time_fut.done():
|
||||
time_to_descriptor = descriptor_time_fut.result()
|
||||
error = DownloadDataTimeout(downloader.sd_hash)
|
||||
self.blob_manager.delete_blob(downloader.sd_hash)
|
||||
await self.storage.delete_stream(downloader.descriptor)
|
||||
else:
|
||||
descriptor_time_fut.cancel()
|
||||
error = DownloadSDTimeout(downloader.sd_hash)
|
||||
if stream:
|
||||
await self.stop_stream(stream)
|
||||
else:
|
||||
downloader.stop()
|
||||
if error:
|
||||
log.warning(error)
|
||||
if self.analytics_manager:
|
||||
self.loop.create_task(
|
||||
self.analytics_manager.send_time_to_first_bytes(
|
||||
resolved_time, self.loop.time() - start_time, download_id, parse_lbry_uri(uri).name, outpoint,
|
||||
None if not stream else len(stream.downloader.blob_downloader.active_connections),
|
||||
None if not stream else len(stream.downloader.blob_downloader.scores),
|
||||
False if not downloader else downloader.added_fixed_peers,
|
||||
self.config.fixed_peer_delay if not downloader else downloader.fixed_peers_delay,
|
||||
claim.source_hash.decode(), time_to_descriptor,
|
||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
|
||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
|
||||
time_to_first_bytes, None if not error else error.__class__.__name__
|
||||
)
|
||||
)
|
||||
if error:
|
||||
raise error
|
||||
return stream
|
||||
|
||||
async def download_stream_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
|
||||
file_name: typing.Optional[str] = None,
|
||||
timeout: typing.Optional[float] = None) -> ManagedStream:
|
||||
timeout = timeout or self.config.download_timeout
|
||||
if uri in self.starting_streams:
|
||||
return await self.starting_streams[uri]
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
self.starting_streams[uri] = fut
|
||||
try:
|
||||
stream = await self._download_stream_from_uri(uri, timeout, exchange_rate_manager, file_name)
|
||||
fut.set_result(stream)
|
||||
except Exception as err:
|
||||
fut.set_exception(err)
|
||||
try:
|
||||
return await fut
|
||||
finally:
|
||||
del self.starting_streams[uri]
|
||||
+54
-7
@@ -7,8 +7,13 @@ import string
|
||||
import json
|
||||
import typing
|
||||
import asyncio
|
||||
import ssl
|
||||
import logging
|
||||
import ipaddress
|
||||
import pkg_resources
|
||||
import contextlib
|
||||
import certifi
|
||||
import aiohttp
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
@@ -68,13 +73,12 @@ def obfuscate(plain):
|
||||
return rot13(base64.b64encode(plain).decode())
|
||||
|
||||
|
||||
def check_connection(server="lbry.io", port=80, timeout=5):
|
||||
def check_connection(server="lbry.io", port=80, timeout=5) -> bool:
|
||||
"""Attempts to open a socket to server:port and returns True if successful."""
|
||||
log.debug('Checking connection to %s:%s', server, port)
|
||||
try:
|
||||
server = socket.gethostbyname(server)
|
||||
conn = socket.create_connection((server, port), timeout)
|
||||
conn.close()
|
||||
socket.create_connection((server, port), timeout).close()
|
||||
log.debug('Connection successful')
|
||||
return True
|
||||
except (socket.gaierror, socket.herror) as ex:
|
||||
@@ -83,19 +87,23 @@ def check_connection(server="lbry.io", port=80, timeout=5):
|
||||
try:
|
||||
server = "8.8.8.8"
|
||||
port = 53
|
||||
socket.create_connection((server, port), timeout)
|
||||
socket.create_connection((server, port), timeout).close()
|
||||
log.debug('Connection successful')
|
||||
return True
|
||||
except Exception as ex:
|
||||
except Exception:
|
||||
log.error("Failed to connect to %s:%s. Maybe the internet connection is not working",
|
||||
server, port)
|
||||
return False
|
||||
except Exception as ex:
|
||||
except Exception:
|
||||
log.error("Failed to connect to %s:%s. Maybe the internet connection is not working",
|
||||
server, port)
|
||||
server, port)
|
||||
return False
|
||||
|
||||
|
||||
async def async_check_connection(server="lbry.io", port=80, timeout=5) -> bool:
|
||||
return await asyncio.get_event_loop().run_in_executor(None, check_connection, server, port, timeout)
|
||||
|
||||
|
||||
def random_string(length=10, chars=string.ascii_lowercase):
|
||||
return ''.join([random.choice(chars) for _ in range(length)])
|
||||
|
||||
@@ -136,3 +144,42 @@ def cancel_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
|
||||
def drain_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
|
||||
while tasks:
|
||||
cancel_task(tasks.pop())
|
||||
|
||||
|
||||
async def resolve_host(url: str, port: int, proto: str) -> str:
|
||||
if proto not in ['udp', 'tcp']:
|
||||
raise Exception("invalid protocol")
|
||||
try:
|
||||
if ipaddress.ip_address(url):
|
||||
return url
|
||||
except ValueError:
|
||||
pass
|
||||
loop = asyncio.get_running_loop()
|
||||
return (await loop.getaddrinfo(
|
||||
url, port,
|
||||
proto=socket.IPPROTO_TCP if proto == 'tcp' else socket.IPPROTO_UDP,
|
||||
type=socket.SOCK_STREAM if proto == 'tcp' else socket.SOCK_DGRAM
|
||||
))[0][4][0]
|
||||
|
||||
|
||||
def get_ssl_context() -> ssl.SSLContext:
|
||||
return ssl.create_default_context(
|
||||
purpose=ssl.Purpose.CLIENT_AUTH, capath=certifi.where()
|
||||
)
|
||||
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def aiohttp_request(method, url, **kwargs) -> typing.AsyncContextManager[aiohttp.ClientResponse]:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.request(method, url, ssl=get_ssl_context(), **kwargs) as response:
|
||||
yield response
|
||||
|
||||
|
||||
async def get_external_ip() -> typing.Optional[str]: # used if upnp is disabled or non-functioning
|
||||
try:
|
||||
async with aiohttp_request("get", "https://api.lbry.io/ip") as resp:
|
||||
response = await resp.json()
|
||||
if response['success']:
|
||||
return response['data']['ip']
|
||||
except Exception as e:
|
||||
return
|
||||
@@ -0,0 +1,53 @@
|
||||
import sys
|
||||
import os
|
||||
import asyncio
|
||||
import socket
|
||||
import ipaddress
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
|
||||
import logging
|
||||
|
||||
log = logging.getLogger("lbrynet")
|
||||
log.addHandler(logging.StreamHandler())
|
||||
log.setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
async def main(blob_hash: str, url: str):
|
||||
conf = Config()
|
||||
loop = asyncio.get_running_loop()
|
||||
host_url, port = url.split(":")
|
||||
try:
|
||||
host = None
|
||||
if ipaddress.ip_address(host_url):
|
||||
host = host_url
|
||||
except ValueError:
|
||||
host = None
|
||||
if not host:
|
||||
host_info = await loop.getaddrinfo(
|
||||
host_url, 'https',
|
||||
proto=socket.IPPROTO_TCP,
|
||||
)
|
||||
host = host_info[0][4][0]
|
||||
|
||||
storage = SQLiteStorage(conf, os.path.join(conf.data_dir, "lbrynet.sqlite"))
|
||||
blob_manager = BlobFileManager(loop, os.path.join(conf.data_dir, "blobfiles"), storage)
|
||||
await storage.open()
|
||||
await blob_manager.setup()
|
||||
|
||||
blob = blob_manager.get_blob(blob_hash)
|
||||
success, keep = await request_blob(loop, blob, host, int(port), conf.peer_connect_timeout,
|
||||
conf.blob_download_timeout)
|
||||
print(f"{'downloaded' if success else 'failed to download'} {blob_hash} from {host}:{port}\n"
|
||||
f"keep connection: {keep}")
|
||||
if blob.get_is_verified():
|
||||
await blob_manager.delete_blobs([blob.blob_hash])
|
||||
print(f"deleted {blob_hash}")
|
||||
|
||||
|
||||
if __name__ == "__main__": # usage: python download_blob_from_peer.py <blob_hash> [host url:port]
|
||||
url = 'reflector.lbry.io:5567'
|
||||
if len(sys.argv) > 2:
|
||||
url = sys.argv[2]
|
||||
asyncio.run(main(sys.argv[1], url))
|
||||
+49
-19
@@ -19,7 +19,8 @@ except ImportError:
|
||||
|
||||
|
||||
AREA_RENAME = {
|
||||
'api': 'API'
|
||||
'api': 'API',
|
||||
'dht': 'DHT'
|
||||
}
|
||||
|
||||
|
||||
@@ -29,16 +30,20 @@ def get_github():
|
||||
with open(config_path, 'r') as config_file:
|
||||
config = json.load(config_file)
|
||||
return github3.login(token=config['token'])
|
||||
print('GitHub Credentials')
|
||||
username = input('username: ')
|
||||
password = getpass('password: ')
|
||||
gh = github3.authorize(
|
||||
username, password, ['repo'], 'lbry release tool',
|
||||
two_factor_callback=lambda: input('Enter 2FA: ')
|
||||
)
|
||||
with open(config_path, 'w') as config_file:
|
||||
json.dump({'token': gh.token}, config_file)
|
||||
return github3.login(token=gh.token)
|
||||
|
||||
token = os.environ.get("GH_TOKEN")
|
||||
if not token:
|
||||
print('GitHub Credentials')
|
||||
username = input('username: ')
|
||||
password = getpass('password: ')
|
||||
gh = github3.authorize(
|
||||
username, password, ['repo'], 'lbry release tool',
|
||||
two_factor_callback=lambda: input('Enter 2FA: ')
|
||||
)
|
||||
with open(config_path, 'w') as config_file:
|
||||
json.dump({'token': gh.token}, config_file)
|
||||
token = gh.token
|
||||
return github3.login(token=token)
|
||||
|
||||
|
||||
def get_labels(pr, prefix):
|
||||
@@ -54,6 +59,7 @@ def get_label(pr, prefix):
|
||||
|
||||
|
||||
BACKWARDS_INCOMPATIBLE = 'backwards-incompatible:'
|
||||
RELEASE_TEXT = "release-text:"
|
||||
|
||||
|
||||
def get_backwards_incompatible(desc: str):
|
||||
@@ -62,6 +68,12 @@ def get_backwards_incompatible(desc: str):
|
||||
yield line[len(BACKWARDS_INCOMPATIBLE):]
|
||||
|
||||
|
||||
def get_release_text(desc: str):
|
||||
for line in desc.splitlines():
|
||||
if line.startswith(RELEASE_TEXT):
|
||||
yield line[len(RELEASE_TEXT):]
|
||||
|
||||
|
||||
def get_previous_final(repo, current_release):
|
||||
assert current_release.rc is not None, "Need an rc to find the previous final release."
|
||||
previous = None
|
||||
@@ -139,15 +151,23 @@ def release(args):
|
||||
previous_release = repo.release_from_tag(current_version.tag)
|
||||
|
||||
incompats = []
|
||||
release_texts = []
|
||||
unlabeled = []
|
||||
areas = {}
|
||||
for pr in gh.search_issues(f"merged:>={previous_release._json_data['created_at']} repo:lbryio/lbry"):
|
||||
for area_name in get_labels(pr, 'area'):
|
||||
area = areas.setdefault(area_name, [])
|
||||
type_label = get_label(pr, "type")
|
||||
for incompat in get_backwards_incompatible(pr.body):
|
||||
incompats.append(f' * [{area_name}] {incompat.strip()} ({pr.html_url})')
|
||||
if not (args.action == '*-rc' and type_label == 'fixup'):
|
||||
area.append(f' * [{type_label}] {pr.title} ({pr.html_url}) by {pr.user["login"]}')
|
||||
area_labels = list(get_labels(pr, 'area'))
|
||||
type_label = get_label(pr, 'type')
|
||||
if area_labels and type_label:
|
||||
for area_name in area_labels:
|
||||
for incompat in get_backwards_incompatible(pr.body):
|
||||
incompats.append(f' * [{area_name}] {incompat.strip()} ({pr.html_url})')
|
||||
for release_text in get_release_text(pr.body):
|
||||
release_texts.append(f'{release_text.strip()} ({pr.html_url})')
|
||||
if not (args.action == '*-rc' and type_label == 'fixup'):
|
||||
area = areas.setdefault(area_name, [])
|
||||
area.append(f' * [{type_label}] {pr.title} ({pr.html_url}) by {pr.user["login"]}')
|
||||
else:
|
||||
unlabeled.append(f' * {pr.title} ({pr.html_url}) by {pr.user["login"]}')
|
||||
|
||||
area_names = list(areas.keys())
|
||||
area_names.sort()
|
||||
@@ -156,6 +176,11 @@ def release(args):
|
||||
w = lambda s: body.write(s+'\n')
|
||||
|
||||
w(f'## [{new_version}] - {date.today().isoformat()}')
|
||||
if release_texts:
|
||||
w('')
|
||||
for release_text in release_texts:
|
||||
w(release_text)
|
||||
w('')
|
||||
if incompats:
|
||||
w('')
|
||||
w(f'### Backwards Incompatible Changes')
|
||||
@@ -163,7 +188,7 @@ def release(args):
|
||||
w(incompat)
|
||||
for area in area_names:
|
||||
prs = areas[area]
|
||||
area = AREA_RENAME.get(area, area.capitalize())
|
||||
area = AREA_RENAME.get(area.lower(), area.capitalize())
|
||||
w('')
|
||||
w(f'### {area}')
|
||||
for pr in prs:
|
||||
@@ -171,6 +196,11 @@ def release(args):
|
||||
|
||||
print(body.getvalue())
|
||||
|
||||
if unlabeled:
|
||||
print('The following PRs were skipped and not included in changelog:')
|
||||
for skipped in unlabeled:
|
||||
print(skipped)
|
||||
|
||||
if not args.dry_run:
|
||||
|
||||
commit = version_file.update(
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import os
|
||||
import binascii
|
||||
import sqlite3
|
||||
from lbrynet.conf import Config
|
||||
|
||||
|
||||
def main():
|
||||
conf = Config()
|
||||
db = sqlite3.connect(os.path.join(conf.data_dir, 'lbrynet.sqlite'))
|
||||
cur = db.cursor()
|
||||
files = cur.execute("select stream_hash, file_name, download_directory from file").fetchall()
|
||||
update = {}
|
||||
for stream_hash, file_name, download_directory in files:
|
||||
try:
|
||||
binascii.unhexlify(file_name)
|
||||
except binascii.Error:
|
||||
try:
|
||||
binascii.unhexlify(download_directory)
|
||||
except binascii.Error:
|
||||
update[stream_hash] = (
|
||||
binascii.hexlify(file_name.encode()).decode(), binascii.hexlify(download_directory.encode()).decode()
|
||||
)
|
||||
if update:
|
||||
print(f"repair {len(update)} streams")
|
||||
for stream_hash, (file_name, download_directory) in update.items():
|
||||
cur.execute('update file set file_name=?, download_directory=? where stream_hash=?',
|
||||
(file_name, download_directory, stream_hash))
|
||||
db.commit()
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -18,12 +18,15 @@ def get_build():
|
||||
try:
|
||||
tag = subprocess.check_output(['git', 'describe', '--exact-match', '--all']).strip()
|
||||
if re.match('tags\/v\d+\.\d+\.\d+rc\d+$', tag.decode()):
|
||||
print('Build: rc')
|
||||
return 'rc'
|
||||
elif re.match('tags\/v\d+\.\d+\.\d+$', tag.decode()):
|
||||
print('Build: release')
|
||||
return 'release'
|
||||
print('Build: qa')
|
||||
return 'qa'
|
||||
except subprocess.CalledProcessError:
|
||||
# if the build doesn't have a tag
|
||||
print("Couldn't determine build type, defaulting to qa.")
|
||||
return 'qa'
|
||||
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import time
|
||||
|
||||
from aiohttp import ClientConnectorError
|
||||
from lbrynet import __version__
|
||||
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.schema.uri import parse_lbry_uri
|
||||
from lbrynet.extras.daemon.client import daemon_rpc
|
||||
@@ -45,11 +46,11 @@ async def report_to_slack(output, webhook):
|
||||
pass
|
||||
|
||||
|
||||
def confidence(times, z):
|
||||
def confidence(times, z, plus_err=True):
|
||||
mean = sum(times) / len(times)
|
||||
standard_dev = (sum(((t - sum(times) / len(times)) ** 2.0 for t in times)) / len(times)) ** 0.5
|
||||
err = (z * standard_dev) / (len(times) ** 0.5)
|
||||
return f"{round(mean, 3) + round(err, 3)}s"
|
||||
return f"{round((mean + err) if plus_err else (mean - err), 3)}"
|
||||
|
||||
|
||||
def variance(times):
|
||||
@@ -57,31 +58,29 @@ def variance(times):
|
||||
return round(sum(((i - mean) ** 2.0 for i in times)) / (len(times) - 1), 3)
|
||||
|
||||
|
||||
async def wait_for_done(api, uri):
|
||||
async def wait_for_done(conf, uri):
|
||||
name = uri.split("#")[0]
|
||||
last_complete = 0
|
||||
hang_count = 0
|
||||
while True:
|
||||
files = await api.file_list(claim_name=name)
|
||||
files = await daemon_rpc(conf, "file_list", claim_name=name)
|
||||
file = files[0]
|
||||
if file['status'] in ['finished', 'stopped']:
|
||||
return True, f"{file['blobs_completed']}/{file['blobs_in_stream']}", int(file['blobs_completed'])
|
||||
return True, file['blobs_completed'], file['blobs_in_stream']
|
||||
if last_complete < int(file['blobs_completed']):
|
||||
print(f"{file['blobs_completed']}/{file['blobs_in_stream']}...")
|
||||
hang_count = 0
|
||||
last_complete = int(file['blobs_completed'])
|
||||
else:
|
||||
hang_count += 1
|
||||
await asyncio.sleep(1.0)
|
||||
if hang_count > 30:
|
||||
return False, f"{file['blobs_completed']}/{file['blobs_in_stream']}", int(file['blobs_completed'])
|
||||
if hang_count > 10:
|
||||
return False, file['blobs_completed'], file['blobs_in_stream']
|
||||
|
||||
|
||||
async def main(start_daemon=True, uris=None):
|
||||
async def main(uris=None, allow_fees=False):
|
||||
if not uris:
|
||||
uris = await get_frontpage_uris()
|
||||
conf = Config()
|
||||
daemon = None
|
||||
try:
|
||||
await daemon_rpc(conf, 'status')
|
||||
except (ClientConnectorError, ConnectionError):
|
||||
@@ -92,15 +91,20 @@ async def main(start_daemon=True, uris=None):
|
||||
|
||||
resolvable = []
|
||||
for name in uris:
|
||||
resolved = await daemon_rpc(conf, 'resolve', uri=name)
|
||||
resolved = await daemon_rpc(conf, 'resolve', name)
|
||||
if 'error' not in resolved.get(name, {}):
|
||||
resolvable.append(name)
|
||||
|
||||
print(f"{len(resolvable)}/{len(uris)} are resolvable")
|
||||
if ("fee" not in resolved[name]['claim']['value']['stream']['metadata']) or allow_fees:
|
||||
resolvable.append(name)
|
||||
else:
|
||||
print(f"{name} has a fee, skipping it")
|
||||
else:
|
||||
print(f"failed to resolve {name}: {resolved[name]['error']}")
|
||||
print(f"attempting to download {len(resolvable)}/{len(uris)} frontpage streams")
|
||||
|
||||
first_byte_times = []
|
||||
downloaded_times = []
|
||||
failures = []
|
||||
download_speeds = []
|
||||
download_successes = []
|
||||
failed_to_start = []
|
||||
download_failures = []
|
||||
|
||||
for uri in resolvable:
|
||||
@@ -113,34 +117,44 @@ async def main(start_daemon=True, uris=None):
|
||||
first_byte = time.time()
|
||||
first_byte_times.append(first_byte - start)
|
||||
print(f"{i + 1}/{len(resolvable)} - {first_byte - start} {uri}")
|
||||
# downloaded, msg, blobs_in_stream = await wait_for_done(api, uri)
|
||||
# if downloaded:
|
||||
# downloaded_times.append((time.time()-start) / downloaded)
|
||||
# print(f"{i + 1}/{len(uris)} - downloaded @ {(time.time()-start) / blobs_in_stream}, {msg} {uri}")
|
||||
# else:
|
||||
# print(f"failed to downlload {uri}, got {msg}")
|
||||
# download_failures.append(uri)
|
||||
downloaded, amount_downloaded, blobs_in_stream = await wait_for_done(conf, uri)
|
||||
if downloaded:
|
||||
download_successes.append(uri)
|
||||
else:
|
||||
download_failures.append(uri)
|
||||
mbs = round((blobs_in_stream * (MAX_BLOB_SIZE - 1)) / (time.time() - start) / 1000000, 2)
|
||||
download_speeds.append(mbs)
|
||||
print(f"downloaded {amount_downloaded}/{blobs_in_stream} blobs for {uri} at "
|
||||
f"{mbs}mb/s")
|
||||
except:
|
||||
print(f"{i + 1}/{len(uris)} - timeout in {time.time() - start} {uri}")
|
||||
failures.append(uri)
|
||||
await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=parse_lbry_uri(uri).name)
|
||||
print(f"{i + 1}/{len(uris)} - failed to start {uri}")
|
||||
failed_to_start.append(uri)
|
||||
return
|
||||
# await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=parse_lbry_uri(uri).name)
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
print("**********************************************")
|
||||
result = f"Tried to start downloading {len(resolvable)} streams from the front page\n" \
|
||||
f"95% confidence time-to-first-byte: {confidence(first_byte_times, 1.984)}\n" \
|
||||
f"99% confidence time-to-first-byte: {confidence(first_byte_times, 2.626)}\n" \
|
||||
result = f"Started {len(first_byte_times)} of {len(resolvable)} attempted front page streams\n" \
|
||||
f"Worst first byte time: {round(max(first_byte_times), 2)}\n" \
|
||||
f"Best first byte time: {round(min(first_byte_times), 2)}\n" \
|
||||
f"95% confidence time-to-first-byte: {confidence(first_byte_times, 1.984)}s\n" \
|
||||
f"99% confidence time-to-first-byte: {confidence(first_byte_times, 2.626)}s\n" \
|
||||
f"Variance: {variance(first_byte_times)}\n" \
|
||||
f"Started {len(first_byte_times)}/{len(resolvable)} streams"
|
||||
if failures:
|
||||
nt = '\n\t'
|
||||
result += f"\nFailures:\n\t{nt.join([f for f in failures])}"
|
||||
f"Downloaded {len(download_successes)}/{len(resolvable)}\n" \
|
||||
f"Best stream download speed: {round(max(download_speeds), 2)}mb/s\n" \
|
||||
f"Worst stream download speed: {round(min(download_speeds), 2)}mb/s\n" \
|
||||
f"95% confidence download speed: {confidence(download_speeds, 1.984, False)}mb/s\n" \
|
||||
f"99% confidence download speed: {confidence(download_speeds, 2.626, False)}mb/s\n"
|
||||
|
||||
if failed_to_start:
|
||||
result += "\nFailed to start:" + "\n".join([f for f in failed_to_start])
|
||||
if download_failures:
|
||||
result += "\nFailed to finish:" + "\n".join([f for f in download_failures])
|
||||
print(result)
|
||||
if daemon:
|
||||
await daemon.shutdown()
|
||||
# webhook = os.environ.get('TTFB_SLACK_TOKEN', None)
|
||||
# if webhook:
|
||||
# await report_to_slack(result, webhook)
|
||||
|
||||
webhook = os.environ.get('TTFB_SLACK_TOKEN', None)
|
||||
if webhook:
|
||||
await report_to_slack(result, webhook)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
@@ -148,5 +162,6 @@ if __name__ == "__main__":
|
||||
parser.add_argument("--data_dir")
|
||||
parser.add_argument("--wallet_dir")
|
||||
parser.add_argument("--download_directory")
|
||||
parser.add_argument("--allow_fees", action='store_true')
|
||||
args = parser.parse_args()
|
||||
asyncio.run(main())
|
||||
asyncio.run(main(allow_fees=args.allow_fees))
|
||||
@@ -5,9 +5,8 @@ rm -rf /tmp/.wine-*
|
||||
apt-get -qq update
|
||||
apt-get -qq install -y git
|
||||
|
||||
git clone https://github.com/lbryio/torba.git --depth 1
|
||||
|
||||
pip install setuptools_scm
|
||||
git clone https://github.com/lbryio/torba.git --depth 1
|
||||
cd torba && pip install -e . && cd ..
|
||||
|
||||
cd lbry
|
||||
|
||||
@@ -23,20 +23,21 @@ setup(
|
||||
'console_scripts': 'lbrynet=lbrynet.extras.cli:main'
|
||||
},
|
||||
install_requires=[
|
||||
'aiohttp',
|
||||
'aiohttp==3.5.4',
|
||||
'aioupnp',
|
||||
'appdirs',
|
||||
'distro',
|
||||
'base58==1.0.0',
|
||||
'jsonrpc',
|
||||
'cryptography',
|
||||
'protobuf==3.6.1',
|
||||
'msgpack',
|
||||
'jsonschema',
|
||||
'ecdsa',
|
||||
'torba',
|
||||
'pyyaml',
|
||||
'docopt',
|
||||
'appdirs==1.4.3',
|
||||
'certifi>=2018.11.29',
|
||||
'colorama==0.3.7',
|
||||
'distro==1.4.0',
|
||||
'base58==1.0.0',
|
||||
'cffi==1.12.1',
|
||||
'cryptography==2.5',
|
||||
'protobuf==3.6.1',
|
||||
'msgpack==0.6.1',
|
||||
'jsonschema==2.6.0',
|
||||
'ecdsa==0.13',
|
||||
'torba',
|
||||
'pyyaml==3.13',
|
||||
'docopt==0.6.2',
|
||||
],
|
||||
)
|
||||
Whitespace-only changes.
@@ -0,0 +1,56 @@
|
||||
from integration.testcase import CommandTestCase
|
||||
|
||||
|
||||
class AccountManagement(CommandTestCase):
|
||||
|
||||
async def test_account_list_set_create_remove_add(self):
|
||||
# check initial account
|
||||
response = await self.daemon.jsonrpc_account_list()
|
||||
self.assertEqual(len(response['lbc_regtest']), 1)
|
||||
|
||||
# change account name and gap
|
||||
account_id = response['lbc_regtest'][0]['id']
|
||||
self.daemon.jsonrpc_account_set(
|
||||
account_id=account_id, new_name='test account',
|
||||
receiving_gap=95, receiving_max_uses=96,
|
||||
change_gap=97, change_max_uses=98
|
||||
)
|
||||
response = (await self.daemon.jsonrpc_account_list())['lbc_regtest'][0]
|
||||
self.assertEqual(response['name'], 'test account')
|
||||
self.assertEqual(
|
||||
response['address_generator']['receiving'],
|
||||
{'gap': 95, 'maximum_uses_per_address': 96}
|
||||
)
|
||||
self.assertEqual(
|
||||
response['address_generator']['change'],
|
||||
{'gap': 97, 'maximum_uses_per_address': 98}
|
||||
)
|
||||
|
||||
# create another account
|
||||
await self.daemon.jsonrpc_account_create('second account')
|
||||
response = await self.daemon.jsonrpc_account_list()
|
||||
self.assertEqual(len(response['lbc_regtest']), 2)
|
||||
self.assertEqual(response['lbc_regtest'][1]['name'], 'second account')
|
||||
account_id2 = response['lbc_regtest'][1]['id']
|
||||
|
||||
# make new account the default
|
||||
self.daemon.jsonrpc_account_set(account_id=account_id2, default=True)
|
||||
response = await self.daemon.jsonrpc_account_list(show_seed=True)
|
||||
self.assertEqual(response['lbc_regtest'][0]['name'], 'second account')
|
||||
|
||||
account_seed = response['lbc_regtest'][1]['seed']
|
||||
|
||||
# remove account
|
||||
self.daemon.jsonrpc_account_remove(response['lbc_regtest'][1]['id'])
|
||||
response = await self.daemon.jsonrpc_account_list()
|
||||
self.assertEqual(len(response['lbc_regtest']), 1)
|
||||
|
||||
# add account
|
||||
await self.daemon.jsonrpc_account_add('recreated account', seed=account_seed)
|
||||
response = await self.daemon.jsonrpc_account_list()
|
||||
self.assertEqual(len(response['lbc_regtest']), 2)
|
||||
self.assertEqual(response['lbc_regtest'][1]['name'], 'recreated account')
|
||||
|
||||
# list specific account
|
||||
response = await self.daemon.jsonrpc_account_list(account_id, include_claims=True)
|
||||
self.assertEqual(response['name'], 'recreated account')
|
||||
@@ -0,0 +1,224 @@
|
||||
import tempfile
|
||||
from integration.testcase import CommandTestCase
|
||||
|
||||
|
||||
class EpicAdventuresOfChris45(CommandTestCase):
|
||||
|
||||
async def test_no_this_is_not_a_test_its_an_adventure(self):
|
||||
# Chris45 is an avid user of LBRY and this is his story. It's fact and fiction
|
||||
# and everything in between; it's also the setting of some record setting
|
||||
# integration tests.
|
||||
|
||||
# Chris45 starts everyday by checking his balance.
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
self.assertEqual(result, '10.0')
|
||||
# "10 LBC, yippy! I can do a lot with that.", he thinks to himself,
|
||||
# enthusiastically. But he is hungry so he goes into the kitchen
|
||||
# to make himself a spamdwich.
|
||||
|
||||
# While making the spamdwich he wonders... has anyone on LBRY
|
||||
# registered the @spam channel yet? "I should do that!" he
|
||||
# exclaims and goes back to his computer to do just that!
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@spam', "1.0"))
|
||||
self.assertTrue(channel['success'])
|
||||
await self.confirm_tx(channel['tx']['txid'])
|
||||
|
||||
# Do we have it locally?
|
||||
channels = await self.out(self.daemon.jsonrpc_channel_list())
|
||||
self.assertEqual(len(channels), 1)
|
||||
self.assertEqual(channels[0]['name'], '@spam')
|
||||
|
||||
# As the new channel claim travels through the intertubes and makes its
|
||||
# way into the mempool and then a block and then into the claimtrie,
|
||||
# Chris doesn't sit idly by: he checks his balance!
|
||||
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
self.assertEqual(result, '8.989893')
|
||||
|
||||
# He waits for 6 more blocks (confirmations) to make sure the balance has been settled.
|
||||
await self.generate(6)
|
||||
result = await self.daemon.jsonrpc_account_balance(confirmations=6)
|
||||
self.assertEqual(result, '8.989893')
|
||||
|
||||
# And is the channel resolvable and empty?
|
||||
response = await self.out(self.daemon.jsonrpc_resolve('lbry://@spam'))
|
||||
self.assertIn('lbry://@spam', response)
|
||||
self.assertIn('certificate', response['lbry://@spam'])
|
||||
|
||||
# "What goes well with spam?" ponders Chris...
|
||||
# "A hovercraft with eels!" he exclaims.
|
||||
# "That's what goes great with spam!" he further confirms.
|
||||
|
||||
# And so, many hours later, Chris is finished writing his epic story
|
||||
# about eels driving a hovercraft across the wetlands while eating spam
|
||||
# and decides it's time to publish it to the @spam channel.
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'blah blah blah...')
|
||||
file.write(b'[insert long story about eels driving hovercraft]')
|
||||
file.write(b'yada yada yada!')
|
||||
file.write(b'the end')
|
||||
file.flush()
|
||||
claim1 = await self.out(self.daemon.jsonrpc_publish(
|
||||
'hovercraft', '1.0', file_path=file.name, channel_id=channel['claim_id']
|
||||
))
|
||||
self.assertTrue(claim1['success'])
|
||||
await self.confirm_tx(claim1['tx']['txid'])
|
||||
|
||||
# He quickly checks the unconfirmed balance to make sure everything looks
|
||||
# correct.
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
self.assertEqual(result, '7.969786')
|
||||
|
||||
# Also checks that his new story can be found on the blockchain before
|
||||
# giving the link to all his friends.
|
||||
response = await self.out(self.daemon.jsonrpc_resolve('lbry://@spam/hovercraft'))
|
||||
self.assertIn('lbry://@spam/hovercraft', response)
|
||||
self.assertIn('claim', response['lbry://@spam/hovercraft'])
|
||||
|
||||
# He goes to tell everyone about it and in the meantime 5 blocks are confirmed.
|
||||
await self.generate(5)
|
||||
# When he comes back he verifies the confirmed balance.
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
self.assertEqual(result, '7.969786')
|
||||
|
||||
# As people start reading his story they discover some typos and notify
|
||||
# Chris who explains in despair "Oh! Noooooos!" but then remembers
|
||||
# "No big deal! I can update my claim." And so he updates his claim.
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'blah blah blah...')
|
||||
file.write(b'[typo fixing sounds being made]')
|
||||
file.write(b'yada yada yada!')
|
||||
file.flush()
|
||||
claim2 = await self.out(self.daemon.jsonrpc_publish(
|
||||
'hovercraft', '1.0', file_path=file.name, channel_name='@spam'
|
||||
))
|
||||
self.assertTrue(claim2['success'])
|
||||
self.assertEqual(claim2['claim_id'], claim1['claim_id'])
|
||||
await self.confirm_tx(claim2['tx']['txid'])
|
||||
|
||||
# After some soul searching Chris decides that his story needs more
|
||||
# heart and a better ending. He takes down the story and begins the rewrite.
|
||||
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(claim1['claim_id'], blocking=False))
|
||||
self.assertTrue(abandon['success'])
|
||||
await self.confirm_tx(abandon['tx']['txid'])
|
||||
|
||||
# And now checks that the claim doesn't resolve anymore.
|
||||
response = await self.out(self.daemon.jsonrpc_resolve('lbry://@spam/hovercraft'))
|
||||
self.assertNotIn('claim', response['lbry://@spam/hovercraft'])
|
||||
|
||||
# After abandoning he just waits for his LBCs to be returned to his account
|
||||
await self.generate(5)
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
self.assertEqual(result, '8.9693585')
|
||||
|
||||
# Amidst all this Chris receives a call from his friend Ramsey
|
||||
# who says that it is of utmost urgency that Chris transfer him
|
||||
# 1 LBC to which Chris readily obliges
|
||||
ramsey_account_id = (await self.daemon.jsonrpc_account_create("Ramsey"))['id']
|
||||
ramsey_account = self.daemon.get_account_or_error(ramsey_account_id)
|
||||
ramsey_address = await self.daemon.jsonrpc_address_unused(ramsey_account_id)
|
||||
result = await self.out(self.daemon.jsonrpc_wallet_send('1.0', ramsey_address))
|
||||
self.assertIn("txid", result)
|
||||
await self.confirm_tx(result['txid'])
|
||||
|
||||
# Chris then eagerly waits for 6 confirmations to check his balance and then calls Ramsey to verify whether
|
||||
# he received it or not
|
||||
await self.generate(5)
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
# Chris' balance was correct
|
||||
self.assertEqual(result, '7.9692345')
|
||||
|
||||
# Ramsey too assured him that he had received the 1 LBC and thanks him
|
||||
result = await self.daemon.jsonrpc_account_balance(ramsey_account_id)
|
||||
self.assertEqual(result, '1.0')
|
||||
|
||||
# After Chris is done with all the "helping other people" stuff he decides that it's time to
|
||||
# write a new story and publish it to lbry. All he needed was a fresh start and he came up with:
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'Amazingly Original First Line')
|
||||
file.write(b'Super plot for the grand novel')
|
||||
file.write(b'Totally un-cliched ending')
|
||||
file.write(b'**Audience Gasps**')
|
||||
file.flush()
|
||||
claim3 = await self.out(self.daemon.jsonrpc_publish(
|
||||
'fresh-start', '1.0', file_path=file.name, channel_name='@spam'
|
||||
))
|
||||
self.assertTrue(claim3['success'])
|
||||
await self.confirm_tx(claim3['tx']['txid'])
|
||||
|
||||
await self.generate(5)
|
||||
|
||||
# He gives the link of his story to all his friends and hopes that this is the much needed break for him
|
||||
uri = 'lbry://@spam/fresh-start'
|
||||
|
||||
# And voila, and bravo and encore! His Best Friend Ramsey read the story and immediately knew this was a hit
|
||||
# Now to keep this claim winning on the lbry blockchain he immediately supports the claim
|
||||
tx = await self.out(self.daemon.jsonrpc_claim_new_support(
|
||||
'fresh-start', claim3['claim_id'], '0.2', account_id=ramsey_account_id
|
||||
))
|
||||
await self.confirm_tx(tx['txid'])
|
||||
|
||||
# And check if his support showed up
|
||||
resolve_result = await self.out(self.daemon.jsonrpc_resolve(uri))
|
||||
# It obviously did! Because, blockchain baby \O/
|
||||
self.assertEqual(resolve_result[uri]['claim']['amount'], '1.0')
|
||||
self.assertEqual(resolve_result[uri]['claim']['effective_amount'], '1.2')
|
||||
self.assertEqual(resolve_result[uri]['claim']['supports'][0]['amount'], '0.2')
|
||||
self.assertEqual(resolve_result[uri]['claim']['supports'][0]['txid'], tx['txid'])
|
||||
await self.generate(5)
|
||||
|
||||
# Now he also wanted to support the original creator of the Award Winning Novel
|
||||
# So he quickly decides to send a tip to him
|
||||
tx = await self.out(
|
||||
self.daemon.jsonrpc_claim_tip(claim3['claim_id'], '0.3', account_id=ramsey_account_id))
|
||||
await self.confirm_tx(tx['txid'])
|
||||
|
||||
# And again checks if it went to the just right place
|
||||
resolve_result = await self.out(self.daemon.jsonrpc_resolve(uri))
|
||||
# Which it obviously did. Because....?????
|
||||
self.assertEqual(resolve_result[uri]['claim']['supports'][1]['amount'], '0.3')
|
||||
self.assertEqual(resolve_result[uri]['claim']['supports'][1]['txid'], tx['txid'])
|
||||
await self.generate(5)
|
||||
|
||||
# Seeing the ravishing success of his novel Chris adds support to his claim too
|
||||
tx = await self.out(self.daemon.jsonrpc_claim_new_support('fresh-start', claim3['claim_id'], '0.4'))
|
||||
await self.confirm_tx(tx['txid'])
|
||||
|
||||
# And check if his support showed up
|
||||
resolve_result = await self.out(self.daemon.jsonrpc_resolve(uri))
|
||||
# It did!
|
||||
self.assertEqual(resolve_result[uri]['claim']['supports'][2]['amount'], '0.4')
|
||||
self.assertEqual(resolve_result[uri]['claim']['supports'][2]['txid'], tx['txid'])
|
||||
await self.generate(5)
|
||||
|
||||
# Now Ramsey who is a singer by profession, is preparing for his new "gig". He has everything in place for that
|
||||
# the instruments, the theatre, the ads, everything, EXCEPT lyrics!! He panicked.. But then he remembered
|
||||
# something, so he un-panicked. He quickly calls up his best bud Chris and requests him to write hit lyrics for
|
||||
# his song, seeing as his novel had smashed all the records, he was the perfect candidate!
|
||||
# .......
|
||||
# Chris agrees.. 17 hours 43 minutes and 14 seconds later, he makes his publish
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'The Whale amd The Bookmark')
|
||||
file.write(b'I know right? Totally a hit song')
|
||||
file.write(b'That\'s what goes around for songs these days anyways')
|
||||
file.flush()
|
||||
claim4 = await self.out(self.daemon.jsonrpc_publish(
|
||||
'hit-song', '1.0', file_path=file.name, channel_id=channel['claim_id']
|
||||
))
|
||||
self.assertTrue(claim4['success'])
|
||||
await self.confirm_tx(claim4['tx']['txid'])
|
||||
|
||||
await self.generate(5)
|
||||
|
||||
# He sends the link to Ramsey, all happy and proud
|
||||
uri = 'lbry://@spam/hit-song'
|
||||
|
||||
# But sadly Ramsey wasn't so pleased. It was hard for him to tell Chris...
|
||||
# Chris, though a bit heartbroken, abandoned the claim for now, but instantly started working on new hit lyrics
|
||||
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=claim4['tx']['txid'], nout=0, blocking=False))
|
||||
self.assertTrue(abandon['success'])
|
||||
await self.confirm_tx(abandon['tx']['txid'])
|
||||
|
||||
# He them checks that the claim doesn't resolve anymore.
|
||||
response = await self.out(self.daemon.jsonrpc_resolve(uri))
|
||||
self.assertNotIn('claim', response[uri])
|
||||
@@ -0,0 +1,406 @@
|
||||
import tempfile
|
||||
|
||||
from lbrynet.extras.wallet.transaction import Transaction
|
||||
from lbrynet.error import InsufficientFundsError
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
|
||||
from integration.testcase import CommandTestCase
|
||||
|
||||
|
||||
class ClaimCommands(CommandTestCase):
|
||||
|
||||
async def craft_claim(self, name, amount_dewies, claim_dict, address):
|
||||
# FIXME: this is here mostly because publish has defensive code for situations that happens accidentally
|
||||
# However, it still happens... So, let's reproduce them.
|
||||
claim = ClaimDict.load_dict(claim_dict)
|
||||
address = address or (await self.account.receiving.get_addresses(limit=1, only_usable=True))[0]
|
||||
tx = await Transaction.claim(name, claim, amount_dewies, address, [self.account], self.account)
|
||||
await self.broadcast(tx)
|
||||
await self.ledger.wait(tx)
|
||||
await self.generate(1)
|
||||
await self.ledger.wait(tx)
|
||||
return tx
|
||||
|
||||
async def test_create_update_and_abandon_claim(self):
|
||||
await self.assertBalance(self.account, '10.0')
|
||||
|
||||
claim = await self.make_claim(amount='2.5') # creates new claim
|
||||
txs = await self.out(self.daemon.jsonrpc_transaction_list())
|
||||
self.assertEqual(len(txs[0]['claim_info']), 1)
|
||||
self.assertEqual(txs[0]['confirmations'], 1)
|
||||
self.assertEqual(txs[0]['claim_info'][0]['balance_delta'], '-2.5')
|
||||
self.assertEqual(txs[0]['claim_info'][0]['claim_id'], claim['claim_id'])
|
||||
self.assertEqual(txs[0]['value'], '0.0')
|
||||
self.assertEqual(txs[0]['fee'], '-0.020107')
|
||||
await self.assertBalance(self.account, '7.479893')
|
||||
|
||||
await self.make_claim(amount='1.0') # updates previous claim
|
||||
txs = await self.out(self.daemon.jsonrpc_transaction_list())
|
||||
self.assertEqual(len(txs[0]['update_info']), 1)
|
||||
self.assertEqual(txs[0]['update_info'][0]['balance_delta'], '1.5')
|
||||
self.assertEqual(txs[0]['update_info'][0]['claim_id'], claim['claim_id'])
|
||||
self.assertEqual(txs[0]['value'], '0.0')
|
||||
self.assertEqual(txs[0]['fee'], '-0.0001985')
|
||||
await self.assertBalance(self.account, '8.9796945')
|
||||
|
||||
await self.out(self.daemon.jsonrpc_claim_abandon(claim['claim_id']))
|
||||
txs = await self.out(self.daemon.jsonrpc_transaction_list())
|
||||
self.assertEqual(len(txs[0]['abandon_info']), 1)
|
||||
self.assertEqual(txs[0]['abandon_info'][0]['balance_delta'], '1.0')
|
||||
self.assertEqual(txs[0]['abandon_info'][0]['claim_id'], claim['claim_id'])
|
||||
self.assertEqual(txs[0]['value'], '0.0')
|
||||
self.assertEqual(txs[0]['fee'], '-0.000107')
|
||||
await self.assertBalance(self.account, '9.9795875')
|
||||
|
||||
async def test_update_claim_holding_address(self):
|
||||
other_account_id = (await self.daemon.jsonrpc_account_create('second account'))['id']
|
||||
other_account = self.daemon.get_account_or_error(other_account_id)
|
||||
other_address = await other_account.receiving.get_or_create_usable_address()
|
||||
|
||||
await self.assertBalance(self.account, '10.0')
|
||||
|
||||
# create the initial name claim
|
||||
claim = await self.make_claim()
|
||||
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list_mine()), 1)
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list_mine(account_id=other_account_id)), 0)
|
||||
tx = await self.daemon.jsonrpc_claim_send_to_address(
|
||||
claim['claim_id'], other_address
|
||||
)
|
||||
await self.ledger.wait(tx)
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list_mine()), 0)
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list_mine(account_id=other_account_id)), 1)
|
||||
|
||||
async def test_publishing_checks_all_accounts_for_certificate(self):
|
||||
account1_id, account1 = self.account.id, self.account
|
||||
new_account = await self.daemon.jsonrpc_account_create('second account')
|
||||
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
|
||||
|
||||
spam_channel = await self.out(self.daemon.jsonrpc_channel_new('@spam', '1.0'))
|
||||
self.assertTrue(spam_channel['success'])
|
||||
await self.confirm_tx(spam_channel['tx']['txid'])
|
||||
|
||||
self.assertEqual('8.989893', await self.daemon.jsonrpc_account_balance())
|
||||
|
||||
result = await self.out(self.daemon.jsonrpc_wallet_send(
|
||||
'5.0', await self.daemon.jsonrpc_address_unused(account2_id)
|
||||
))
|
||||
await self.confirm_tx(result['txid'])
|
||||
|
||||
self.assertEqual('3.989769', await self.daemon.jsonrpc_account_balance())
|
||||
self.assertEqual('5.0', await self.daemon.jsonrpc_account_balance(account2_id))
|
||||
|
||||
baz_channel = await self.out(self.daemon.jsonrpc_channel_new('@baz', '1.0', account2_id))
|
||||
self.assertTrue(baz_channel['success'])
|
||||
await self.confirm_tx(baz_channel['tx']['txid'])
|
||||
|
||||
channels = await self.out(self.daemon.jsonrpc_channel_list(account1_id))
|
||||
self.assertEqual(len(channels), 1)
|
||||
self.assertEqual(channels[0]['name'], '@spam')
|
||||
self.assertEqual(channels, await self.out(self.daemon.jsonrpc_channel_list()))
|
||||
|
||||
channels = await self.out(self.daemon.jsonrpc_channel_list(account2_id))
|
||||
self.assertEqual(len(channels), 1)
|
||||
self.assertEqual(channels[0]['name'], '@baz')
|
||||
|
||||
# defaults to using all accounts to lookup channel
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'hi!')
|
||||
file.flush()
|
||||
claim1 = await self.out(self.daemon.jsonrpc_publish(
|
||||
'hovercraft', '1.0', file_path=file.name, channel_name='@baz'
|
||||
))
|
||||
self.assertTrue(claim1['success'])
|
||||
await self.confirm_tx(claim1['tx']['txid'])
|
||||
|
||||
# uses only the specific accounts which contains the channel
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'hi!')
|
||||
file.flush()
|
||||
claim1 = await self.out(self.daemon.jsonrpc_publish(
|
||||
'hovercraft', '1.0', file_path=file.name,
|
||||
channel_name='@baz', channel_account_id=[account2_id]
|
||||
))
|
||||
self.assertTrue(claim1['success'])
|
||||
await self.confirm_tx(claim1['tx']['txid'])
|
||||
|
||||
# fails when specifying account which does not contain channel
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'hi!')
|
||||
file.flush()
|
||||
with self.assertRaisesRegex(ValueError, "Couldn't find channel with name '@baz'."):
|
||||
await self.out(self.daemon.jsonrpc_publish(
|
||||
'hovercraft', '1.0', file_path=file.name,
|
||||
channel_name='@baz', channel_account_id=[account1_id]
|
||||
))
|
||||
|
||||
async def test_updating_claim_includes_claim_value_in_balance_check(self):
|
||||
await self.assertBalance(self.account, '10.0')
|
||||
|
||||
await self.make_claim(amount='9.0')
|
||||
await self.assertBalance(self.account, '0.979893')
|
||||
|
||||
# update the same claim
|
||||
await self.make_claim(amount='9.0')
|
||||
await self.assertBalance(self.account, '0.9796205')
|
||||
|
||||
# update the claim a second time but use even more funds
|
||||
await self.make_claim(amount='9.97')
|
||||
await self.assertBalance(self.account, '0.009348')
|
||||
|
||||
# fails when specifying more than available
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'hi!')
|
||||
file.flush()
|
||||
with self.assertRaisesRegex(
|
||||
InsufficientFundsError,
|
||||
"Please lower the bid value, the maximum amount"
|
||||
" you can specify for this claim is 9.979274."
|
||||
):
|
||||
await self.out(self.daemon.jsonrpc_publish(
|
||||
'hovercraft', '9.98', file_path=file.name
|
||||
))
|
||||
|
||||
async def test_abandoning_claim_at_loss(self):
|
||||
await self.assertBalance(self.account, '10.0')
|
||||
claim = await self.make_claim(amount='0.0001')
|
||||
await self.assertBalance(self.account, '9.979793')
|
||||
await self.out(self.daemon.jsonrpc_claim_abandon(claim['claim_id']))
|
||||
await self.assertBalance(self.account, '9.97968399')
|
||||
|
||||
async def test_claim_show(self):
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
|
||||
self.assertTrue(channel['success'])
|
||||
await self.confirm_tx(channel['tx']['txid'])
|
||||
channel_from_claim_show = await self.out(
|
||||
self.daemon.jsonrpc_claim_show(txid=channel['tx']['txid'], nout=channel['output']['nout'])
|
||||
)
|
||||
self.assertEqual(channel_from_claim_show['value'], channel['output']['value'])
|
||||
channel_from_claim_show = await self.out(
|
||||
self.daemon.jsonrpc_claim_show(claim_id=channel['claim_id'])
|
||||
)
|
||||
self.assertEqual(channel_from_claim_show['value'], channel['output']['value'])
|
||||
|
||||
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=channel['tx']['txid'], nout=0, blocking=False))
|
||||
self.assertTrue(abandon['success'])
|
||||
await self.confirm_tx(abandon['tx']['txid'])
|
||||
not_a_claim = await self.out(
|
||||
self.daemon.jsonrpc_claim_show(txid=abandon['tx']['txid'], nout=0)
|
||||
)
|
||||
self.assertEqual(not_a_claim, 'claim not found')
|
||||
|
||||
async def test_claim_list(self):
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
|
||||
self.assertTrue(channel['success'])
|
||||
await self.confirm_tx(channel['tx']['txid'])
|
||||
claim = await self.make_claim(amount='0.0001', name='on-channel-claim', channel_name='@abc')
|
||||
self.assertTrue(claim['success'])
|
||||
unsigned_claim = await self.make_claim(amount='0.0001', name='unsigned')
|
||||
self.assertTrue(claim['success'])
|
||||
|
||||
channel_from_claim_list = await self.out(self.daemon.jsonrpc_claim_list('@abc'))
|
||||
self.assertEqual(channel_from_claim_list['claims'][0]['value'], channel['output']['value'])
|
||||
signed_claim_from_claim_list = await self.out(self.daemon.jsonrpc_claim_list('on-channel-claim'))
|
||||
self.assertEqual(signed_claim_from_claim_list['claims'][0]['value'], claim['output']['value'])
|
||||
unsigned_claim_from_claim_list = await self.out(self.daemon.jsonrpc_claim_list('unsigned'))
|
||||
self.assertEqual(unsigned_claim_from_claim_list['claims'][0]['value'], unsigned_claim['output']['value'])
|
||||
|
||||
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=channel['tx']['txid'], nout=0, blocking=False))
|
||||
self.assertTrue(abandon['success'])
|
||||
await self.confirm_tx(abandon['tx']['txid'])
|
||||
|
||||
empty = await self.out(self.daemon.jsonrpc_claim_list('@abc'))
|
||||
self.assertEqual(len(empty['claims']), 0)
|
||||
|
||||
async def test_abandoned_channel_with_signed_claims(self):
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
|
||||
self.assertTrue(channel['success'])
|
||||
await self.confirm_tx(channel['tx']['txid'])
|
||||
claim = await self.make_claim(amount='0.0001', name='on-channel-claim', channel_name='@abc')
|
||||
self.assertTrue(claim['success'])
|
||||
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=channel['tx']['txid'], nout=0, blocking=False))
|
||||
self.assertTrue(abandon['success'])
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
|
||||
self.assertTrue(channel['success'])
|
||||
await self.confirm_tx(channel['tx']['txid'])
|
||||
|
||||
# Original channel doesnt exists anymore, so the signature is invalid. For invalid signatures, resolution is
|
||||
# only possible outside a channel
|
||||
response = await self.resolve('lbry://@abc/on-channel-claim')
|
||||
self.assertNotIn('claim', response['lbry://@abc/on-channel-claim'])
|
||||
response = await self.resolve('lbry://on-channel-claim')
|
||||
self.assertIn('claim', response['lbry://on-channel-claim'])
|
||||
self.assertFalse(response['lbry://on-channel-claim']['claim']['signature_is_valid'])
|
||||
direct_uri = 'lbry://on-channel-claim#' + claim['claim_id']
|
||||
response = await self.resolve(direct_uri)
|
||||
self.assertIn('claim', response[direct_uri])
|
||||
self.assertFalse(response[direct_uri]['claim']['signature_is_valid'])
|
||||
|
||||
uri = 'lbry://@abc/on-channel-claim'
|
||||
# now, claim something on this channel (it will update the invalid claim, but we save and forcefully restore)
|
||||
original_claim = await self.make_claim(amount='0.00000001', name='on-channel-claim', channel_name='@abc')
|
||||
self.assertTrue(original_claim['success'])
|
||||
# resolves normally
|
||||
response = await self.resolve(uri)
|
||||
self.assertIn('claim', response[uri])
|
||||
self.assertTrue(response[uri]['claim']['signature_is_valid'])
|
||||
|
||||
# tamper it, invalidating the signature
|
||||
value = response[uri]['claim']['value'].copy()
|
||||
value['stream']['metadata']['author'] = 'some troll'
|
||||
address = response[uri]['claim']['address']
|
||||
await self.craft_claim('on-channel-claim', 1, value, address)
|
||||
# it resolves to the now only valid claim under the channel, ignoring the fake one
|
||||
response = await self.resolve(uri)
|
||||
self.assertIn('claim', response[uri])
|
||||
self.assertTrue(response[uri]['claim']['signature_is_valid'])
|
||||
|
||||
# ooops! claimed a valid conflict! (this happens on the wild, mostly by accident or race condition)
|
||||
await self.craft_claim('on-channel-claim', 1, response[uri]['claim']['value'], address)
|
||||
|
||||
# it still resolves! but to the older claim
|
||||
response = await self.resolve(uri)
|
||||
self.assertIn('claim', response[uri])
|
||||
self.assertTrue(response[uri]['claim']['signature_is_valid'])
|
||||
self.assertEqual(response[uri]['claim']['txid'], original_claim['tx']['txid'])
|
||||
|
||||
async def test_claim_list_by_channel(self):
|
||||
self.maxDiff = None
|
||||
tx = await self.daemon.jsonrpc_account_fund(None, None, '0.001', outputs=100, broadcast=True)
|
||||
await self.ledger.wait(tx)
|
||||
await self.generate(1)
|
||||
await self.ledger.wait(tx)
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "0.0001"))
|
||||
self.assertTrue(channel['success'])
|
||||
await self.confirm_tx(channel['tx']['txid'])
|
||||
|
||||
# 4 claims per block, 3 blocks. Sorted by height (descending) then claim_id (ascending).
|
||||
claims = []
|
||||
for j in range(3):
|
||||
same_height_claims = []
|
||||
for k in range(3):
|
||||
claim = await self.make_claim(amount='0.000001', name=f'c{j}-{k}', channel_name='@abc', confirm=False)
|
||||
self.assertTrue(claim['success'])
|
||||
same_height_claims.append(claim['claim_id'])
|
||||
await self.on_transaction_dict(claim['tx'])
|
||||
claim = await self.make_claim(amount='0.000001', name=f'c{j}-4', channel_name='@abc', confirm=True)
|
||||
self.assertTrue(claim['success'])
|
||||
same_height_claims.append(claim['claim_id'])
|
||||
same_height_claims.sort(key=lambda x: int(x, 16))
|
||||
claims = same_height_claims + claims
|
||||
|
||||
page = await self.out(self.daemon.jsonrpc_claim_list_by_channel(1, page_size=20, uri='@abc'))
|
||||
page_claim_ids = [item['claim_id'] for item in page['@abc']['claims_in_channel']]
|
||||
self.assertEqual(page_claim_ids, claims)
|
||||
page = await self.out(self.daemon.jsonrpc_claim_list_by_channel(1, page_size=6, uri='@abc'))
|
||||
page_claim_ids = [item['claim_id'] for item in page['@abc']['claims_in_channel']]
|
||||
self.assertEqual(page_claim_ids, claims[:6])
|
||||
out_of_bounds = await self.out(self.daemon.jsonrpc_claim_list_by_channel(2, page_size=20, uri='@abc'))
|
||||
self.assertEqual(out_of_bounds['error'], 'claim 20 greater than max 12')
|
||||
|
||||
async def test_regular_supports_and_tip_supports(self):
|
||||
# account2 will be used to send tips and supports to account1
|
||||
account2_id = (await self.daemon.jsonrpc_account_create('second account'))['id']
|
||||
|
||||
# send account2 5 LBC out of the 10 LBC in account1
|
||||
result = await self.out(self.daemon.jsonrpc_wallet_send(
|
||||
'5.0', await self.daemon.jsonrpc_address_unused(account2_id)
|
||||
))
|
||||
await self.confirm_tx(result['txid'])
|
||||
|
||||
# account1 and account2 balances:
|
||||
self.assertEqual('4.999876', await self.daemon.jsonrpc_account_balance())
|
||||
self.assertEqual('5.0', await self.daemon.jsonrpc_account_balance(account2_id))
|
||||
|
||||
# create the claim we'll be tipping and supporting
|
||||
claim = await self.make_claim()
|
||||
|
||||
# account1 and account2 balances:
|
||||
self.assertEqual('3.979769', await self.daemon.jsonrpc_account_balance())
|
||||
self.assertEqual('5.0', await self.daemon.jsonrpc_account_balance(account2_id))
|
||||
|
||||
# send a tip to the claim using account2
|
||||
tip = await self.out(
|
||||
self.daemon.jsonrpc_claim_tip(claim['claim_id'], '1.0', account2_id)
|
||||
)
|
||||
await self.on_transaction_dict(tip)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(tip)
|
||||
|
||||
# tips don't affect balance so account1 balance is same but account2 balance went down
|
||||
self.assertEqual('3.979769', await self.daemon.jsonrpc_account_balance())
|
||||
self.assertEqual('3.9998585', await self.daemon.jsonrpc_account_balance(account2_id))
|
||||
|
||||
# verify that the incoming tip is marked correctly as is_tip=True in account1
|
||||
txs = await self.out(self.daemon.jsonrpc_transaction_list())
|
||||
self.assertEqual(len(txs[0]['support_info']), 1)
|
||||
self.assertEqual(txs[0]['support_info'][0]['balance_delta'], '1.0')
|
||||
self.assertEqual(txs[0]['support_info'][0]['claim_id'], claim['claim_id'])
|
||||
self.assertEqual(txs[0]['support_info'][0]['is_tip'], True)
|
||||
self.assertEqual(txs[0]['value'], '1.0')
|
||||
self.assertEqual(txs[0]['fee'], '0.0')
|
||||
|
||||
# verify that the outgoing tip is marked correctly as is_tip=True in account2
|
||||
txs2 = await self.out(
|
||||
self.daemon.jsonrpc_transaction_list(account2_id)
|
||||
)
|
||||
self.assertEqual(len(txs2[0]['support_info']), 1)
|
||||
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-1.0')
|
||||
self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim['claim_id'])
|
||||
self.assertEqual(txs2[0]['support_info'][0]['is_tip'], True)
|
||||
self.assertEqual(txs2[0]['value'], '-1.0')
|
||||
self.assertEqual(txs2[0]['fee'], '-0.0001415')
|
||||
|
||||
# send a support to the claim using account2
|
||||
support = await self.out(
|
||||
self.daemon.jsonrpc_claim_new_support('hovercraft', claim['claim_id'], '2.0', account2_id)
|
||||
)
|
||||
await self.on_transaction_dict(support)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(support)
|
||||
|
||||
# account2 balance went down ~2
|
||||
self.assertEqual('3.979769', await self.daemon.jsonrpc_account_balance())
|
||||
self.assertEqual('1.999717', await self.daemon.jsonrpc_account_balance(account2_id))
|
||||
|
||||
# verify that the outgoing support is marked correctly as is_tip=False in account2
|
||||
txs2 = await self.out(self.daemon.jsonrpc_transaction_list(account2_id))
|
||||
self.assertEqual(len(txs2[0]['support_info']), 1)
|
||||
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-2.0')
|
||||
self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim['claim_id'])
|
||||
self.assertEqual(txs2[0]['support_info'][0]['is_tip'], False)
|
||||
self.assertEqual(txs2[0]['value'], '0.0')
|
||||
self.assertEqual(txs2[0]['fee'], '-0.0001415')
|
||||
|
||||
async def test_normalization_resolution(self):
|
||||
|
||||
# this test assumes that the lbrycrd forks normalization at height == 250 on regtest
|
||||
|
||||
c1 = await self.make_claim('ΣίσυφοςfiÆ', '0.1')
|
||||
c2 = await self.make_claim('ΣΊΣΥΦΟσFIæ', '0.2')
|
||||
|
||||
r1 = await self.daemon.jsonrpc_resolve(urls='lbry://ΣίσυφοςfiÆ')
|
||||
r2 = await self.daemon.jsonrpc_resolve(urls='lbry://ΣΊΣΥΦΟσFIæ')
|
||||
|
||||
r1c = list(r1.values())[0]['claim']['claim_id']
|
||||
r2c = list(r2.values())[0]['claim']['claim_id']
|
||||
self.assertEqual(c1['claim_id'], r1c)
|
||||
self.assertEqual(c2['claim_id'], r2c)
|
||||
self.assertNotEqual(r1c, r2c)
|
||||
|
||||
await self.generate(50)
|
||||
head = await self.daemon.jsonrpc_block_show()
|
||||
self.assertTrue(head['height'] > 250)
|
||||
|
||||
r3 = await self.daemon.jsonrpc_resolve(urls='lbry://ΣίσυφοςfiÆ')
|
||||
r4 = await self.daemon.jsonrpc_resolve(urls='lbry://ΣΊΣΥΦΟσFIæ')
|
||||
|
||||
r3c = list(r3.values())[0]['claim']['claim_id']
|
||||
r4c = list(r4.values())[0]['claim']['claim_id']
|
||||
r3n = list(r3.values())[0]['claim']['name']
|
||||
r4n = list(r4.values())[0]['claim']['name']
|
||||
|
||||
self.assertEqual(c2['claim_id'], r3c)
|
||||
self.assertEqual(c2['claim_id'], r4c)
|
||||
self.assertEqual(r3c, r4c)
|
||||
self.assertEqual(r3n, r4n)
|
||||
File renamed without changes.
@@ -0,0 +1,191 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
|
||||
from integration.testcase import CommandTestCase
|
||||
from lbrynet.blob_exchange.downloader import BlobDownloader
|
||||
from lbrynet.error import InsufficientFundsError
|
||||
|
||||
|
||||
class FileCommands(CommandTestCase):
|
||||
|
||||
VERBOSITY = logging.WARN
|
||||
|
||||
async def test_file_management(self):
|
||||
await self.make_claim('foo', '0.01')
|
||||
await self.make_claim('foo2', '0.01')
|
||||
|
||||
file1, file2 = self.daemon.jsonrpc_file_list('claim_name')
|
||||
self.assertEqual(file1['claim_name'], 'foo')
|
||||
self.assertEqual(file2['claim_name'], 'foo2')
|
||||
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo')
|
||||
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo2')
|
||||
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 0)
|
||||
|
||||
await self.daemon.jsonrpc_get('lbry://foo')
|
||||
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
|
||||
|
||||
async def test_download_different_timeouts(self):
|
||||
claim = await self.make_claim('foo', '0.01')
|
||||
sd_hash = claim['output']['value']['stream']['source']['source']
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo')
|
||||
all_except_sd = [
|
||||
blob_hash for blob_hash in self.server.blob_manager.completed_blob_hashes if blob_hash != sd_hash
|
||||
]
|
||||
await self.server.blob_manager.delete_blobs(all_except_sd)
|
||||
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2)
|
||||
self.assertIn('error', resp)
|
||||
self.assertEquals('Failed to download data blobs for sd hash %s within timeout' % sd_hash, resp['error'])
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo')
|
||||
await self.server.blob_manager.delete_blobs([sd_hash])
|
||||
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2)
|
||||
self.assertIn('error', resp)
|
||||
self.assertEquals('Failed to download sd blob %s within timeout' % sd_hash, resp['error'])
|
||||
|
||||
async def wait_files_to_complete(self):
|
||||
while self.daemon.jsonrpc_file_list(status='running'):
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
async def test_filename_conflicts_management_on_resume_download(self):
|
||||
await self.make_claim('foo', '0.01', data=bytes([0]*(1<<23)))
|
||||
file_info = self.daemon.jsonrpc_file_list()[0]
|
||||
original_path = os.path.join(self.daemon.conf.download_dir, file_info['file_name'])
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo')
|
||||
await self.daemon.jsonrpc_get('lbry://foo')
|
||||
with open(original_path, 'wb') as handle:
|
||||
handle.write(b'some other stuff was there instead')
|
||||
self.daemon.stream_manager.stop()
|
||||
await self.daemon.stream_manager.start()
|
||||
await asyncio.wait_for(self.wait_files_to_complete(), timeout=5) # if this hangs, file didnt get set completed
|
||||
# check that internal state got through up to the file list API
|
||||
downloader = self.daemon.stream_manager.get_stream_by_stream_hash(file_info['stream_hash']).downloader
|
||||
file_info = self.daemon.jsonrpc_file_list()[0]
|
||||
self.assertEqual(downloader.output_file_name, file_info['file_name'])
|
||||
# checks if what the API shows is what he have at the very internal level.
|
||||
self.assertEqual(downloader.output_path, file_info['download_path'])
|
||||
# if you got here refactoring just change above, but ensure what gets set internally gets reflected externally!
|
||||
self.assertTrue(downloader.output_path.endswith(downloader.output_file_name))
|
||||
# this used to be inconsistent, if it becomes again it would create weird bugs, so worth checking
|
||||
|
||||
async def test_incomplete_downloads_erases_output_file_on_stop(self):
|
||||
claim = await self.make_claim('foo', '0.01')
|
||||
sd_hash = claim['output']['value']['stream']['source']['source']
|
||||
file_info = self.daemon.jsonrpc_file_list()[0]
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo')
|
||||
blobs = await self.server_storage.get_blobs_for_stream(
|
||||
await self.server_storage.get_stream_hash_for_sd_hash(sd_hash)
|
||||
)
|
||||
all_except_sd_and_head = [
|
||||
blob.blob_hash for blob in blobs[1:] if blob.blob_hash
|
||||
]
|
||||
await self.server.blob_manager.delete_blobs(all_except_sd_and_head)
|
||||
self.assertFalse(os.path.isfile(os.path.join(self.daemon.conf.download_dir, file_info['file_name'])))
|
||||
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2)
|
||||
self.assertNotIn('error', resp)
|
||||
self.assertTrue(os.path.isfile(os.path.join(self.daemon.conf.download_dir, file_info['file_name'])))
|
||||
self.daemon.stream_manager.stop()
|
||||
self.assertFalse(os.path.isfile(os.path.join(self.daemon.conf.download_dir, file_info['file_name'])))
|
||||
|
||||
async def test_incomplete_downloads_retry(self):
|
||||
claim = await self.make_claim('foo', '0.01')
|
||||
sd_hash = claim['output']['value']['stream']['source']['source']
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo')
|
||||
blobs = await self.server_storage.get_blobs_for_stream(
|
||||
await self.server_storage.get_stream_hash_for_sd_hash(sd_hash)
|
||||
)
|
||||
all_except_sd_and_head = [
|
||||
blob.blob_hash for blob in blobs[1:] if blob.blob_hash
|
||||
]
|
||||
|
||||
# backup server blobs
|
||||
for blob_hash in all_except_sd_and_head:
|
||||
blob = self.server_blob_manager.get_blob(blob_hash)
|
||||
os.rename(blob.file_path, blob.file_path + '__')
|
||||
|
||||
# erase all except sd blob
|
||||
await self.server.blob_manager.delete_blobs(all_except_sd_and_head)
|
||||
|
||||
# start the download
|
||||
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2)
|
||||
self.assertNotIn('error', resp)
|
||||
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
|
||||
self.assertEqual('running', self.daemon.jsonrpc_file_list()[0]['status'])
|
||||
await self.daemon.jsonrpc_file_set_status('stop', claim_name='foo')
|
||||
|
||||
# recover blobs
|
||||
for blob_hash in all_except_sd_and_head:
|
||||
blob = self.server_blob_manager.get_blob(blob_hash)
|
||||
os.rename(blob.file_path + '__', blob.file_path)
|
||||
self.server_blob_manager.blobs.clear()
|
||||
await self.server_blob_manager.blob_completed(self.server_blob_manager.get_blob(blob_hash))
|
||||
|
||||
await self.daemon.jsonrpc_file_set_status('start', claim_name='foo')
|
||||
await asyncio.wait_for(self.wait_files_to_complete(), timeout=5)
|
||||
file_info = self.daemon.jsonrpc_file_list()[0]
|
||||
self.assertEqual(file_info['blobs_completed'], file_info['blobs_in_stream'])
|
||||
self.assertEqual('finished', file_info['status'])
|
||||
|
||||
async def test_unban_recovers_stream(self):
|
||||
BlobDownloader.BAN_TIME = .5 # fixme: temporary field, will move to connection manager or a conf
|
||||
claim = await self.make_claim('foo', '0.01', data=bytes([0]*(1<<23)))
|
||||
sd_hash = claim['output']['value']['stream']['source']['source']
|
||||
missing_blob_hash = (await self.daemon.jsonrpc_blob_list(sd_hash=sd_hash))[-2]
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo')
|
||||
# backup blob
|
||||
missing_blob = self.server_blob_manager.get_blob(missing_blob_hash)
|
||||
os.rename(missing_blob.file_path, missing_blob.file_path + '__')
|
||||
self.server_blob_manager.delete_blob(missing_blob_hash)
|
||||
await self.daemon.jsonrpc_get('lbry://foo')
|
||||
with self.assertRaises(asyncio.TimeoutError):
|
||||
await asyncio.wait_for(self.wait_files_to_complete(), timeout=1)
|
||||
# restore blob
|
||||
os.rename(missing_blob.file_path + '__', missing_blob.file_path)
|
||||
self.server_blob_manager.blobs.clear()
|
||||
missing_blob = self.server_blob_manager.get_blob(missing_blob_hash)
|
||||
await self.server_blob_manager.blob_completed(missing_blob)
|
||||
await asyncio.wait_for(self.wait_files_to_complete(), timeout=1)
|
||||
|
||||
async def test_paid_download(self):
|
||||
target_address = await self.blockchain.get_raw_change_address()
|
||||
|
||||
# FAIL: beyond available balance
|
||||
await self.make_claim(
|
||||
'expensive', '0.01', data=b'pay me if you can',
|
||||
fee={'currency': 'LBC', 'amount': 11.0, 'address': target_address})
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='expensive')
|
||||
response = await self.daemon.jsonrpc_get('lbry://expensive')
|
||||
self.assertEqual(response['error'], 'fee of 11.0 exceeds max available balance')
|
||||
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 0)
|
||||
|
||||
# FAIL: beyond maximum key fee
|
||||
await self.make_claim(
|
||||
'maxkey', '0.01', data=b'no pay me, no',
|
||||
fee={'currency': 'LBC', 'amount': 111.0, 'address': target_address})
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='maxkey')
|
||||
response = await self.daemon.jsonrpc_get('lbry://maxkey')
|
||||
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 0)
|
||||
self.assertEqual(response['error'], 'fee of 111.0 exceeds max configured to allow of 50.0')
|
||||
|
||||
# PASS: purchase is successful
|
||||
await self.make_claim(
|
||||
'icanpay', '0.01', data=b'I got the power!',
|
||||
fee={'currency': 'LBC', 'amount': 1.0, 'address': target_address})
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='icanpay')
|
||||
await self.assertBalance(self.account, '9.925679')
|
||||
response = await self.daemon.jsonrpc_get('lbry://icanpay')
|
||||
self.assertNotIn('error', response)
|
||||
await self.ledger.wait(response['tx'])
|
||||
await self.assertBalance(self.account, '8.925555')
|
||||
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
|
||||
|
||||
await asyncio.wait_for(self.wait_files_to_complete(), timeout=1)
|
||||
|
||||
# check that the fee was received
|
||||
starting_balance = await self.blockchain.get_balance()
|
||||
await self.generate(1)
|
||||
block_reward_and_claim_fee = 2.0
|
||||
self.assertEqual(
|
||||
await self.blockchain.get_balance(), starting_balance + block_reward_and_claim_fee
|
||||
)
|
||||
+2
-1
@@ -8,6 +8,7 @@ from lbrynet.schema.signature import Signature, NAMED_SECP256K1
|
||||
from lbrynet.schema.signer import get_signer
|
||||
from torba.testcase import IntegrationTestCase
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
import lbrynet.extras.wallet
|
||||
from lbrynet.extras.wallet.transaction import Transaction
|
||||
from lbrynet.extras.wallet.account import generate_certificate
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc as d2l, lbc_to_dewies as l2d
|
||||
@@ -45,7 +46,7 @@ example_claim_dict = {
|
||||
|
||||
class BasicTransactionTest(IntegrationTestCase):
|
||||
|
||||
VERBOSITY = logging.WARN
|
||||
LEDGER = lbrynet.extras.wallet
|
||||
|
||||
async def test_creating_updating_and_abandoning_claim_with_channel(self):
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
from integration.testcase import CommandTestCase
|
||||
|
||||
|
||||
class ResolveCommand(CommandTestCase):
|
||||
|
||||
async def test_resolve(self):
|
||||
await self.make_channel('@abc', '0.01')
|
||||
|
||||
# resolving a channel @abc
|
||||
response = await self.resolve('lbry://@abc')
|
||||
self.assertSetEqual({'lbry://@abc'}, set(response))
|
||||
self.assertIn('certificate', response['lbry://@abc'])
|
||||
self.assertNotIn('claim', response['lbry://@abc'])
|
||||
self.assertEqual(response['lbry://@abc']['certificate']['name'], '@abc')
|
||||
self.assertEqual(response['lbry://@abc']['claims_in_channel'], 0)
|
||||
|
||||
await self.make_claim('foo', '0.01', channel_name='@abc')
|
||||
await self.make_claim('foo2', '0.01', channel_name='@abc')
|
||||
|
||||
# resolving a channel @abc with some claims in it
|
||||
response = await self.resolve('lbry://@abc')
|
||||
self.assertSetEqual({'lbry://@abc'}, set(response))
|
||||
self.assertIn('certificate', response['lbry://@abc'])
|
||||
self.assertNotIn('claim', response['lbry://@abc'])
|
||||
self.assertEqual(response['lbry://@abc']['certificate']['name'], '@abc')
|
||||
self.assertEqual(response['lbry://@abc']['claims_in_channel'], 2)
|
||||
|
||||
# resolving claim foo within channel @abc
|
||||
response = await self.resolve('lbry://@abc/foo')
|
||||
self.assertSetEqual({'lbry://@abc/foo'}, set(response))
|
||||
claim = response['lbry://@abc/foo']
|
||||
self.assertIn('certificate', claim)
|
||||
self.assertIn('claim', claim)
|
||||
self.assertEqual(claim['claim']['name'], 'foo')
|
||||
self.assertEqual(claim['claim']['channel_name'], '@abc')
|
||||
self.assertEqual(claim['certificate']['name'], '@abc')
|
||||
self.assertEqual(claim['claims_in_channel'], 0)
|
||||
|
||||
# resolving claim foo by itself
|
||||
response = await self.resolve('lbry://foo')
|
||||
self.assertSetEqual({'lbry://foo'}, set(response))
|
||||
claim = response['lbry://foo']
|
||||
self.assertIn('certificate', claim)
|
||||
self.assertIn('claim', claim)
|
||||
self.assertEqual(claim['claim']['name'], 'foo')
|
||||
self.assertEqual(claim['claim']['channel_name'], '@abc')
|
||||
self.assertEqual(claim['certificate']['name'], '@abc')
|
||||
self.assertEqual(claim['claims_in_channel'], 0)
|
||||
|
||||
# resolving multiple at once
|
||||
response = await self.resolve(['lbry://foo', 'lbry://foo2'])
|
||||
self.assertSetEqual({'lbry://foo', 'lbry://foo2'}, set(response))
|
||||
claim = response['lbry://foo2']
|
||||
self.assertIn('certificate', claim)
|
||||
self.assertIn('claim', claim)
|
||||
self.assertEqual(claim['claim']['name'], 'foo2')
|
||||
self.assertEqual(claim['claim']['channel_name'], '@abc')
|
||||
self.assertEqual(claim['certificate']['name'], '@abc')
|
||||
self.assertEqual(claim['claims_in_channel'], 0)
|
||||
@@ -0,0 +1,91 @@
|
||||
from unittest import mock
|
||||
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
|
||||
from torba.orchstr8.node import WalletNode, SPVNode
|
||||
from torba.testcase import AsyncioTestCase
|
||||
|
||||
import lbrynet.schema
|
||||
lbrynet.schema.BLOCKCHAIN_NAME = 'lbrycrd_regtest'
|
||||
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.Daemon import Daemon
|
||||
from lbrynet.extras.wallet import LbryWalletManager, RegTestLedger
|
||||
from lbrynet.extras.daemon.Components import WalletComponent
|
||||
from lbrynet.extras.daemon.Components import (
|
||||
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
)
|
||||
from lbrynet.extras.daemon.ComponentManager import ComponentManager
|
||||
|
||||
|
||||
class AccountSynchronization(AsyncioTestCase):
|
||||
|
||||
async def asyncSetUp(self):
|
||||
self.wallet_node = WalletNode(LbryWalletManager, RegTestLedger)
|
||||
await self.wallet_node.start(
|
||||
SPVNode(None),
|
||||
"carbon smart garage balance margin twelve chest sword toast envelope bottom stomach absent",
|
||||
False
|
||||
)
|
||||
self.account = self.wallet_node.account
|
||||
|
||||
conf = Config()
|
||||
conf.data_dir = self.wallet_node.data_path
|
||||
conf.wallet_dir = self.wallet_node.data_path
|
||||
conf.download_dir = self.wallet_node.data_path
|
||||
conf.share_usage_data = False
|
||||
conf.use_upnp = False
|
||||
conf.reflect_streams = False
|
||||
conf.blockchain_name = 'lbrycrd_regtest'
|
||||
conf.lbryum_servers = [('localhost', 50001)]
|
||||
conf.reflector_servers = []
|
||||
conf.known_dht_nodes = []
|
||||
|
||||
def wallet_maker(component_manager):
|
||||
self.wallet_component = WalletComponent(component_manager)
|
||||
self.wallet_component.wallet_manager = self.wallet_node.manager
|
||||
self.wallet_component._running = True
|
||||
return self.wallet_component
|
||||
|
||||
conf.components_to_skip = [
|
||||
DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
|
||||
PEER_PROTOCOL_SERVER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
]
|
||||
self.daemon = Daemon(conf, ComponentManager(
|
||||
conf, skip_components=conf.components_to_skip, wallet=wallet_maker
|
||||
))
|
||||
await self.daemon.initialize()
|
||||
|
||||
async def asyncTearDown(self):
|
||||
self.wallet_component._running = False
|
||||
await self.daemon.stop()
|
||||
|
||||
@mock.patch('time.time', mock.Mock(return_value=12345))
|
||||
def test_sync(self):
|
||||
starting_hash = '69afcd60a300f47933917d77ef011beeeb4decfafebbda91c144c84282c6814f'
|
||||
self.account.modified_on = 123.456
|
||||
self.assertEqual(self.daemon.jsonrpc_sync_hash(), starting_hash)
|
||||
self.assertEqual(self.daemon.jsonrpc_sync_apply('password')['hash'], starting_hash)
|
||||
self.assertFalse(self.account.certificates)
|
||||
|
||||
hash_w_cert = '974721f42dab42657b5911b7caf4af98ce4d3879eea6ac23d50c1d79bc5020ef'
|
||||
add_cert = (
|
||||
'czo4MTkyOjE2OjE6qs3JRvS/bhX8p1JD68sqyA2Qhx3EVTqskhqEAwtfAUfUsQqeJ1rtMdRf40vkGKnpt4NT0b'
|
||||
'XEqb5O+lba4nkLF7vZENhc2zuOrjobCPVbiVHNwOfH56Ayrh1ts5LMcnl5+Mk1BUyGCwXcqEg2KiUkd3YZpiHQ'
|
||||
'T7WfcODcU6l7IRivb8iawCebZJx9waVyQoqEDKwZUY1i5HA0VLC+s5cV7it1AWbewiyWOQtZdEPzNY44oXLJex'
|
||||
'SirElQqDqNZyl3Hjy8YqacBbSYoejIRnmXpC9y25keP6hep3f9i1K2HDNwhwns1W1vhuzuO2Gy9+a0JlVm5mwc'
|
||||
'N2pqO4tCZr6tE3aym2FaSAunOi7QYVFMI6arb9Gvn9P+T+WRiFYfzwDFVR+j5ZPmUDXxHisy5OF163jH61wbBY'
|
||||
'pPienjlVtDOxoZmA8+AwWXKRdINsRcull9pu7EVCq5yQmrmxoPbLxNh5pRGrBB0JwCCOMIf+KPwS+7Z6dDbiwO'
|
||||
'2NUpk8USJMTmXmFDCr2B0PJiG6Od2dD2oGN0F7aYZvUuKbqj8eDrJMe/zLbhq47jUjkJFCvtxUioo63ORk1pzH'
|
||||
'S0/X4/6/95PRSMaXm4DcZ9BdyxR2E/AKc8UN6AL5rrn6quXkC6R3ZhKgN3Si2S9y6EGFsL7dgzX331U08ZviLj'
|
||||
'NsrG0EKUnf+TGQ42MqnLQBOiO/ZoAwleOzNZnCYOQQ14Mm8y17xUpmdWRDiRKpAOJU22jKnxtqQ='
|
||||
)
|
||||
self.daemon.jsonrpc_sync_apply('password', data=add_cert)
|
||||
self.assertEqual(self.daemon.jsonrpc_sync_hash(), hash_w_cert)
|
||||
self.assertEqual(self.account.certificates, {'abcdefg1234:0': '---PRIVATE KEY---'})
|
||||
|
||||
# applying the same diff is idempotent
|
||||
self.daemon.jsonrpc_sync_apply('password', data=add_cert)
|
||||
self.assertEqual(self.daemon.jsonrpc_sync_hash(), hash_w_cert)
|
||||
self.assertEqual(self.account.certificates, {'abcdefg1234:0': '---PRIVATE KEY---'})
|
||||
@@ -0,0 +1,38 @@
|
||||
from integration.testcase import CommandTestCase
|
||||
|
||||
|
||||
class TransactionCommandsTestCase(CommandTestCase):
|
||||
|
||||
async def test_transaction_show(self):
|
||||
# local tx
|
||||
result = await self.out(self.daemon.jsonrpc_wallet_send(
|
||||
'5.0', await self.daemon.jsonrpc_address_unused(self.account.id)
|
||||
))
|
||||
await self.confirm_tx(result['txid'])
|
||||
tx = await self.daemon.jsonrpc_transaction_show(result['txid'])
|
||||
self.assertEqual(tx.id, result['txid'])
|
||||
|
||||
# someone's tx
|
||||
change_address = await self.blockchain.get_raw_change_address()
|
||||
sendtxid = await self.blockchain.send_to_address(change_address, 10)
|
||||
tx = await self.daemon.jsonrpc_transaction_show(sendtxid)
|
||||
self.assertEqual(tx.id, sendtxid)
|
||||
self.assertEqual(tx.height, -2)
|
||||
await self.generate(1)
|
||||
tx = await self.daemon.jsonrpc_transaction_show(sendtxid)
|
||||
self.assertEqual(tx.height, self.ledger.headers.height)
|
||||
|
||||
# inexistent
|
||||
result = await self.daemon.jsonrpc_transaction_show('0'*64)
|
||||
self.assertFalse(result['success'])
|
||||
|
||||
async def test_utxo_release(self):
|
||||
sendtxid = await self.blockchain.send_to_address(
|
||||
await self.account.receiving.get_or_create_usable_address(), 1
|
||||
)
|
||||
await self.confirm_tx(sendtxid)
|
||||
await self.assertBalance(self.account, '11.0')
|
||||
await self.ledger.reserve_outputs(await self.account.get_utxos())
|
||||
await self.assertBalance(self.account, '0.0')
|
||||
await self.daemon.jsonrpc_utxo_release()
|
||||
await self.assertBalance(self.account, '11.0')
|
||||
@@ -0,0 +1,187 @@
|
||||
import json
|
||||
import shutil
|
||||
import tempfile
|
||||
import logging
|
||||
from binascii import unhexlify
|
||||
|
||||
import lbrynet.extras.wallet
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
|
||||
from torba.testcase import IntegrationTestCase
|
||||
|
||||
import lbrynet.schema
|
||||
lbrynet.schema.BLOCKCHAIN_NAME = 'lbrycrd_regtest'
|
||||
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.Daemon import Daemon, jsonrpc_dumps_pretty
|
||||
from lbrynet.extras.wallet import LbryWalletManager
|
||||
from lbrynet.extras.daemon.Components import Component, WalletComponent
|
||||
from lbrynet.extras.daemon.Components import (
|
||||
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
)
|
||||
from lbrynet.extras.daemon.ComponentManager import ComponentManager
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.stream.reflector.server import ReflectorServer
|
||||
from lbrynet.blob_exchange.server import BlobServer
|
||||
|
||||
|
||||
class ExchangeRateManager:
|
||||
|
||||
def start(self):
|
||||
pass
|
||||
|
||||
def stop(self):
|
||||
pass
|
||||
|
||||
def convert_currency(self, from_currency, to_currency, amount):
|
||||
return amount
|
||||
|
||||
def fee_dict(self):
|
||||
return {}
|
||||
|
||||
|
||||
class ExchangeRateManagerComponent(Component):
|
||||
component_name = EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.exchange_rate_manager = ExchangeRateManager()
|
||||
|
||||
@property
|
||||
def component(self) -> ExchangeRateManager:
|
||||
return self.exchange_rate_manager
|
||||
|
||||
async def start(self):
|
||||
self.exchange_rate_manager.start()
|
||||
|
||||
async def stop(self):
|
||||
self.exchange_rate_manager.stop()
|
||||
|
||||
|
||||
class CommandTestCase(IntegrationTestCase):
|
||||
|
||||
timeout = 180
|
||||
LEDGER = lbrynet.extras.wallet
|
||||
MANAGER = LbryWalletManager
|
||||
VERBOSITY = logging.WARN
|
||||
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
|
||||
logging.getLogger('lbrynet.blob_exchange').setLevel(self.VERBOSITY)
|
||||
logging.getLogger('lbrynet.daemon').setLevel(self.VERBOSITY)
|
||||
logging.getLogger('lbrynet.stream').setLevel(self.VERBOSITY)
|
||||
|
||||
conf = Config()
|
||||
conf.data_dir = self.wallet_node.data_path
|
||||
conf.wallet_dir = self.wallet_node.data_path
|
||||
conf.download_dir = self.wallet_node.data_path
|
||||
conf.share_usage_data = False
|
||||
conf.use_upnp = False
|
||||
conf.reflect_streams = True
|
||||
conf.blockchain_name = 'lbrycrd_regtest'
|
||||
conf.lbryum_servers = [('127.0.0.1', 50001)]
|
||||
conf.reflector_servers = [('127.0.0.1', 5566)]
|
||||
conf.known_dht_nodes = []
|
||||
|
||||
await self.account.ensure_address_gap()
|
||||
address = (await self.account.receiving.get_addresses(limit=1, only_usable=True))[0]
|
||||
sendtxid = await self.blockchain.send_to_address(address, 10)
|
||||
await self.confirm_tx(sendtxid)
|
||||
await self.generate(5)
|
||||
|
||||
def wallet_maker(component_manager):
|
||||
self.wallet_component = WalletComponent(component_manager)
|
||||
self.wallet_component.wallet_manager = self.manager
|
||||
self.wallet_component._running = True
|
||||
return self.wallet_component
|
||||
|
||||
conf.components_to_skip = [
|
||||
DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
|
||||
PEER_PROTOCOL_SERVER_COMPONENT
|
||||
]
|
||||
self.daemon = Daemon(conf, ComponentManager(
|
||||
conf, skip_components=conf.components_to_skip, wallet=wallet_maker,
|
||||
exchange_rate_manager=ExchangeRateManagerComponent
|
||||
))
|
||||
await self.daemon.initialize()
|
||||
self.manager.old_db = self.daemon.storage
|
||||
|
||||
server_tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, server_tmp_dir)
|
||||
self.server_storage = SQLiteStorage(Config(), ':memory:')
|
||||
await self.server_storage.open()
|
||||
self.server_blob_manager = BlobFileManager(self.loop, server_tmp_dir, self.server_storage)
|
||||
self.server = BlobServer(self.loop, self.server_blob_manager, 'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP')
|
||||
self.server.start_server(5567, '127.0.0.1')
|
||||
await self.server.started_listening.wait()
|
||||
|
||||
self.reflector = ReflectorServer(self.server_blob_manager)
|
||||
self.reflector.start_server(5566, '127.0.0.1')
|
||||
await self.reflector.started_listening.wait()
|
||||
self.addCleanup(self.reflector.stop_server)
|
||||
|
||||
async def asyncTearDown(self):
|
||||
await super().asyncTearDown()
|
||||
self.wallet_component._running = False
|
||||
await self.daemon.stop()
|
||||
|
||||
async def confirm_tx(self, txid):
|
||||
""" Wait for tx to be in mempool, then generate a block, wait for tx to be in a block. """
|
||||
await self.on_transaction_id(txid)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_id(txid)
|
||||
|
||||
async def on_transaction_dict(self, tx):
|
||||
await self.ledger.wait(
|
||||
self.ledger.transaction_class(unhexlify(tx['hex']))
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_all_addresses(tx):
|
||||
addresses = set()
|
||||
for txi in tx['inputs']:
|
||||
addresses.add(txi['address'])
|
||||
for txo in tx['outputs']:
|
||||
addresses.add(txo['address'])
|
||||
return list(addresses)
|
||||
|
||||
async def generate(self, blocks):
|
||||
""" Ask lbrycrd to generate some blocks and wait until ledger has them. """
|
||||
await self.blockchain.generate(blocks)
|
||||
await self.ledger.on_header.where(self.blockchain.is_expected_block)
|
||||
|
||||
async def out(self, awaitable):
|
||||
""" Converts Daemon API call results (dictionary)
|
||||
to JSON and then back to a dictionary. """
|
||||
return json.loads(jsonrpc_dumps_pretty(await awaitable, ledger=self.ledger))['result']
|
||||
|
||||
async def make_claim(self, name='hovercraft', amount='1.0', data=b'hi!',
|
||||
channel_name=None, confirm=True, account_id=None, fee=None):
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(data)
|
||||
file.flush()
|
||||
claim = await self.out(self.daemon.jsonrpc_publish(
|
||||
name, amount, file_path=file.name, channel_name=channel_name, account_id=account_id,
|
||||
fee=fee
|
||||
))
|
||||
self.assertTrue(claim['success'])
|
||||
if confirm:
|
||||
await self.on_transaction_dict(claim['tx'])
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(claim['tx'])
|
||||
return claim
|
||||
|
||||
async def make_channel(self, name='@arena', amount='1.0', confirm=True, account_id=None):
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new(name, amount, account_id))
|
||||
self.assertTrue(channel['success'])
|
||||
if confirm:
|
||||
await self.on_transaction_dict(channel['tx'])
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(channel['tx'])
|
||||
return channel
|
||||
|
||||
async def resolve(self, uri):
|
||||
return await self.out(self.daemon.jsonrpc_resolve(uri))
|
||||
Whitespace-only changes.
@@ -1,804 +0,0 @@
|
||||
import json
|
||||
import tempfile
|
||||
import logging
|
||||
from binascii import unhexlify
|
||||
|
||||
from lbrynet.extras.wallet.transaction import Transaction
|
||||
from lbrynet.error import InsufficientFundsError
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
|
||||
from torba.testcase import IntegrationTestCase
|
||||
|
||||
import lbrynet.schema
|
||||
lbrynet.schema.BLOCKCHAIN_NAME = 'lbrycrd_regtest'
|
||||
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.Daemon import Daemon, jsonrpc_dumps_pretty
|
||||
from lbrynet.extras.wallet import LbryWalletManager
|
||||
from lbrynet.extras.daemon.Components import WalletComponent
|
||||
from lbrynet.extras.daemon.Components import (
|
||||
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
)
|
||||
from lbrynet.extras.daemon.ComponentManager import ComponentManager
|
||||
|
||||
|
||||
class CommandTestCase(IntegrationTestCase):
|
||||
|
||||
timeout = 180
|
||||
MANAGER = LbryWalletManager
|
||||
VERBOSITY = logging.WARN
|
||||
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
|
||||
logging.getLogger('lbrynet.blob_exchange').setLevel(self.VERBOSITY)
|
||||
logging.getLogger('lbrynet.daemon').setLevel(self.VERBOSITY)
|
||||
|
||||
conf = Config()
|
||||
conf.data_dir = self.wallet_node.data_path
|
||||
conf.wallet_dir = self.wallet_node.data_path
|
||||
conf.download_dir = self.wallet_node.data_path
|
||||
conf.share_usage_data = False
|
||||
conf.use_upnp = False
|
||||
conf.reflect_uploads = False
|
||||
conf.blockchain_name = 'lbrycrd_regtest'
|
||||
conf.lbryum_servers = [('localhost', 50001)]
|
||||
conf.known_dht_nodes = []
|
||||
|
||||
await self.account.ensure_address_gap()
|
||||
address = (await self.account.receiving.get_addresses(limit=1, only_usable=True))[0]
|
||||
sendtxid = await self.blockchain.send_to_address(address, 10)
|
||||
await self.confirm_tx(sendtxid)
|
||||
await self.generate(5)
|
||||
|
||||
def wallet_maker(component_manager):
|
||||
self.wallet_component = WalletComponent(component_manager)
|
||||
self.wallet_component.wallet_manager = self.manager
|
||||
self.wallet_component._running = True
|
||||
return self.wallet_component
|
||||
|
||||
conf.components_to_skip = [
|
||||
DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
|
||||
PEER_PROTOCOL_SERVER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
]
|
||||
self.daemon = Daemon(conf, ComponentManager(
|
||||
conf, skip_components=conf.components_to_skip, wallet=wallet_maker
|
||||
))
|
||||
await self.daemon.initialize()
|
||||
self.manager.old_db = self.daemon.storage
|
||||
|
||||
async def asyncTearDown(self):
|
||||
await super().asyncTearDown()
|
||||
self.wallet_component._running = False
|
||||
await self.daemon.stop()
|
||||
|
||||
async def confirm_tx(self, txid):
|
||||
""" Wait for tx to be in mempool, then generate a block, wait for tx to be in a block. """
|
||||
await self.on_transaction_id(txid)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_id(txid)
|
||||
|
||||
async def on_transaction_dict(self, tx):
|
||||
await self.ledger.wait(
|
||||
self.ledger.transaction_class(unhexlify(tx['hex']))
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_all_addresses(tx):
|
||||
addresses = set()
|
||||
for txi in tx['inputs']:
|
||||
addresses.add(txi['address'])
|
||||
for txo in tx['outputs']:
|
||||
addresses.add(txo['address'])
|
||||
return list(addresses)
|
||||
|
||||
async def generate(self, blocks):
|
||||
""" Ask lbrycrd to generate some blocks and wait until ledger has them. """
|
||||
await self.blockchain.generate(blocks)
|
||||
await self.ledger.on_header.where(self.blockchain.is_expected_block)
|
||||
|
||||
async def out(self, awaitable):
|
||||
""" Converts Daemon API call results (dictionary)
|
||||
to JSON and then back to a dictionary. """
|
||||
return json.loads(jsonrpc_dumps_pretty(await awaitable, ledger=self.ledger))['result']
|
||||
|
||||
|
||||
class EpicAdventuresOfChris45(CommandTestCase):
|
||||
|
||||
VERBOSITY = logging.WARN
|
||||
|
||||
async def test_no_this_is_not_a_test_its_an_adventure(self):
|
||||
# Chris45 is an avid user of LBRY and this is his story. It's fact and fiction
|
||||
# and everything in between; it's also the setting of some record setting
|
||||
# integration tests.
|
||||
|
||||
# Chris45 starts everyday by checking his balance.
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
self.assertEqual(result, '10.0')
|
||||
# "10 LBC, yippy! I can do a lot with that.", he thinks to himself,
|
||||
# enthusiastically. But he is hungry so he goes into the kitchen
|
||||
# to make himself a spamdwich.
|
||||
|
||||
# While making the spamdwich he wonders... has anyone on LBRY
|
||||
# registered the @spam channel yet? "I should do that!" he
|
||||
# exclaims and goes back to his computer to do just that!
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@spam', "1.0"))
|
||||
self.assertTrue(channel['success'])
|
||||
await self.confirm_tx(channel['tx']['txid'])
|
||||
|
||||
# Do we have it locally?
|
||||
channels = await self.out(self.daemon.jsonrpc_channel_list())
|
||||
self.assertEqual(len(channels), 1)
|
||||
self.assertEqual(channels[0]['name'], '@spam')
|
||||
|
||||
# As the new channel claim travels through the intertubes and makes its
|
||||
# way into the mempool and then a block and then into the claimtrie,
|
||||
# Chris doesn't sit idly by: he checks his balance!
|
||||
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
self.assertEqual(result, '8.989893')
|
||||
|
||||
# He waits for 6 more blocks (confirmations) to make sure the balance has been settled.
|
||||
await self.generate(6)
|
||||
result = await self.daemon.jsonrpc_account_balance(confirmations=6)
|
||||
self.assertEqual(result, '8.989893')
|
||||
|
||||
# And is the channel resolvable and empty?
|
||||
response = await self.out(self.daemon.jsonrpc_resolve(uri='lbry://@spam'))
|
||||
self.assertIn('lbry://@spam', response)
|
||||
self.assertIn('certificate', response['lbry://@spam'])
|
||||
|
||||
# "What goes well with spam?" ponders Chris...
|
||||
# "A hovercraft with eels!" he exclaims.
|
||||
# "That's what goes great with spam!" he further confirms.
|
||||
|
||||
# And so, many hours later, Chris is finished writing his epic story
|
||||
# about eels driving a hovercraft across the wetlands while eating spam
|
||||
# and decides it's time to publish it to the @spam channel.
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'blah blah blah...')
|
||||
file.write(b'[insert long story about eels driving hovercraft]')
|
||||
file.write(b'yada yada yada!')
|
||||
file.write(b'the end')
|
||||
file.flush()
|
||||
claim1 = await self.out(self.daemon.jsonrpc_publish(
|
||||
'hovercraft', '1.0', file_path=file.name, channel_id=channel['claim_id']
|
||||
))
|
||||
self.assertTrue(claim1['success'])
|
||||
await self.confirm_tx(claim1['tx']['txid'])
|
||||
|
||||
# He quickly checks the unconfirmed balance to make sure everything looks
|
||||
# correct.
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
self.assertEqual(result, '7.969786')
|
||||
|
||||
# Also checks that his new story can be found on the blockchain before
|
||||
# giving the link to all his friends.
|
||||
response = await self.out(self.daemon.jsonrpc_resolve(uri='lbry://@spam/hovercraft'))
|
||||
self.assertIn('lbry://@spam/hovercraft', response)
|
||||
self.assertIn('claim', response['lbry://@spam/hovercraft'])
|
||||
|
||||
# He goes to tell everyone about it and in the meantime 5 blocks are confirmed.
|
||||
await self.generate(5)
|
||||
# When he comes back he verifies the confirmed balance.
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
self.assertEqual(result, '7.969786')
|
||||
|
||||
# As people start reading his story they discover some typos and notify
|
||||
# Chris who explains in despair "Oh! Noooooos!" but then remembers
|
||||
# "No big deal! I can update my claim." And so he updates his claim.
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'blah blah blah...')
|
||||
file.write(b'[typo fixing sounds being made]')
|
||||
file.write(b'yada yada yada!')
|
||||
file.flush()
|
||||
claim2 = await self.out(self.daemon.jsonrpc_publish(
|
||||
'hovercraft', '1.0', file_path=file.name, channel_name='@spam'
|
||||
))
|
||||
self.assertTrue(claim2['success'])
|
||||
self.assertEqual(claim2['claim_id'], claim1['claim_id'])
|
||||
await self.confirm_tx(claim2['tx']['txid'])
|
||||
|
||||
# After some soul searching Chris decides that his story needs more
|
||||
# heart and a better ending. He takes down the story and begins the rewrite.
|
||||
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(claim1['claim_id'], blocking=False))
|
||||
self.assertTrue(abandon['success'])
|
||||
await self.confirm_tx(abandon['tx']['txid'])
|
||||
|
||||
# And now checks that the claim doesn't resolve anymore.
|
||||
response = await self.out(self.daemon.jsonrpc_resolve(uri='lbry://@spam/hovercraft'))
|
||||
self.assertNotIn('claim', response['lbry://@spam/hovercraft'])
|
||||
|
||||
# After abandoning he just waits for his LBCs to be returned to his account
|
||||
await self.generate(5)
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
self.assertEqual(result, '8.9693585')
|
||||
|
||||
# Amidst all this Chris receives a call from his friend Ramsey
|
||||
# who says that it is of utmost urgency that Chris transfer him
|
||||
# 1 LBC to which Chris readily obliges
|
||||
ramsey_account_id = (await self.daemon.jsonrpc_account_create("Ramsey"))['id']
|
||||
ramsey_account = self.daemon.get_account_or_error(ramsey_account_id)
|
||||
ramsey_address = await self.daemon.jsonrpc_address_unused(ramsey_account_id)
|
||||
result = await self.out(self.daemon.jsonrpc_wallet_send('1.0', ramsey_address))
|
||||
self.assertIn("txid", result)
|
||||
await self.confirm_tx(result['txid'])
|
||||
|
||||
# Chris then eagerly waits for 6 confirmations to check his balance and then calls Ramsey to verify whether
|
||||
# he received it or not
|
||||
await self.generate(5)
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
# Chris' balance was correct
|
||||
self.assertEqual(result, '7.9692345')
|
||||
|
||||
# Ramsey too assured him that he had received the 1 LBC and thanks him
|
||||
result = await self.daemon.jsonrpc_account_balance(ramsey_account_id)
|
||||
self.assertEqual(result, '1.0')
|
||||
|
||||
# After Chris is done with all the "helping other people" stuff he decides that it's time to
|
||||
# write a new story and publish it to lbry. All he needed was a fresh start and he came up with:
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'Amazingly Original First Line')
|
||||
file.write(b'Super plot for the grand novel')
|
||||
file.write(b'Totally un-cliched ending')
|
||||
file.write(b'**Audience Gasps**')
|
||||
file.flush()
|
||||
claim3 = await self.out(self.daemon.jsonrpc_publish(
|
||||
'fresh-start', '1.0', file_path=file.name, channel_name='@spam'
|
||||
))
|
||||
self.assertTrue(claim3['success'])
|
||||
await self.confirm_tx(claim3['tx']['txid'])
|
||||
|
||||
await self.generate(5)
|
||||
|
||||
# He gives the link of his story to all his friends and hopes that this is the much needed break for him
|
||||
uri = 'lbry://@spam/fresh-start'
|
||||
|
||||
# And voila, and bravo and encore! His Best Friend Ramsey read the story and immediately knew this was a hit
|
||||
# Now to keep this claim winning on the lbry blockchain he immediately supports the claim
|
||||
tx = await self.out(self.daemon.jsonrpc_claim_new_support(
|
||||
'fresh-start', claim3['claim_id'], '0.2', account_id=ramsey_account_id
|
||||
))
|
||||
await self.confirm_tx(tx['txid'])
|
||||
|
||||
# And check if his support showed up
|
||||
resolve_result = await self.out(self.daemon.jsonrpc_resolve(uri=uri))
|
||||
# It obviously did! Because, blockchain baby \O/
|
||||
self.assertEqual(resolve_result[uri]['claim']['amount'], '1.0')
|
||||
self.assertEqual(resolve_result[uri]['claim']['effective_amount'], '1.2')
|
||||
self.assertEqual(resolve_result[uri]['claim']['supports'][0]['amount'], '0.2')
|
||||
self.assertEqual(resolve_result[uri]['claim']['supports'][0]['txid'], tx['txid'])
|
||||
await self.generate(5)
|
||||
|
||||
# Now he also wanted to support the original creator of the Award Winning Novel
|
||||
# So he quickly decides to send a tip to him
|
||||
tx = await self.out(
|
||||
self.daemon.jsonrpc_claim_tip(claim3['claim_id'], '0.3', account_id=ramsey_account_id))
|
||||
await self.confirm_tx(tx['txid'])
|
||||
|
||||
# And again checks if it went to the just right place
|
||||
resolve_result = await self.out(self.daemon.jsonrpc_resolve(uri=uri))
|
||||
# Which it obviously did. Because....?????
|
||||
self.assertEqual(resolve_result[uri]['claim']['supports'][1]['amount'], '0.3')
|
||||
self.assertEqual(resolve_result[uri]['claim']['supports'][1]['txid'], tx['txid'])
|
||||
await self.generate(5)
|
||||
|
||||
# Seeing the ravishing success of his novel Chris adds support to his claim too
|
||||
tx = await self.out(self.daemon.jsonrpc_claim_new_support('fresh-start', claim3['claim_id'], '0.4'))
|
||||
await self.confirm_tx(tx['txid'])
|
||||
|
||||
# And check if his support showed up
|
||||
resolve_result = await self.out(self.daemon.jsonrpc_resolve(uri=uri))
|
||||
# It did!
|
||||
self.assertEqual(resolve_result[uri]['claim']['supports'][2]['amount'], '0.4')
|
||||
self.assertEqual(resolve_result[uri]['claim']['supports'][2]['txid'], tx['txid'])
|
||||
await self.generate(5)
|
||||
|
||||
# Now Ramsey who is a singer by profession, is preparing for his new "gig". He has everything in place for that
|
||||
# the instruments, the theatre, the ads, everything, EXCEPT lyrics!! He panicked.. But then he remembered
|
||||
# something, so he un-panicked. He quickly calls up his best bud Chris and requests him to write hit lyrics for
|
||||
# his song, seeing as his novel had smashed all the records, he was the perfect candidate!
|
||||
# .......
|
||||
# Chris agrees.. 17 hours 43 minutes and 14 seconds later, he makes his publish
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'The Whale amd The Bookmark')
|
||||
file.write(b'I know right? Totally a hit song')
|
||||
file.write(b'That\'s what goes around for songs these days anyways')
|
||||
file.flush()
|
||||
claim4 = await self.out(self.daemon.jsonrpc_publish(
|
||||
'hit-song', '1.0', file_path=file.name, channel_id=channel['claim_id']
|
||||
))
|
||||
self.assertTrue(claim4['success'])
|
||||
await self.confirm_tx(claim4['tx']['txid'])
|
||||
|
||||
await self.generate(5)
|
||||
|
||||
# He sends the link to Ramsey, all happy and proud
|
||||
uri = 'lbry://@spam/hit-song'
|
||||
|
||||
# But sadly Ramsey wasn't so pleased. It was hard for him to tell Chris...
|
||||
# Chris, though a bit heartbroken, abandoned the claim for now, but instantly started working on new hit lyrics
|
||||
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=claim4['tx']['txid'], nout=0, blocking=False))
|
||||
self.assertTrue(abandon['success'])
|
||||
await self.confirm_tx(abandon['tx']['txid'])
|
||||
|
||||
# He them checks that the claim doesn't resolve anymore.
|
||||
response = await self.out(self.daemon.jsonrpc_resolve(uri=uri))
|
||||
self.assertNotIn('claim', response[uri])
|
||||
|
||||
|
||||
class AccountManagement(CommandTestCase):
|
||||
|
||||
VERBOSE = False
|
||||
|
||||
async def test_performing_account_management_commands(self):
|
||||
# check initial account
|
||||
response = await self.daemon.jsonrpc_account_list()
|
||||
self.assertEqual(len(response['lbc_regtest']), 1)
|
||||
|
||||
# change account name and gap
|
||||
account_id = response['lbc_regtest'][0]['id']
|
||||
self.daemon.jsonrpc_account_set(
|
||||
account_id=account_id, new_name='test account',
|
||||
receiving_gap=95, receiving_max_uses=96,
|
||||
change_gap=97, change_max_uses=98
|
||||
)
|
||||
response = (await self.daemon.jsonrpc_account_list())['lbc_regtest'][0]
|
||||
self.assertEqual(response['name'], 'test account')
|
||||
self.assertEqual(
|
||||
response['address_generator']['receiving'],
|
||||
{'gap': 95, 'maximum_uses_per_address': 96}
|
||||
)
|
||||
self.assertEqual(
|
||||
response['address_generator']['change'],
|
||||
{'gap': 97, 'maximum_uses_per_address': 98}
|
||||
)
|
||||
|
||||
# create another account
|
||||
await self.daemon.jsonrpc_account_create('second account')
|
||||
response = await self.daemon.jsonrpc_account_list()
|
||||
self.assertEqual(len(response['lbc_regtest']), 2)
|
||||
self.assertEqual(response['lbc_regtest'][1]['name'], 'second account')
|
||||
account_id2 = response['lbc_regtest'][1]['id']
|
||||
|
||||
# make new account the default
|
||||
self.daemon.jsonrpc_account_set(account_id=account_id2, default=True)
|
||||
response = await self.daemon.jsonrpc_account_list(show_seed=True)
|
||||
self.assertEqual(response['lbc_regtest'][0]['name'], 'second account')
|
||||
|
||||
account_seed = response['lbc_regtest'][1]['seed']
|
||||
|
||||
# remove account
|
||||
self.daemon.jsonrpc_account_remove(response['lbc_regtest'][1]['id'])
|
||||
response = await self.daemon.jsonrpc_account_list()
|
||||
self.assertEqual(len(response['lbc_regtest']), 1)
|
||||
|
||||
# add account
|
||||
await self.daemon.jsonrpc_account_add('recreated account', seed=account_seed)
|
||||
response = await self.daemon.jsonrpc_account_list()
|
||||
self.assertEqual(len(response['lbc_regtest']), 2)
|
||||
self.assertEqual(response['lbc_regtest'][1]['name'], 'recreated account')
|
||||
|
||||
# list specific account
|
||||
response = await self.daemon.jsonrpc_account_list(account_id, include_claims=True)
|
||||
self.assertEqual(response['name'], 'recreated account')
|
||||
|
||||
|
||||
class ClaimManagement(CommandTestCase):
|
||||
|
||||
VERBOSITY = logging.WARN
|
||||
|
||||
async def make_claim(self, name='hovercraft', amount='1.0', data=b'hi!', channel_name=None, confirm=True):
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(data)
|
||||
file.flush()
|
||||
claim = await self.out(self.daemon.jsonrpc_publish(
|
||||
name, amount, file_path=file.name, channel_name=channel_name
|
||||
))
|
||||
self.assertTrue(claim['success'])
|
||||
if confirm:
|
||||
await self.on_transaction_dict(claim['tx'])
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(claim['tx'])
|
||||
return claim
|
||||
|
||||
async def craft_claim(self, name, amount_dewies, claim_dict, address):
|
||||
# FIXME: this is here mostly because publish has defensive code for situations that happens accidentally
|
||||
# However, it still happens... So, let's reproduce them.
|
||||
claim = ClaimDict.load_dict(claim_dict)
|
||||
address = address or (await self.account.receiving.get_addresses(limit=1, only_usable=True))[0]
|
||||
tx = await Transaction.claim(name, claim, amount_dewies, address, [self.account], self.account)
|
||||
await self.broadcast(tx)
|
||||
await self.ledger.wait(tx)
|
||||
await self.generate(1)
|
||||
await self.ledger.wait(tx)
|
||||
return tx
|
||||
|
||||
async def test_create_update_and_abandon_claim(self):
|
||||
self.assertEqual('10.0', await self.daemon.jsonrpc_account_balance())
|
||||
|
||||
claim = await self.make_claim(amount='2.5') # creates new claim
|
||||
txs = await self.out(self.daemon.jsonrpc_transaction_list())
|
||||
self.assertEqual(len(txs[0]['claim_info']), 1)
|
||||
self.assertEqual(txs[0]['confirmations'], 1)
|
||||
self.assertEqual(txs[0]['claim_info'][0]['balance_delta'], '-2.5')
|
||||
self.assertEqual(txs[0]['claim_info'][0]['claim_id'], claim['claim_id'])
|
||||
self.assertEqual(txs[0]['value'], '0.0')
|
||||
self.assertEqual(txs[0]['fee'], '-0.020107')
|
||||
self.assertEqual('7.479893', await self.daemon.jsonrpc_account_balance())
|
||||
|
||||
await self.make_claim(amount='1.0') # updates previous claim
|
||||
txs = await self.out(self.daemon.jsonrpc_transaction_list())
|
||||
self.assertEqual(len(txs[0]['update_info']), 1)
|
||||
self.assertEqual(txs[0]['update_info'][0]['balance_delta'], '1.5')
|
||||
self.assertEqual(txs[0]['update_info'][0]['claim_id'], claim['claim_id'])
|
||||
self.assertEqual(txs[0]['value'], '0.0')
|
||||
self.assertEqual(txs[0]['fee'], '-0.0001985')
|
||||
self.assertEqual('8.9796945', await self.daemon.jsonrpc_account_balance())
|
||||
|
||||
await self.out(self.daemon.jsonrpc_claim_abandon(claim['claim_id']))
|
||||
txs = await self.out(self.daemon.jsonrpc_transaction_list())
|
||||
self.assertEqual(len(txs[0]['abandon_info']), 1)
|
||||
self.assertEqual(txs[0]['abandon_info'][0]['balance_delta'], '1.0')
|
||||
self.assertEqual(txs[0]['abandon_info'][0]['claim_id'], claim['claim_id'])
|
||||
self.assertEqual(txs[0]['value'], '0.0')
|
||||
self.assertEqual(txs[0]['fee'], '-0.000107')
|
||||
self.assertEqual('9.9795875', await self.daemon.jsonrpc_account_balance())
|
||||
|
||||
async def test_update_claim_holding_address(self):
|
||||
other_account_id = (await self.daemon.jsonrpc_account_create('second account'))['id']
|
||||
other_account = self.daemon.get_account_or_error(other_account_id)
|
||||
other_address = await other_account.receiving.get_or_create_usable_address()
|
||||
|
||||
self.assertEqual('10.0', await self.daemon.jsonrpc_account_balance())
|
||||
|
||||
# create the initial name claim
|
||||
claim = await self.make_claim()
|
||||
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list_mine()), 1)
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list_mine(account_id=other_account_id)), 0)
|
||||
tx = await self.daemon.jsonrpc_claim_send_to_address(
|
||||
claim['claim_id'], other_address
|
||||
)
|
||||
await self.ledger.wait(tx)
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list_mine()), 0)
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list_mine(account_id=other_account_id)), 1)
|
||||
|
||||
async def test_publishing_checks_all_accounts_for_certificate(self):
|
||||
account1_id, account1 = self.account.id, self.account
|
||||
new_account = await self.daemon.jsonrpc_account_create('second account')
|
||||
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
|
||||
|
||||
spam_channel = await self.out(self.daemon.jsonrpc_channel_new('@spam', '1.0'))
|
||||
self.assertTrue(spam_channel['success'])
|
||||
await self.confirm_tx(spam_channel['tx']['txid'])
|
||||
|
||||
self.assertEqual('8.989893', await self.daemon.jsonrpc_account_balance())
|
||||
|
||||
result = await self.out(self.daemon.jsonrpc_wallet_send(
|
||||
'5.0', await self.daemon.jsonrpc_address_unused(account2_id)
|
||||
))
|
||||
await self.confirm_tx(result['txid'])
|
||||
|
||||
self.assertEqual('3.989769', await self.daemon.jsonrpc_account_balance())
|
||||
self.assertEqual('5.0', await self.daemon.jsonrpc_account_balance(account2_id))
|
||||
|
||||
baz_channel = await self.out(self.daemon.jsonrpc_channel_new('@baz', '1.0', account2_id))
|
||||
self.assertTrue(baz_channel['success'])
|
||||
await self.confirm_tx(baz_channel['tx']['txid'])
|
||||
|
||||
channels = await self.out(self.daemon.jsonrpc_channel_list(account1_id))
|
||||
self.assertEqual(len(channels), 1)
|
||||
self.assertEqual(channels[0]['name'], '@spam')
|
||||
self.assertEqual(channels, await self.out(self.daemon.jsonrpc_channel_list()))
|
||||
|
||||
channels = await self.out(self.daemon.jsonrpc_channel_list(account2_id))
|
||||
self.assertEqual(len(channels), 1)
|
||||
self.assertEqual(channels[0]['name'], '@baz')
|
||||
|
||||
# defaults to using all accounts to lookup channel
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'hi!')
|
||||
file.flush()
|
||||
claim1 = await self.out(self.daemon.jsonrpc_publish(
|
||||
'hovercraft', '1.0', file_path=file.name, channel_name='@baz'
|
||||
))
|
||||
self.assertTrue(claim1['success'])
|
||||
await self.confirm_tx(claim1['tx']['txid'])
|
||||
|
||||
# uses only the specific accounts which contains the channel
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'hi!')
|
||||
file.flush()
|
||||
claim1 = await self.out(self.daemon.jsonrpc_publish(
|
||||
'hovercraft', '1.0', file_path=file.name,
|
||||
channel_name='@baz', channel_account_id=[account2_id]
|
||||
))
|
||||
self.assertTrue(claim1['success'])
|
||||
await self.confirm_tx(claim1['tx']['txid'])
|
||||
|
||||
# fails when specifying account which does not contain channel
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'hi!')
|
||||
file.flush()
|
||||
with self.assertRaisesRegex(ValueError, "Couldn't find channel with name '@baz'."):
|
||||
await self.out(self.daemon.jsonrpc_publish(
|
||||
'hovercraft', '1.0', file_path=file.name,
|
||||
channel_name='@baz', channel_account_id=[account1_id]
|
||||
))
|
||||
|
||||
async def test_updating_claim_includes_claim_value_in_balance_check(self):
|
||||
self.assertEqual('10.0', await self.daemon.jsonrpc_account_balance())
|
||||
|
||||
await self.make_claim(amount='9.0')
|
||||
self.assertEqual('0.979893', await self.daemon.jsonrpc_account_balance())
|
||||
|
||||
# update the same claim
|
||||
await self.make_claim(amount='9.0')
|
||||
self.assertEqual('0.9796205', await self.daemon.jsonrpc_account_balance())
|
||||
|
||||
# update the claim a second time but use even more funds
|
||||
await self.make_claim(amount='9.97')
|
||||
self.assertEqual('0.009348', await self.daemon.jsonrpc_account_balance())
|
||||
|
||||
# fails when specifying more than available
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'hi!')
|
||||
file.flush()
|
||||
with self.assertRaisesRegex(
|
||||
InsufficientFundsError,
|
||||
"Please lower the bid value, the maximum amount"
|
||||
" you can specify for this claim is 9.979274."
|
||||
):
|
||||
await self.out(self.daemon.jsonrpc_publish(
|
||||
'hovercraft', '9.98', file_path=file.name
|
||||
))
|
||||
|
||||
async def test_abandoning_claim_at_loss(self):
|
||||
self.assertEqual('10.0', await self.daemon.jsonrpc_account_balance())
|
||||
claim = await self.make_claim(amount='0.0001')
|
||||
self.assertEqual('9.979793', await self.daemon.jsonrpc_account_balance())
|
||||
await self.out(self.daemon.jsonrpc_claim_abandon(claim['claim_id']))
|
||||
self.assertEqual('9.97968399', await self.daemon.jsonrpc_account_balance())
|
||||
|
||||
async def test_claim_show(self):
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
|
||||
self.assertTrue(channel['success'])
|
||||
await self.confirm_tx(channel['tx']['txid'])
|
||||
channel_from_claim_show = await self.out(
|
||||
self.daemon.jsonrpc_claim_show(txid=channel['tx']['txid'], nout=channel['output']['nout'])
|
||||
)
|
||||
self.assertEqual(channel_from_claim_show['value'], channel['output']['value'])
|
||||
channel_from_claim_show = await self.out(
|
||||
self.daemon.jsonrpc_claim_show(claim_id=channel['claim_id'])
|
||||
)
|
||||
self.assertEqual(channel_from_claim_show['value'], channel['output']['value'])
|
||||
|
||||
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=channel['tx']['txid'], nout=0, blocking=False))
|
||||
self.assertTrue(abandon['success'])
|
||||
await self.confirm_tx(abandon['tx']['txid'])
|
||||
not_a_claim = await self.out(
|
||||
self.daemon.jsonrpc_claim_show(txid=abandon['tx']['txid'], nout=0)
|
||||
)
|
||||
self.assertEqual(not_a_claim, 'claim not found')
|
||||
|
||||
async def test_claim_list(self):
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
|
||||
self.assertTrue(channel['success'])
|
||||
await self.confirm_tx(channel['tx']['txid'])
|
||||
claim = await self.make_claim(amount='0.0001', name='on-channel-claim', channel_name='@abc')
|
||||
self.assertTrue(claim['success'])
|
||||
unsigned_claim = await self.make_claim(amount='0.0001', name='unsigned')
|
||||
self.assertTrue(claim['success'])
|
||||
|
||||
channel_from_claim_list = await self.out(self.daemon.jsonrpc_claim_list('@abc'))
|
||||
self.assertEqual(channel_from_claim_list['claims'][0]['value'], channel['output']['value'])
|
||||
signed_claim_from_claim_list = await self.out(self.daemon.jsonrpc_claim_list('on-channel-claim'))
|
||||
self.assertEqual(signed_claim_from_claim_list['claims'][0]['value'], claim['output']['value'])
|
||||
unsigned_claim_from_claim_list = await self.out(self.daemon.jsonrpc_claim_list('unsigned'))
|
||||
self.assertEqual(unsigned_claim_from_claim_list['claims'][0]['value'], unsigned_claim['output']['value'])
|
||||
|
||||
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=channel['tx']['txid'], nout=0, blocking=False))
|
||||
self.assertTrue(abandon['success'])
|
||||
await self.confirm_tx(abandon['tx']['txid'])
|
||||
|
||||
empty = await self.out(self.daemon.jsonrpc_claim_list('@abc'))
|
||||
self.assertEqual(len(empty['claims']), 0)
|
||||
|
||||
async def test_abandoned_channel_with_signed_claims(self):
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
|
||||
self.assertTrue(channel['success'])
|
||||
await self.confirm_tx(channel['tx']['txid'])
|
||||
claim = await self.make_claim(amount='0.0001', name='on-channel-claim', channel_name='@abc')
|
||||
self.assertTrue(claim['success'])
|
||||
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=channel['tx']['txid'], nout=0, blocking=False))
|
||||
self.assertTrue(abandon['success'])
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
|
||||
self.assertTrue(channel['success'])
|
||||
await self.confirm_tx(channel['tx']['txid'])
|
||||
|
||||
# Original channel doesnt exists anymore, so the signature is invalid. For invalid signatures, resolution is
|
||||
# only possible outside a channel
|
||||
response = await self.out(self.daemon.jsonrpc_resolve(uri='lbry://@abc/on-channel-claim'))
|
||||
self.assertNotIn('claim', response['lbry://@abc/on-channel-claim'])
|
||||
response = await self.out(self.daemon.jsonrpc_resolve(uri='lbry://on-channel-claim'))
|
||||
self.assertIn('claim', response['lbry://on-channel-claim'])
|
||||
self.assertFalse(response['lbry://on-channel-claim']['claim']['signature_is_valid'])
|
||||
direct_uri = 'lbry://on-channel-claim#' + claim['claim_id']
|
||||
response = await self.out(self.daemon.jsonrpc_resolve(uri=direct_uri))
|
||||
self.assertIn('claim', response[direct_uri])
|
||||
self.assertFalse(response[direct_uri]['claim']['signature_is_valid'])
|
||||
|
||||
uri = 'lbry://@abc/on-channel-claim'
|
||||
# now, claim something on this channel (it will update the invalid claim, but we save and forcefully restore)
|
||||
original_claim = await self.make_claim(amount='0.00000001', name='on-channel-claim', channel_name='@abc')
|
||||
self.assertTrue(original_claim['success'])
|
||||
# resolves normally
|
||||
response = await self.out(self.daemon.jsonrpc_resolve(uri=uri))
|
||||
self.assertIn('claim', response[uri])
|
||||
self.assertTrue(response[uri]['claim']['signature_is_valid'])
|
||||
|
||||
# tamper it, invalidating the signature
|
||||
value = response[uri]['claim']['value'].copy()
|
||||
value['stream']['metadata']['author'] = 'some troll'
|
||||
address = response[uri]['claim']['address']
|
||||
await self.craft_claim('on-channel-claim', 1, value, address)
|
||||
# it resolves to the now only valid claim under the channel, ignoring the fake one
|
||||
response = await self.out(self.daemon.jsonrpc_resolve(uri=uri))
|
||||
self.assertIn('claim', response[uri])
|
||||
self.assertTrue(response[uri]['claim']['signature_is_valid'])
|
||||
|
||||
# ooops! claimed a valid conflict! (this happens on the wild, mostly by accident or race condition)
|
||||
await self.craft_claim('on-channel-claim', 1, response[uri]['claim']['value'], address)
|
||||
|
||||
# it still resolves! but to the older claim
|
||||
response = await self.out(self.daemon.jsonrpc_resolve(uri=uri))
|
||||
self.assertIn('claim', response[uri])
|
||||
self.assertTrue(response[uri]['claim']['signature_is_valid'])
|
||||
self.assertEqual(response[uri]['claim']['txid'], original_claim['tx']['txid'])
|
||||
|
||||
async def test_claim_list_by_channel(self):
|
||||
self.maxDiff = None
|
||||
tx = await self.daemon.jsonrpc_account_fund(None, None, '0.001', outputs=100, broadcast=True)
|
||||
await self.ledger.wait(tx)
|
||||
await self.generate(1)
|
||||
await self.ledger.wait(tx)
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "0.0001"))
|
||||
self.assertTrue(channel['success'])
|
||||
await self.confirm_tx(channel['tx']['txid'])
|
||||
|
||||
# 4 claims per block, 3 blocks. Sorted by height (descending) then claim_id (ascending).
|
||||
claims = []
|
||||
for j in range(3):
|
||||
same_height_claims = []
|
||||
for k in range(3):
|
||||
claim = await self.make_claim(amount='0.000001', name=f'c{j}-{k}', channel_name='@abc', confirm=False)
|
||||
self.assertTrue(claim['success'])
|
||||
same_height_claims.append(claim['claim_id'])
|
||||
await self.on_transaction_dict(claim['tx'])
|
||||
claim = await self.make_claim(amount='0.000001', name=f'c{j}-4', channel_name='@abc', confirm=True)
|
||||
self.assertTrue(claim['success'])
|
||||
same_height_claims.append(claim['claim_id'])
|
||||
same_height_claims.sort(key=lambda x: int(x, 16))
|
||||
claims = same_height_claims + claims
|
||||
|
||||
page = await self.out(self.daemon.jsonrpc_claim_list_by_channel(1, page_size=20, uri='@abc'))
|
||||
page_claim_ids = [item['claim_id'] for item in page['@abc']['claims_in_channel']]
|
||||
self.assertEqual(page_claim_ids, claims)
|
||||
page = await self.out(self.daemon.jsonrpc_claim_list_by_channel(1, page_size=6, uri='@abc'))
|
||||
page_claim_ids = [item['claim_id'] for item in page['@abc']['claims_in_channel']]
|
||||
self.assertEqual(page_claim_ids, claims[:6])
|
||||
out_of_bounds = await self.out(self.daemon.jsonrpc_claim_list_by_channel(2, page_size=20, uri='@abc'))
|
||||
self.assertEqual(out_of_bounds['error'], 'claim 20 greater than max 12')
|
||||
|
||||
async def test_regular_supports_and_tip_supports(self):
|
||||
# account2 will be used to send tips and supports to account1
|
||||
account2_id = (await self.daemon.jsonrpc_account_create('second account'))['id']
|
||||
|
||||
# send account2 5 LBC out of the 10 LBC in account1
|
||||
result = await self.out(self.daemon.jsonrpc_wallet_send(
|
||||
'5.0', await self.daemon.jsonrpc_address_unused(account2_id)
|
||||
))
|
||||
await self.confirm_tx(result['txid'])
|
||||
|
||||
# account1 and account2 balances:
|
||||
self.assertEqual('4.999876', await self.daemon.jsonrpc_account_balance())
|
||||
self.assertEqual('5.0', await self.daemon.jsonrpc_account_balance(account2_id))
|
||||
|
||||
# create the claim we'll be tipping and supporting
|
||||
claim = await self.make_claim()
|
||||
|
||||
# account1 and account2 balances:
|
||||
self.assertEqual('3.979769', await self.daemon.jsonrpc_account_balance())
|
||||
self.assertEqual('5.0', await self.daemon.jsonrpc_account_balance(account2_id))
|
||||
|
||||
# send a tip to the claim using account2
|
||||
tip = await self.out(
|
||||
self.daemon.jsonrpc_claim_tip(claim['claim_id'], '1.0', account2_id)
|
||||
)
|
||||
await self.on_transaction_dict(tip)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(tip)
|
||||
|
||||
# tips don't affect balance so account1 balance is same but account2 balance went down
|
||||
self.assertEqual('3.979769', await self.daemon.jsonrpc_account_balance())
|
||||
self.assertEqual('3.9998585', await self.daemon.jsonrpc_account_balance(account2_id))
|
||||
|
||||
# verify that the incoming tip is marked correctly as is_tip=True in account1
|
||||
txs = await self.out(self.daemon.jsonrpc_transaction_list())
|
||||
self.assertEqual(len(txs[0]['support_info']), 1)
|
||||
self.assertEqual(txs[0]['support_info'][0]['balance_delta'], '1.0')
|
||||
self.assertEqual(txs[0]['support_info'][0]['claim_id'], claim['claim_id'])
|
||||
self.assertEqual(txs[0]['support_info'][0]['is_tip'], True)
|
||||
self.assertEqual(txs[0]['value'], '1.0')
|
||||
self.assertEqual(txs[0]['fee'], '0.0')
|
||||
|
||||
# verify that the outgoing tip is marked correctly as is_tip=True in account2
|
||||
txs2 = await self.out(
|
||||
self.daemon.jsonrpc_transaction_list(account2_id)
|
||||
)
|
||||
self.assertEqual(len(txs2[0]['support_info']), 1)
|
||||
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-1.0')
|
||||
self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim['claim_id'])
|
||||
self.assertEqual(txs2[0]['support_info'][0]['is_tip'], True)
|
||||
self.assertEqual(txs2[0]['value'], '-1.0')
|
||||
self.assertEqual(txs2[0]['fee'], '-0.0001415')
|
||||
|
||||
# send a support to the claim using account2
|
||||
support = await self.out(
|
||||
self.daemon.jsonrpc_claim_new_support('hovercraft', claim['claim_id'], '2.0', account2_id)
|
||||
)
|
||||
await self.on_transaction_dict(support)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(support)
|
||||
|
||||
# account2 balance went down ~2
|
||||
self.assertEqual('3.979769', await self.daemon.jsonrpc_account_balance())
|
||||
self.assertEqual('1.999717', await self.daemon.jsonrpc_account_balance(account2_id))
|
||||
|
||||
# verify that the outgoing support is marked correctly as is_tip=False in account2
|
||||
txs2 = await self.out(self.daemon.jsonrpc_transaction_list(account2_id))
|
||||
self.assertEqual(len(txs2[0]['support_info']), 1)
|
||||
self.assertEqual(txs2[0]['support_info'][0]['balance_delta'], '-2.0')
|
||||
self.assertEqual(txs2[0]['support_info'][0]['claim_id'], claim['claim_id'])
|
||||
self.assertEqual(txs2[0]['support_info'][0]['is_tip'], False)
|
||||
self.assertEqual(txs2[0]['value'], '0.0')
|
||||
self.assertEqual(txs2[0]['fee'], '-0.0001415')
|
||||
|
||||
|
||||
class TransactionCommandsTestCase(CommandTestCase):
|
||||
|
||||
async def test_transaction_show(self):
|
||||
# local tx
|
||||
result = await self.out(self.daemon.jsonrpc_wallet_send(
|
||||
'5.0', await self.daemon.jsonrpc_address_unused(self.account.id)
|
||||
))
|
||||
await self.confirm_tx(result['txid'])
|
||||
tx = await self.daemon.jsonrpc_transaction_show(result['txid'])
|
||||
self.assertEqual(tx.id, result['txid'])
|
||||
|
||||
# someone's tx
|
||||
change_address = await self.blockchain.get_raw_change_address()
|
||||
sendtxid = await self.blockchain.send_to_address(change_address, 10)
|
||||
tx = await self.daemon.jsonrpc_transaction_show(sendtxid)
|
||||
self.assertEqual(tx.id, sendtxid)
|
||||
self.assertEqual(tx.height, -2)
|
||||
await self.generate(1)
|
||||
tx = await self.daemon.jsonrpc_transaction_show(sendtxid)
|
||||
self.assertEqual(tx.height, self.ledger.headers.height)
|
||||
|
||||
# inexistent
|
||||
result = await self.daemon.jsonrpc_transaction_show('0'*64)
|
||||
self.assertFalse(result['success'])
|
||||
|
||||
async def test_utxo_release(self):
|
||||
sendtxid = await self.blockchain.send_to_address(
|
||||
await self.account.receiving.get_or_create_usable_address(), 1
|
||||
)
|
||||
await self.confirm_tx(sendtxid)
|
||||
await self.assertBalance(self.account, '11.0')
|
||||
await self.ledger.reserve_outputs(await self.account.get_utxos())
|
||||
await self.assertBalance(self.account, '0.0')
|
||||
await self.daemon.jsonrpc_utxo_release()
|
||||
await self.assertBalance(self.account, '11.0')
|
||||
@@ -0,0 +1,54 @@
|
||||
import asyncio
|
||||
import tempfile
|
||||
import shutil
|
||||
import os
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
|
||||
|
||||
class TestBlobManager(AsyncioTestCase):
|
||||
async def test_sync_blob_manager_on_startup(self):
|
||||
loop = asyncio.get_event_loop()
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
|
||||
|
||||
storage = SQLiteStorage(Config(), os.path.join(tmp_dir, "lbrynet.sqlite"))
|
||||
blob_manager = BlobFileManager(loop, tmp_dir, storage)
|
||||
|
||||
# add a blob file
|
||||
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
|
||||
blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
|
||||
with open(os.path.join(blob_manager.blob_dir, blob_hash), 'wb') as f:
|
||||
f.write(blob_bytes)
|
||||
|
||||
# it should not have been added automatically on startup
|
||||
await storage.open()
|
||||
await blob_manager.setup()
|
||||
self.assertSetEqual(blob_manager.completed_blob_hashes, set())
|
||||
|
||||
# make sure we can add the blob
|
||||
await blob_manager.blob_completed(blob_manager.get_blob(blob_hash, len(blob_bytes)))
|
||||
self.assertSetEqual(blob_manager.completed_blob_hashes, {blob_hash})
|
||||
|
||||
# stop the blob manager and restart it, make sure the blob is there
|
||||
blob_manager.stop()
|
||||
self.assertSetEqual(blob_manager.completed_blob_hashes, set())
|
||||
await blob_manager.setup()
|
||||
self.assertSetEqual(blob_manager.completed_blob_hashes, {blob_hash})
|
||||
|
||||
# test that the blob is removed upon the next startup after the file being manually deleted
|
||||
blob_manager.stop()
|
||||
|
||||
# manually delete the blob file and restart the blob manager
|
||||
os.remove(os.path.join(blob_manager.blob_dir, blob_hash))
|
||||
await blob_manager.setup()
|
||||
self.assertSetEqual(blob_manager.completed_blob_hashes, set())
|
||||
|
||||
# check that the deleted blob was updated in the database
|
||||
self.assertEqual(
|
||||
'pending', (
|
||||
await storage.run_and_return_one_or_none('select status from blob where blob_hash=?', blob_hash)
|
||||
)
|
||||
)
|
||||
@@ -1,12 +1,16 @@
|
||||
import asyncio
|
||||
import tempfile
|
||||
from io import BytesIO
|
||||
|
||||
import shutil
|
||||
import os
|
||||
|
||||
from lbrynet.blob_exchange.serialization import BlobRequest
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob_exchange.server import BlobServer
|
||||
from lbrynet.blob_exchange.server import BlobServer, BlobServerProtocol
|
||||
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
|
||||
from lbrynet.dht.peer import KademliaPeer, PeerManager
|
||||
|
||||
@@ -14,6 +18,12 @@ from lbrynet.dht.peer import KademliaPeer, PeerManager
|
||||
# logging.getLogger("lbrynet").setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
def mock_config():
|
||||
config = Config()
|
||||
config.fixed_peer_delay = 10000
|
||||
return config
|
||||
|
||||
|
||||
class BlobExchangeTestBase(AsyncioTestCase):
|
||||
async def asyncSetUp(self):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
@@ -22,12 +32,15 @@ class BlobExchangeTestBase(AsyncioTestCase):
|
||||
self.server_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, self.client_dir)
|
||||
self.addCleanup(shutil.rmtree, self.server_dir)
|
||||
|
||||
self.server_storage = SQLiteStorage(Config(), os.path.join(self.server_dir, "lbrynet.sqlite"))
|
||||
self.server_config = Config(data_dir=self.server_dir, download_dir=self.server_dir, wallet=self.server_dir,
|
||||
reflector_servers=[])
|
||||
self.server_storage = SQLiteStorage(self.server_config, os.path.join(self.server_dir, "lbrynet.sqlite"))
|
||||
self.server_blob_manager = BlobFileManager(self.loop, self.server_dir, self.server_storage)
|
||||
self.server = BlobServer(self.loop, self.server_blob_manager, 'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP')
|
||||
|
||||
self.client_storage = SQLiteStorage(Config(), os.path.join(self.client_dir, "lbrynet.sqlite"))
|
||||
self.client_config = Config(data_dir=self.client_dir, download_dir=self.client_dir, wallet=self.client_dir,
|
||||
reflector_servers=[])
|
||||
self.client_storage = SQLiteStorage(self.client_config, os.path.join(self.client_dir, "lbrynet.sqlite"))
|
||||
self.client_blob_manager = BlobFileManager(self.loop, self.client_dir, self.client_storage)
|
||||
self.client_peer_manager = PeerManager(self.loop)
|
||||
self.server_from_client = KademliaPeer(self.loop, "127.0.0.1", b'1' * 48, tcp_port=33333)
|
||||
@@ -53,12 +66,11 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
|
||||
async def _test_transfer_blob(self, blob_hash: str):
|
||||
client_blob = self.client_blob_manager.get_blob(blob_hash)
|
||||
protocol = BlobExchangeClientProtocol(self.loop, 2)
|
||||
|
||||
# download the blob
|
||||
downloaded = await request_blob(self.loop, client_blob, protocol, self.server_from_client.address,
|
||||
self.server_from_client.tcp_port, 2)
|
||||
await protocol.close()
|
||||
downloaded = await request_blob(self.loop, client_blob, self.server_from_client.address,
|
||||
self.server_from_client.tcp_port, 2, 3)
|
||||
await client_blob.finished_writing.wait()
|
||||
self.assertEqual(client_blob.get_is_verified(), True)
|
||||
self.assertTrue(downloaded)
|
||||
|
||||
@@ -91,17 +103,16 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
await self._add_blob_to_server(blob_hash, mock_blob_bytes)
|
||||
|
||||
second_client_blob = self.client_blob_manager.get_blob(blob_hash)
|
||||
protocol = BlobExchangeClientProtocol(self.loop, 2)
|
||||
|
||||
# download the blob
|
||||
await asyncio.gather(
|
||||
request_blob(
|
||||
self.loop, second_client_blob, protocol, server_from_second_client.address,
|
||||
server_from_second_client.tcp_port, 2
|
||||
self.loop, second_client_blob, server_from_second_client.address,
|
||||
server_from_second_client.tcp_port, 2, 3
|
||||
),
|
||||
self._test_transfer_blob(blob_hash)
|
||||
)
|
||||
await protocol.close()
|
||||
await second_client_blob.finished_writing.wait()
|
||||
self.assertEqual(second_client_blob.get_is_verified(), True)
|
||||
|
||||
async def test_host_different_blobs_to_multiple_peers_at_once(self):
|
||||
@@ -125,14 +136,26 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
await self._add_blob_to_server(sd_hash, mock_sd_blob_bytes)
|
||||
|
||||
second_client_blob = self.client_blob_manager.get_blob(blob_hash)
|
||||
protocol = BlobExchangeClientProtocol(self.loop, 2)
|
||||
|
||||
await asyncio.gather(
|
||||
request_blob(
|
||||
self.loop, second_client_blob, protocol, server_from_second_client.address,
|
||||
server_from_second_client.tcp_port, 2
|
||||
self.loop, second_client_blob, server_from_second_client.address,
|
||||
server_from_second_client.tcp_port, 2, 3
|
||||
),
|
||||
self._test_transfer_blob(sd_hash)
|
||||
self._test_transfer_blob(sd_hash),
|
||||
second_client_blob.finished_writing.wait()
|
||||
)
|
||||
await protocol.close()
|
||||
self.assertEqual(second_client_blob.get_is_verified(), True)
|
||||
|
||||
async def test_server_chunked_request(self):
|
||||
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
|
||||
server_protocol = BlobServerProtocol(self.loop, self.server_blob_manager, self.server.lbrycrd_address)
|
||||
transport = asyncio.Transport(extra={'peername': ('ip', 90)})
|
||||
received_data = BytesIO()
|
||||
transport.write = received_data.write
|
||||
server_protocol.connection_made(transport)
|
||||
blob_request = BlobRequest.make_request_for_blob_hash(blob_hash).serialize()
|
||||
for byte in blob_request:
|
||||
server_protocol.data_received(bytes([byte]))
|
||||
await asyncio.sleep(0.1) # yield execution
|
||||
self.assertTrue(len(received_data.getvalue()) > 0)
|
||||
@@ -78,8 +78,7 @@ class StorageTest(AsyncioTestCase):
|
||||
await self.storage.close()
|
||||
|
||||
async def store_fake_blob(self, blob_hash, length=100):
|
||||
await self.storage.add_known_blob(blob_hash, length)
|
||||
await self.storage.add_completed_blob(blob_hash)
|
||||
await self.storage.add_completed_blob(blob_hash, length)
|
||||
|
||||
async def store_fake_stream(self, stream_hash, blobs=None, file_name="fake_file", key="DEADBEEF"):
|
||||
blobs = blobs or [BlobInfo(1, 100, "DEADBEEF", random_lbry_hash())]
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import asyncio
|
||||
import binascii
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from tests import dht_mocks
|
||||
from lbrynet.dht.serialization.bencoding import bencode, bdecode
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.protocol.protocol import KademliaProtocol
|
||||
from lbrynet.dht.peer import PeerManager
|
||||
@@ -67,6 +69,7 @@ class TestProtocol(AsyncioTestCase):
|
||||
peer2_from_peer1 = peer1.peer_manager.get_kademlia_peer(
|
||||
peer2.node_id, peer2.external_ip, udp_port=peer2.udp_port
|
||||
)
|
||||
peer2_from_peer1.update_tcp_port(3333)
|
||||
peer3 = peer1.peer_manager.get_kademlia_peer(
|
||||
constants.generate_id(), '1.2.3.6', udp_port=4444
|
||||
)
|
||||
@@ -76,16 +79,79 @@ class TestProtocol(AsyncioTestCase):
|
||||
self.assertEqual(True, peer1.data_store.has_peers_for_blob(b'2' * 48))
|
||||
self.assertEqual(False, peer1.data_store.has_peers_for_blob(b'3' * 48))
|
||||
self.assertListEqual([peer2_from_peer1], peer1.data_store.get_storing_contacts())
|
||||
|
||||
peer1.data_store.completed_blobs.add(binascii.hexlify(b'2' * 48).decode())
|
||||
find_value_response = peer1.node_rpc.find_value(peer3, b'2' * 48)
|
||||
self.assertEqual(len(find_value_response[b'contacts']), 0)
|
||||
self.assertSetEqual(
|
||||
{b'2' * 48, b'token', b'protocolVersion'}, set(find_value_response.keys())
|
||||
{b'2' * 48, b'token', b'protocolVersion', b'contacts'}, set(find_value_response.keys())
|
||||
)
|
||||
self.assertEqual(1, len(find_value_response[b'2' * 48]))
|
||||
self.assertEqual(find_value_response[b'2' * 48][0], peer2_from_peer1)
|
||||
# self.assertEqual(peer2_from_peer1.tcp_port, 3333)
|
||||
self.assertEqual(2, len(find_value_response[b'2' * 48]))
|
||||
self.assertEqual(find_value_response[b'2' * 48][0], peer2_from_peer1.compact_address_tcp())
|
||||
self.assertDictEqual(bdecode(bencode(find_value_response)), find_value_response)
|
||||
|
||||
peer1.stop()
|
||||
peer2.stop()
|
||||
peer1.disconnect()
|
||||
peer2.disconnect()
|
||||
|
||||
async def _make_protocol(self, other_peer, node_id, address, udp_port, tcp_port):
|
||||
proto = KademliaProtocol(
|
||||
self.loop, PeerManager(self.loop), node_id, address, udp_port, tcp_port
|
||||
)
|
||||
await self.loop.create_datagram_endpoint(lambda: proto, (address, 4444))
|
||||
return proto, other_peer.peer_manager.get_kademlia_peer(node_id, address, udp_port=udp_port)
|
||||
|
||||
async def test_add_peer_after_handle_request(self):
|
||||
with dht_mocks.mock_network_loop(self.loop):
|
||||
node_id1 = constants.generate_id()
|
||||
node_id2 = constants.generate_id()
|
||||
node_id3 = constants.generate_id()
|
||||
node_id4 = constants.generate_id()
|
||||
|
||||
peer1 = KademliaProtocol(
|
||||
self.loop, PeerManager(self.loop), node_id1, '1.2.3.4', 4444, 3333
|
||||
)
|
||||
await self.loop.create_datagram_endpoint(lambda: peer1, ('1.2.3.4', 4444))
|
||||
|
||||
peer2, peer_2_from_peer_1 = await self._make_protocol(peer1, node_id2, '1.2.3.5', 4444, 3333)
|
||||
peer3, peer_3_from_peer_1 = await self._make_protocol(peer1, node_id3, '1.2.3.6', 4444, 3333)
|
||||
peer4, peer_4_from_peer_1 = await self._make_protocol(peer1, node_id4, '1.2.3.7', 4444, 3333)
|
||||
|
||||
# peers who reply should be added
|
||||
await peer1.get_rpc_peer(peer_2_from_peer_1).ping()
|
||||
self.assertListEqual([peer_2_from_peer_1], peer1.routing_table.get_peers())
|
||||
peer1.routing_table.remove_peer(peer_2_from_peer_1)
|
||||
|
||||
# peers not known by be good/bad should be enqueued to maybe-ping
|
||||
peer1_from_peer3 = peer3.get_rpc_peer(peer3.peer_manager.get_kademlia_peer(node_id1, '1.2.3.4', 4444))
|
||||
self.assertEqual(0, len(peer1.ping_queue._pending_contacts))
|
||||
pong = await peer1_from_peer3.ping()
|
||||
self.assertEqual(b'pong', pong)
|
||||
self.assertEqual(1, len(peer1.ping_queue._pending_contacts))
|
||||
peer1.ping_queue._pending_contacts.clear()
|
||||
|
||||
# peers who are already good should be added
|
||||
peer1_from_peer4 = peer4.get_rpc_peer(peer4.peer_manager.get_kademlia_peer(node_id1, '1.2.3.4', 4444))
|
||||
peer1.peer_manager.update_contact_triple(node_id4,'1.2.3.7', 4444)
|
||||
peer1.peer_manager.report_last_replied('1.2.3.7', 4444)
|
||||
self.assertEqual(0, len(peer1.ping_queue._pending_contacts))
|
||||
pong = await peer1_from_peer4.ping()
|
||||
self.assertEqual(b'pong', pong)
|
||||
self.assertEqual(1, len(peer1.routing_table.get_peers()))
|
||||
self.assertEqual(0, len(peer1.ping_queue._pending_contacts))
|
||||
peer1.routing_table.buckets[0].peers.clear()
|
||||
|
||||
# peers who are known to be bad recently should not be added or maybe-pinged
|
||||
peer1_from_peer4 = peer4.get_rpc_peer(peer4.peer_manager.get_kademlia_peer(node_id1, '1.2.3.4', 4444))
|
||||
peer1.peer_manager.update_contact_triple(node_id4,'1.2.3.7', 4444)
|
||||
peer1.peer_manager.report_failure('1.2.3.7', 4444)
|
||||
peer1.peer_manager.report_failure('1.2.3.7', 4444)
|
||||
self.assertEqual(0, len(peer1.ping_queue._pending_contacts))
|
||||
pong = await peer1_from_peer4.ping()
|
||||
self.assertEqual(b'pong', pong)
|
||||
self.assertEqual(0, len(peer1.routing_table.get_peers()))
|
||||
self.assertEqual(0, len(peer1.ping_queue._pending_contacts))
|
||||
|
||||
for p in [peer1, peer2, peer3, peer4]:
|
||||
p.stop()
|
||||
p.disconnect()
|
||||
@@ -2,7 +2,7 @@ import struct
|
||||
import asyncio
|
||||
from lbrynet.utils import generate_id
|
||||
from lbrynet.dht.protocol.routing_table import KBucket
|
||||
from lbrynet.dht.peer import PeerManager
|
||||
from lbrynet.dht.peer import PeerManager, KademliaPeer
|
||||
from lbrynet.dht import constants
|
||||
from torba.testcase import AsyncioTestCase
|
||||
|
||||
@@ -29,6 +29,33 @@ class TestKBucket(AsyncioTestCase):
|
||||
self.kbucket = KBucket(self.peer_manager, 0, 2**constants.hash_bits, generate_id())
|
||||
|
||||
def test_add_peer(self):
|
||||
peer = KademliaPeer(None, '1.2.3.4', constants.generate_id(2), udp_port=4444)
|
||||
peer_update2 = KademliaPeer(None, '1.2.3.4', constants.generate_id(2), udp_port=4445)
|
||||
|
||||
self.assertListEqual([], self.kbucket.peers)
|
||||
|
||||
# add the peer
|
||||
self.kbucket.add_peer(peer)
|
||||
self.assertListEqual([peer], self.kbucket.peers)
|
||||
|
||||
# re-add it
|
||||
self.kbucket.add_peer(peer)
|
||||
self.assertListEqual([peer], self.kbucket.peers)
|
||||
self.assertEqual(self.kbucket.peers[0].udp_port, 4444)
|
||||
|
||||
# add a new peer object with the same id and address but a different port
|
||||
self.kbucket.add_peer(peer_update2)
|
||||
self.assertListEqual([peer_update2], self.kbucket.peers)
|
||||
self.assertEqual(self.kbucket.peers[0].udp_port, 4445)
|
||||
|
||||
# modify the peer object to have a different port
|
||||
peer_update2.udp_port = 4444
|
||||
self.kbucket.add_peer(peer_update2)
|
||||
self.assertListEqual([peer_update2], self.kbucket.peers)
|
||||
self.assertEqual(self.kbucket.peers[0].udp_port, 4444)
|
||||
|
||||
self.kbucket.peers.clear()
|
||||
|
||||
# Test if contacts can be added to empty list
|
||||
# Add k contacts to bucket
|
||||
for i in range(constants.k):
|
||||
|
||||
@@ -6,6 +6,29 @@ from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.peer import PeerManager
|
||||
|
||||
|
||||
expected_ranges = [
|
||||
(
|
||||
0,
|
||||
2462625387274654950767440006258975862817483704404090416746768337765357610718575663213391640930307227550414249394176
|
||||
),
|
||||
(
|
||||
2462625387274654950767440006258975862817483704404090416746768337765357610718575663213391640930307227550414249394176,
|
||||
4925250774549309901534880012517951725634967408808180833493536675530715221437151326426783281860614455100828498788352
|
||||
),
|
||||
(
|
||||
4925250774549309901534880012517951725634967408808180833493536675530715221437151326426783281860614455100828498788352,
|
||||
9850501549098619803069760025035903451269934817616361666987073351061430442874302652853566563721228910201656997576704
|
||||
),
|
||||
(
|
||||
9850501549098619803069760025035903451269934817616361666987073351061430442874302652853566563721228910201656997576704,
|
||||
19701003098197239606139520050071806902539869635232723333974146702122860885748605305707133127442457820403313995153408
|
||||
),
|
||||
(
|
||||
19701003098197239606139520050071806902539869635232723333974146702122860885748605305707133127442457820403313995153408,
|
||||
39402006196394479212279040100143613805079739270465446667948293404245721771497210611414266254884915640806627990306816
|
||||
)
|
||||
]
|
||||
|
||||
class TestRouting(AsyncioTestCase):
|
||||
async def test_fill_one_bucket(self):
|
||||
loop = asyncio.get_event_loop()
|
||||
@@ -43,6 +66,42 @@ class TestRouting(AsyncioTestCase):
|
||||
for node in nodes.values():
|
||||
node.protocol.stop()
|
||||
|
||||
async def test_split_buckets(self):
|
||||
loop = asyncio.get_event_loop()
|
||||
peer_addresses = [
|
||||
(constants.generate_id(1), '1.2.3.1'),
|
||||
]
|
||||
for i in range(2, 200):
|
||||
peer_addresses.append((constants.generate_id(i), f'1.2.3.{i}'))
|
||||
with dht_mocks.mock_network_loop(loop):
|
||||
nodes = {
|
||||
i: Node(loop, PeerManager(loop), node_id, 4444, 4444, 3333, address)
|
||||
for i, (node_id, address) in enumerate(peer_addresses)
|
||||
}
|
||||
node_1 = nodes[0]
|
||||
for i in range(1, len(peer_addresses)):
|
||||
node = nodes[i]
|
||||
peer = node_1.protocol.peer_manager.get_kademlia_peer(
|
||||
node.protocol.node_id, node.protocol.external_ip,
|
||||
udp_port=node.protocol.udp_port
|
||||
)
|
||||
# set all of the peers to good (as to not attempt pinging stale ones during split)
|
||||
node_1.protocol.peer_manager.report_last_replied(peer.address, peer.udp_port)
|
||||
node_1.protocol.peer_manager.report_last_replied(peer.address, peer.udp_port)
|
||||
await node_1.protocol.add_peer(peer)
|
||||
# check that bucket 0 is always the one covering the local node id
|
||||
self.assertEqual(True, node_1.protocol.routing_table.buckets[0].key_in_range(node_1.protocol.node_id))
|
||||
self.assertEqual(40, len(node_1.protocol.routing_table.get_peers()))
|
||||
self.assertEqual(len(expected_ranges), len(node_1.protocol.routing_table.buckets))
|
||||
covered = 0
|
||||
for (expected_min, expected_max), bucket in zip(expected_ranges, node_1.protocol.routing_table.buckets):
|
||||
self.assertEqual(expected_min, bucket.range_min)
|
||||
self.assertEqual(expected_max, bucket.range_max)
|
||||
covered += bucket.range_max - bucket.range_min
|
||||
self.assertEqual(2**384, covered)
|
||||
for node in nodes.values():
|
||||
node.stop()
|
||||
|
||||
|
||||
# from binascii import hexlify, unhexlify
|
||||
#
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import unittest
|
||||
import asyncio
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from lbrynet.dht.protocol.async_generator_junction import AsyncGeneratorJunction
|
||||
@@ -41,16 +42,17 @@ class TestAsyncGeneratorJunction(AsyncioTestCase):
|
||||
|
||||
async def test_yield_order(self):
|
||||
expected_order = [1, 2, 1, 1, 2, 1, 1, 2, 1, 1, 2, 1, 1, 2, 1, 2, 2, 2, 2, 2]
|
||||
fast_gen = MockAsyncGen(self.loop, 1, 0.1)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.2)
|
||||
fast_gen = MockAsyncGen(self.loop, 1, 0.2)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.4)
|
||||
await self._test_junction(expected_order, fast_gen, slow_gen)
|
||||
self.assertEqual(fast_gen.called_close, True)
|
||||
self.assertEqual(slow_gen.called_close, True)
|
||||
|
||||
@unittest.SkipTest
|
||||
async def test_one_stopped_first(self):
|
||||
expected_order = [1, 2, 1, 1, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2]
|
||||
fast_gen = MockAsyncGen(self.loop, 1, 0.101, 5)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.201)
|
||||
fast_gen = MockAsyncGen(self.loop, 1, 0.2, 5)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.4)
|
||||
await self._test_junction(expected_order, fast_gen, slow_gen)
|
||||
self.assertEqual(fast_gen.called_close, True)
|
||||
self.assertEqual(slow_gen.called_close, True)
|
||||
@@ -62,16 +64,16 @@ class TestAsyncGeneratorJunction(AsyncioTestCase):
|
||||
for i in range(10):
|
||||
if i == 5:
|
||||
return
|
||||
await asyncio.sleep(0.101)
|
||||
await asyncio.sleep(0.2)
|
||||
yield 1
|
||||
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.201)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.4)
|
||||
await self._test_junction(expected_order, fast_gen(), slow_gen)
|
||||
self.assertEqual(slow_gen.called_close, True)
|
||||
|
||||
async def test_stop_when_encapsulating_task_cancelled(self):
|
||||
fast_gen = MockAsyncGen(self.loop, 1, 0.1)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.2)
|
||||
fast_gen = MockAsyncGen(self.loop, 1, 0.2)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.4)
|
||||
|
||||
async def _task():
|
||||
async with AsyncGeneratorJunction(self.loop) as junction:
|
||||
@@ -81,7 +83,7 @@ class TestAsyncGeneratorJunction(AsyncioTestCase):
|
||||
pass
|
||||
|
||||
task = self.loop.create_task(_task())
|
||||
self.loop.call_later(0.5, task.cancel)
|
||||
self.loop.call_later(1.0, task.cancel)
|
||||
with self.assertRaises(asyncio.CancelledError):
|
||||
await task
|
||||
self.assertEqual(fast_gen.called_close, True)
|
||||
|
||||
@@ -31,9 +31,9 @@ def get_test_daemon(conf: Config, with_fee=False):
|
||||
component_manager = ComponentManager(
|
||||
conf, skip_components=[
|
||||
DATABASE_COMPONENT, DHT_COMPONENT, WALLET_COMPONENT, UPNP_COMPONENT,
|
||||
PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, HASH_ANNOUNCER_COMPONENT,
|
||||
EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT,
|
||||
HEADERS_COMPONENT, RATE_LIMITER_COMPONENT],
|
||||
PEER_PROTOCOL_SERVER_COMPONENT, HASH_ANNOUNCER_COMPONENT,
|
||||
EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT, HEADERS_COMPONENT,
|
||||
RATE_LIMITER_COMPONENT],
|
||||
file_manager=FakeFileManager
|
||||
)
|
||||
daemon = LBRYDaemon(conf, component_manager=component_manager)
|
||||
@@ -84,15 +84,6 @@ class TestCostEst(unittest.TestCase):
|
||||
result = yield f2d(daemon.get_est_cost("test", size))
|
||||
self.assertEqual(result, correct_result)
|
||||
|
||||
def test_fee_and_ungenerous_data(self):
|
||||
conf = Config(is_generous_host=False)
|
||||
size = 10000000
|
||||
fake_fee_amount = 4.5
|
||||
correct_result = size / 10 ** 6 * conf.data_rate + fake_fee_amount
|
||||
daemon = get_test_daemon(conf, with_fee=True)
|
||||
result = yield f2d(daemon.get_est_cost("test", size))
|
||||
self.assertEqual(result, round(correct_result, 1))
|
||||
|
||||
def test_generous_data_and_no_fee(self):
|
||||
size = 10000000
|
||||
correct_result = 0.0
|
||||
@@ -100,14 +91,6 @@ class TestCostEst(unittest.TestCase):
|
||||
result = yield f2d(daemon.get_est_cost("test", size))
|
||||
self.assertEqual(result, correct_result)
|
||||
|
||||
def test_ungenerous_data_and_no_fee(self):
|
||||
conf = Config(is_generous_host=False)
|
||||
size = 10000000
|
||||
correct_result = size / 10 ** 6 * conf.data_rate
|
||||
daemon = get_test_daemon(conf)
|
||||
result = yield f2d(daemon.get_est_cost("test", size))
|
||||
self.assertEqual(result, round(correct_result, 1))
|
||||
|
||||
|
||||
@unittest.SkipTest
|
||||
class TestJsonRpc(unittest.TestCase):
|
||||
|
||||
@@ -35,6 +35,14 @@ class DummyExchangeRateManager(exchange_rate_manager.ExchangeRateManager):
|
||||
feed.market, rates[feed.market]['spot'], rates[feed.market]['ts'])
|
||||
|
||||
|
||||
def get_dummy_exchange_rate_manager(time):
|
||||
rates = {
|
||||
'BTCLBC': {'spot': 3.0, 'ts': time.time() + 1},
|
||||
'USDBTC': {'spot': 2.0, 'ts': time.time() + 2}
|
||||
}
|
||||
return DummyExchangeRateManager([BTCLBCFeed(), USDBTCFeed()], rates)
|
||||
|
||||
|
||||
class FeeFormatTest(unittest.TestCase):
|
||||
def test_fee_created_with_correct_inputs(self):
|
||||
fee_dict = {
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
import unittest
|
||||
from lbrynet.extras.daemon import mime_types
|
||||
|
||||
|
||||
class TestMimeTypes(unittest.TestCase):
|
||||
def test_mp4_video(self):
|
||||
self.assertEqual("video/mp4", mime_types.guess_media_type("test.mp4"))
|
||||
self.assertEqual("video/mp4", mime_types.guess_media_type("test.MP4"))
|
||||
|
||||
def test_x_ext_(self):
|
||||
self.assertEqual("application/x-ext-lbry", mime_types.guess_media_type("test.lbry"))
|
||||
self.assertEqual("application/x-ext-lbry", mime_types.guess_media_type("test.LBRY"))
|
||||
|
||||
def test_octet_stream(self):
|
||||
self.assertEqual("application/octet-stream", mime_types.guess_media_type("test."))
|
||||
self.assertEqual("application/octet-stream", mime_types.guess_media_type("test"))
|
||||
@@ -72,7 +72,6 @@ parsed_uri_raises = [
|
||||
("lbry://test:1:1:1", URIParseError),
|
||||
("whatever/lbry://test", URIParseError),
|
||||
("lbry://lbry://test", URIParseError),
|
||||
("lbry://❀", URIParseError),
|
||||
("lbry://@/what", URIParseError),
|
||||
("lbry://abc:0x123", URIParseError),
|
||||
("lbry://abc:0x123/page", URIParseError),
|
||||
|
||||
@@ -2,13 +2,15 @@ import os
|
||||
import asyncio
|
||||
import tempfile
|
||||
import shutil
|
||||
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.stream.assembler import StreamAssembler
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.stream.stream_manager import StreamManager
|
||||
|
||||
|
||||
class TestStreamAssembler(AsyncioTestCase):
|
||||
@@ -17,10 +19,10 @@ class TestStreamAssembler(AsyncioTestCase):
|
||||
self.key = b'deadbeef' * 4
|
||||
self.cleartext = b'test'
|
||||
|
||||
async def test_create_and_decrypt_one_blob_stream(self):
|
||||
async def test_create_and_decrypt_one_blob_stream(self, corrupt=False):
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
|
||||
self.storage = SQLiteStorage(Config(), os.path.join(tmp_dir, "lbrynet.sqlite"))
|
||||
self.storage = SQLiteStorage(Config(), ":memory:")
|
||||
await self.storage.open()
|
||||
self.blob_manager = BlobFileManager(self.loop, tmp_dir, self.storage)
|
||||
|
||||
@@ -40,6 +42,11 @@ class TestStreamAssembler(AsyncioTestCase):
|
||||
for blob_info in sd.blobs:
|
||||
if blob_info.blob_hash:
|
||||
shutil.copy(os.path.join(tmp_dir, blob_info.blob_hash), os.path.join(download_dir, blob_info.blob_hash))
|
||||
if corrupt and blob_info.length == MAX_BLOB_SIZE:
|
||||
with open(os.path.join(download_dir, blob_info.blob_hash), "rb+") as handle:
|
||||
handle.truncate()
|
||||
handle.flush()
|
||||
|
||||
downloader_storage = SQLiteStorage(Config(), os.path.join(download_dir, "lbrynet.sqlite"))
|
||||
await downloader_storage.open()
|
||||
|
||||
@@ -50,11 +57,19 @@ class TestStreamAssembler(AsyncioTestCase):
|
||||
# assemble the decrypted file
|
||||
assembler = StreamAssembler(self.loop, downloader_blob_manager, descriptor.sd_hash)
|
||||
await assembler.assemble_decrypted_stream(download_dir)
|
||||
if corrupt:
|
||||
return self.assertFalse(os.path.isfile(os.path.join(download_dir, "test_file")))
|
||||
|
||||
with open(os.path.join(download_dir, "test_file"), "rb") as f:
|
||||
decrypted = f.read()
|
||||
self.assertEqual(decrypted, self.cleartext)
|
||||
self.assertEqual(True, self.blob_manager.get_blob(sd_hash).get_is_verified())
|
||||
self.assertEqual(True, self.blob_manager.get_blob(descriptor.blobs[0].blob_hash).get_is_verified())
|
||||
# its all blobs + sd blob - last blob, which is the same size as descriptor.blobs
|
||||
self.assertEqual(len(descriptor.blobs), len(await downloader_storage.get_all_finished_blobs()))
|
||||
self.assertEqual(
|
||||
[descriptor.sd_hash, descriptor.blobs[0].blob_hash], await downloader_storage.get_blobs_to_announce()
|
||||
)
|
||||
|
||||
await downloader_storage.close()
|
||||
await self.storage.close()
|
||||
@@ -75,3 +90,28 @@ class TestStreamAssembler(AsyncioTestCase):
|
||||
async def test_create_and_decrypt_random(self):
|
||||
self.cleartext = os.urandom(20000000)
|
||||
await self.test_create_and_decrypt_one_blob_stream()
|
||||
|
||||
async def test_create_managed_stream_announces(self):
|
||||
# setup a blob manager
|
||||
storage = SQLiteStorage(Config(), ":memory:")
|
||||
await storage.open()
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
|
||||
blob_manager = BlobFileManager(self.loop, tmp_dir, storage)
|
||||
stream_manager = StreamManager(self.loop, Config(), blob_manager, None, storage, None)
|
||||
# create the stream
|
||||
download_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(download_dir))
|
||||
file_path = os.path.join(download_dir, "test_file")
|
||||
with open(file_path, 'wb') as f:
|
||||
f.write(b'testtest')
|
||||
|
||||
stream = await stream_manager.create_stream(file_path)
|
||||
self.assertEqual(
|
||||
[stream.sd_hash, stream.descriptor.blobs[0].blob_hash],
|
||||
await storage.get_blobs_to_announce())
|
||||
|
||||
async def test_create_truncate_and_handle_stream(self):
|
||||
self.cleartext = b'potato' * 1337 * 5279
|
||||
# The purpose of this test is just to make sure it can finish even if a blob is corrupt/truncated
|
||||
await asyncio.wait_for(self.test_create_and_decrypt_one_blob_stream(corrupt=True), timeout=5)
|
||||
@@ -1,7 +1,12 @@
|
||||
import os
|
||||
import time
|
||||
import unittest
|
||||
from unittest import mock
|
||||
import asyncio
|
||||
import contextlib
|
||||
|
||||
from lbrynet.blob_exchange.serialization import BlobResponse
|
||||
from lbrynet.blob_exchange.server import BlobServerProtocol
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.stream.downloader import StreamDownloader
|
||||
from lbrynet.dht.node import Node
|
||||
@@ -21,61 +26,77 @@ class TestStreamDownloader(BlobExchangeTestBase):
|
||||
f.write(self.stream_bytes)
|
||||
descriptor = await StreamDescriptor.create_stream(self.loop, self.server_blob_manager.blob_dir, file_path)
|
||||
self.sd_hash = descriptor.calculate_sd_hash()
|
||||
self.downloader = StreamDownloader(self.loop, self.client_blob_manager, self.sd_hash, 3, 3, self.client_dir)
|
||||
conf = Config(data_dir=self.server_dir, wallet_dir=self.server_dir, download_dir=self.server_dir,
|
||||
reflector_servers=[])
|
||||
self.downloader = StreamDownloader(self.loop, conf, self.client_blob_manager, self.sd_hash)
|
||||
|
||||
async def _test_transfer_stream(self, blob_count: int, mock_peer_search=None):
|
||||
async def _test_transfer_stream(self, blob_count: int, mock_accumulate_peers=None):
|
||||
await self.setup_stream(blob_count)
|
||||
|
||||
mock_node = mock.Mock(spec=Node)
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _mock_peer_search(*_):
|
||||
async def _gen():
|
||||
yield [self.server_from_client]
|
||||
return
|
||||
|
||||
yield _gen()
|
||||
|
||||
mock_node.stream_peer_search_junction = mock_peer_search or _mock_peer_search
|
||||
def _mock_accumulate_peers(q1, q2):
|
||||
async def _task():
|
||||
pass
|
||||
q2.put_nowait([self.server_from_client])
|
||||
return q2, self.loop.create_task(_task())
|
||||
|
||||
mock_node.accumulate_peers = mock_accumulate_peers or _mock_accumulate_peers
|
||||
self.downloader.download(mock_node)
|
||||
await self.downloader.stream_finished_event.wait()
|
||||
await self.downloader.stop()
|
||||
self.assertTrue(self.downloader.stream_handle.closed)
|
||||
self.assertTrue(os.path.isfile(self.downloader.output_path))
|
||||
self.downloader.stop()
|
||||
self.assertIs(self.downloader.stream_handle, None)
|
||||
self.assertTrue(os.path.isfile(self.downloader.output_path))
|
||||
with open(self.downloader.output_path, 'rb') as f:
|
||||
self.assertEqual(f.read(), self.stream_bytes)
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
async def test_transfer_stream(self):
|
||||
await self._test_transfer_stream(10)
|
||||
|
||||
# async def test_transfer_hundred_blob_stream(self):
|
||||
# await self._test_transfer_stream(100)
|
||||
@unittest.SkipTest
|
||||
async def test_transfer_hundred_blob_stream(self):
|
||||
await self._test_transfer_stream(100)
|
||||
|
||||
async def test_transfer_stream_bad_first_peer_good_second(self):
|
||||
await self.setup_stream(2)
|
||||
|
||||
mock_node = mock.Mock(spec=Node)
|
||||
q = asyncio.Queue()
|
||||
|
||||
bad_peer = KademliaPeer(self.loop, "127.0.0.1", b'2' * 48, tcp_port=3334)
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def mock_peer_search(*_):
|
||||
async def _gen():
|
||||
await asyncio.sleep(0.05, loop=self.loop)
|
||||
yield [bad_peer]
|
||||
await asyncio.sleep(0.1, loop=self.loop)
|
||||
yield [self.server_from_client]
|
||||
return
|
||||
def _mock_accumulate_peers(q1, q2):
|
||||
async def _task():
|
||||
pass
|
||||
|
||||
yield _gen()
|
||||
q2.put_nowait([bad_peer])
|
||||
self.loop.call_later(1, q2.put_nowait, [self.server_from_client])
|
||||
return q2, self.loop.create_task(_task())
|
||||
|
||||
mock_node.stream_peer_search_junction = mock_peer_search
|
||||
mock_node.accumulate_peers = _mock_accumulate_peers
|
||||
|
||||
self.downloader.download(mock_node)
|
||||
await self.downloader.stream_finished_event.wait()
|
||||
await self.downloader.stop()
|
||||
self.assertTrue(os.path.isfile(self.downloader.output_path))
|
||||
with open(self.downloader.output_path, 'rb') as f:
|
||||
self.assertEqual(f.read(), self.stream_bytes)
|
||||
# self.assertIs(self.server_from_client.tcp_last_down, None)
|
||||
# self.assertIsNot(bad_peer.tcp_last_down, None)
|
||||
|
||||
async def test_client_chunked_response(self):
|
||||
self.server.stop_server()
|
||||
class ChunkedServerProtocol(BlobServerProtocol):
|
||||
|
||||
def send_response(self, responses):
|
||||
to_send = []
|
||||
while responses:
|
||||
to_send.append(responses.pop())
|
||||
for byte in BlobResponse(to_send).serialize():
|
||||
self.transport.write(bytes([byte]))
|
||||
self.server.server_protocol_class = ChunkedServerProtocol
|
||||
self.server.start_server(33333, '127.0.0.1')
|
||||
self.assertEqual(0, len(self.client_blob_manager.completed_blob_hashes))
|
||||
await asyncio.wait_for(self._test_transfer_stream(10), timeout=2)
|
||||
self.assertEqual(11, len(self.client_blob_manager.completed_blob_hashes))
|
||||
@@ -21,7 +21,7 @@ class TestStreamAssembler(AsyncioTestCase):
|
||||
self.storage = SQLiteStorage(Config(), os.path.join(tmp_dir, "lbrynet.sqlite"))
|
||||
await self.storage.open()
|
||||
self.blob_manager = BlobFileManager(self.loop, tmp_dir, self.storage)
|
||||
self.stream_manager = StreamManager(self.loop, self.blob_manager, None, self.storage, None, 3.0, 3.0)
|
||||
self.stream_manager = StreamManager(self.loop, Config(), self.blob_manager, None, self.storage, None)
|
||||
|
||||
server_tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(server_tmp_dir))
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
import os
|
||||
import asyncio
|
||||
import tempfile
|
||||
import shutil
|
||||
import json
|
||||
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.error import InvalidStreamDescriptorError
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
|
||||
|
||||
class TestStreamDescriptor(AsyncioTestCase):
|
||||
async def asyncSetUp(self):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.key = b'deadbeef' * 4
|
||||
self.cleartext = os.urandom(20000000)
|
||||
self.tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(self.tmp_dir))
|
||||
self.storage = SQLiteStorage(Config(), ":memory:")
|
||||
await self.storage.open()
|
||||
self.blob_manager = BlobFileManager(self.loop, self.tmp_dir, self.storage)
|
||||
|
||||
self.file_path = os.path.join(self.tmp_dir, "test_file")
|
||||
with open(self.file_path, 'wb') as f:
|
||||
f.write(self.cleartext)
|
||||
|
||||
self.descriptor = await StreamDescriptor.create_stream(self.loop, self.tmp_dir, self.file_path, key=self.key)
|
||||
self.sd_hash = self.descriptor.calculate_sd_hash()
|
||||
self.sd_dict = json.loads(self.descriptor.as_json())
|
||||
|
||||
def _write_sd(self):
|
||||
with open(os.path.join(self.tmp_dir, self.sd_hash), 'wb') as f:
|
||||
f.write(json.dumps(self.sd_dict, sort_keys=True).encode())
|
||||
|
||||
async def _test_invalid_sd(self):
|
||||
self._write_sd()
|
||||
with self.assertRaises(InvalidStreamDescriptorError):
|
||||
await self.blob_manager.get_stream_descriptor(self.sd_hash)
|
||||
|
||||
async def test_load_sd_blob(self):
|
||||
self._write_sd()
|
||||
descriptor = await self.blob_manager.get_stream_descriptor(self.sd_hash)
|
||||
self.assertEqual(descriptor.calculate_sd_hash(), self.sd_hash)
|
||||
|
||||
async def test_missing_terminator(self):
|
||||
self.sd_dict['blobs'].pop()
|
||||
await self._test_invalid_sd()
|
||||
|
||||
async def test_terminator_not_at_end(self):
|
||||
terminator = self.sd_dict['blobs'].pop()
|
||||
self.sd_dict['blobs'] = [terminator] + self.sd_dict['blobs']
|
||||
await self._test_invalid_sd()
|
||||
|
||||
async def test_terminator_has_blob_hash(self):
|
||||
self.sd_dict['blobs'][-1]['blob_hash'] = '1' * 96
|
||||
await self._test_invalid_sd()
|
||||
|
||||
async def test_blob_order(self):
|
||||
terminator = self.sd_dict['blobs'].pop()
|
||||
self.sd_dict['blobs'].reverse()
|
||||
self.sd_dict['blobs'].append(terminator)
|
||||
await self._test_invalid_sd()
|
||||
|
||||
async def test_skip_blobs(self):
|
||||
self.sd_dict['blobs'][-2]['blob_num'] = self.sd_dict['blobs'][-2]['blob_num'] + 1
|
||||
await self._test_invalid_sd()
|
||||
|
||||
async def test_invalid_stream_hash(self):
|
||||
self.sd_dict['blobs'][-2]['blob_hash'] = '1' * 96
|
||||
await self._test_invalid_sd()
|
||||
|
||||
async def test_zero_length_blob(self):
|
||||
self.sd_dict['blobs'][-2]['length'] = 0
|
||||
await self._test_invalid_sd()
|
||||
|
||||
|
||||
class TestRecoverOldStreamDescriptors(AsyncioTestCase):
|
||||
async def test_old_key_sort_sd_blob(self):
|
||||
loop = asyncio.get_event_loop()
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
|
||||
storage = SQLiteStorage(Config(), ":memory:")
|
||||
await storage.open()
|
||||
blob_manager = BlobFileManager(loop, tmp_dir, storage)
|
||||
|
||||
sd_bytes = b'{"stream_name": "4f62616d6120446f6e6b65792d322e73746c", "blobs": [{"length": 1153488, "blob_num' \
|
||||
b'": 0, "blob_hash": "9fa32a249ce3f2d4e46b78599800f368b72f2a7f22b81df443c7f6bdbef496bd61b4c0079c7' \
|
||||
b'3d79c8bb9be9a6bf86592", "iv": "0bf348867244019c9e22196339016ea6"}, {"length": 0, "blob_num": 1,' \
|
||||
b' "iv": "9f36abae16955463919b07ed530a3d18"}], "stream_type": "lbryfile", "key": "a03742b87628aa7' \
|
||||
b'228e48f1dcd207e48", "suggested_file_name": "4f62616d6120446f6e6b65792d322e73746c", "stream_hash' \
|
||||
b'": "b43f4b1379780caf60d20aa06ac38fb144df61e514ebfa97537018ba73bce8fe37ae712f473ff0ba0be0eef44e1' \
|
||||
b'60207"}'
|
||||
sd_hash = '9313d1807551186126acc3662e74d9de29cede78d4f133349ace846273ef116b9bb86be86c54509eb84840e4b032f6b2'
|
||||
stream_hash = 'b43f4b1379780caf60d20aa06ac38fb144df61e514ebfa97537018ba73bce8fe37ae712f473ff0ba0be0eef44e160207'
|
||||
|
||||
blob = blob_manager.get_blob(sd_hash)
|
||||
blob.set_length(len(sd_bytes))
|
||||
writer = blob.open_for_writing()
|
||||
writer.write(sd_bytes)
|
||||
await blob.verified.wait()
|
||||
descriptor = await StreamDescriptor.from_stream_descriptor_blob(
|
||||
loop, blob_manager.blob_dir, blob
|
||||
)
|
||||
self.assertEqual(stream_hash, descriptor.get_stream_hash())
|
||||
self.assertEqual(sd_hash, descriptor.calculate_old_sort_sd_hash())
|
||||
self.assertNotEqual(sd_hash, descriptor.calculate_sd_hash())
|
||||
|
||||
async def test_decode_corrupt_blob_raises_proper_exception_and_deletes_corrupt_file(self):
|
||||
loop = asyncio.get_event_loop()
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
|
||||
sd_hash = '9313d1807551186126acc3662e74d9de29cede78d4f133349ace846273ef116b9bb86be86c54509eb84840e4b032f6b2'
|
||||
with open(os.path.join(tmp_dir, sd_hash), 'wb') as handle:
|
||||
handle.write(b'doesnt work')
|
||||
blob = BlobFile(loop, tmp_dir, sd_hash)
|
||||
self.assertTrue(blob.file_exists)
|
||||
self.assertIsNotNone(blob.length)
|
||||
with self.assertRaises(InvalidStreamDescriptorError):
|
||||
await StreamDescriptor.from_stream_descriptor_blob(
|
||||
loop, tmp_dir, blob
|
||||
)
|
||||
self.assertFalse(blob.file_exists)
|
||||
# fixme: this is an emergency PR, plase move this to blob_file tests later
|
||||
self.assertIsNone(blob.length)
|
||||
@@ -0,0 +1,361 @@
|
||||
import os
|
||||
import binascii
|
||||
from unittest import mock
|
||||
import asyncio
|
||||
import time
|
||||
import json
|
||||
from tests.unit.blob_exchange.test_transfer_blob import BlobExchangeTestBase
|
||||
from tests.unit.lbrynet_daemon.test_ExchangeRateManager import get_dummy_exchange_rate_manager
|
||||
from lbrynet.utils import generate_id
|
||||
from lbrynet.error import InsufficientFundsError, KeyFeeAboveMaxAllowed, ResolveError, DownloadSDTimeout, \
|
||||
DownloadDataTimeout
|
||||
from lbrynet.extras.wallet.manager import LbryWalletManager
|
||||
from lbrynet.extras.daemon.analytics import AnalyticsManager
|
||||
from lbrynet.stream.stream_manager import StreamManager
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.protocol.protocol import KademliaProtocol
|
||||
from lbrynet.dht.protocol.routing_table import TreeRoutingTable
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
|
||||
|
||||
def get_mock_node(peer=None):
|
||||
def mock_accumulate_peers(q1: asyncio.Queue, q2: asyncio.Queue):
|
||||
async def _task():
|
||||
pass
|
||||
if peer:
|
||||
q2.put_nowait([peer])
|
||||
return q2, asyncio.create_task(_task())
|
||||
|
||||
mock_node = mock.Mock(spec=Node)
|
||||
mock_node.protocol = mock.Mock(spec=KademliaProtocol)
|
||||
mock_node.protocol.routing_table = mock.Mock(spec=TreeRoutingTable)
|
||||
mock_node.protocol.routing_table.get_peers = lambda: []
|
||||
mock_node.accumulate_peers = mock_accumulate_peers
|
||||
mock_node.joined = asyncio.Event()
|
||||
mock_node.joined.set()
|
||||
return mock_node
|
||||
|
||||
|
||||
def get_mock_wallet(sd_hash, storage, balance=10.0, fee=None):
|
||||
claim = {
|
||||
"address": "bYFeMtSL7ARuG1iMpjFyrnTe4oJHSAVNXF",
|
||||
"amount": "0.1",
|
||||
"claim_id": "c49566d631226492317d06ad7fdbe1ed32925124",
|
||||
"claim_sequence": 1,
|
||||
"decoded_claim": True,
|
||||
"depth": 1057,
|
||||
"effective_amount": "0.1",
|
||||
"has_signature": False,
|
||||
"height": 514081,
|
||||
"hex": "",
|
||||
"name": "33rpm",
|
||||
"nout": 0,
|
||||
"permanent_url": "33rpm#c49566d631226492317d06ad7fdbe1ed32925124",
|
||||
"supports": [],
|
||||
"txid": "81ac52662af926fdf639d56920069e0f63449d4cde074c61717cb99ddde40e3c",
|
||||
"value": {
|
||||
"claimType": "streamType",
|
||||
"stream": {
|
||||
"metadata": {
|
||||
"author": "",
|
||||
"description": "",
|
||||
"language": "en",
|
||||
"license": "None",
|
||||
"licenseUrl": "",
|
||||
"nsfw": False,
|
||||
"preview": "",
|
||||
"thumbnail": "",
|
||||
"title": "33rpm",
|
||||
"version": "_0_1_0"
|
||||
},
|
||||
"source": {
|
||||
"contentType": "image/png",
|
||||
"source": sd_hash,
|
||||
"sourceType": "lbry_sd_hash",
|
||||
"version": "_0_0_1"
|
||||
},
|
||||
"version": "_0_0_1"
|
||||
},
|
||||
"version": "_0_0_1"
|
||||
}
|
||||
}
|
||||
if fee:
|
||||
claim['value']['stream']['metadata']['fee'] = fee
|
||||
claim_dict = ClaimDict.load_dict(claim['value'])
|
||||
claim['hex'] = binascii.hexlify(claim_dict.serialized).decode()
|
||||
|
||||
async def mock_resolve(*args):
|
||||
await storage.save_claims([claim])
|
||||
return {
|
||||
claim['permanent_url']: claim
|
||||
}
|
||||
|
||||
mock_wallet = mock.Mock(spec=LbryWalletManager)
|
||||
mock_wallet.resolve = mock_resolve
|
||||
|
||||
async def get_balance(*_):
|
||||
return balance
|
||||
|
||||
mock_wallet.default_account.get_balance = get_balance
|
||||
return mock_wallet, claim['permanent_url']
|
||||
|
||||
|
||||
class TestStreamManager(BlobExchangeTestBase):
|
||||
async def setup_stream_manager(self, balance=10.0, fee=None, old_sort=False):
|
||||
file_path = os.path.join(self.server_dir, "test_file")
|
||||
with open(file_path, 'wb') as f:
|
||||
f.write(os.urandom(20000000))
|
||||
descriptor = await StreamDescriptor.create_stream(
|
||||
self.loop, self.server_blob_manager.blob_dir, file_path, old_sort=old_sort
|
||||
)
|
||||
self.sd_hash = descriptor.sd_hash
|
||||
self.mock_wallet, self.uri = get_mock_wallet(self.sd_hash, self.client_storage, balance, fee)
|
||||
self.stream_manager = StreamManager(self.loop, self.client_config, self.client_blob_manager, self.mock_wallet,
|
||||
self.client_storage, get_mock_node(self.server_from_client),
|
||||
AnalyticsManager(self.client_config,
|
||||
binascii.hexlify(generate_id()).decode(),
|
||||
binascii.hexlify(generate_id()).decode()))
|
||||
self.exchange_rate_manager = get_dummy_exchange_rate_manager(time)
|
||||
|
||||
async def _test_time_to_first_bytes(self, check_post, error=None, after_setup=None):
|
||||
await self.setup_stream_manager()
|
||||
if after_setup:
|
||||
after_setup()
|
||||
checked_analytics_event = False
|
||||
|
||||
async def _check_post(event):
|
||||
check_post(event)
|
||||
nonlocal checked_analytics_event
|
||||
checked_analytics_event = True
|
||||
|
||||
self.stream_manager.analytics_manager._post = _check_post
|
||||
if error:
|
||||
with self.assertRaises(error):
|
||||
await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager)
|
||||
else:
|
||||
await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager)
|
||||
await asyncio.sleep(0, loop=self.loop)
|
||||
self.assertTrue(checked_analytics_event)
|
||||
|
||||
async def test_time_to_first_bytes(self):
|
||||
def check_post(event):
|
||||
self.assertEqual(event['event'], 'Time To First Bytes')
|
||||
total_duration = event['properties']['total_duration']
|
||||
resolve_duration = event['properties']['resolve_duration']
|
||||
head_blob_duration = event['properties']['head_blob_duration']
|
||||
sd_blob_duration = event['properties']['sd_blob_duration']
|
||||
self.assertFalse(event['properties']['added_fixed_peers'])
|
||||
self.assertTrue(total_duration >= resolve_duration + head_blob_duration + sd_blob_duration)
|
||||
|
||||
await self._test_time_to_first_bytes(check_post)
|
||||
|
||||
async def test_fixed_peer_delay_dht_peers_found(self):
|
||||
self.client_config.reflector_servers = [(self.server_from_client.address, self.server_from_client.tcp_port - 1)]
|
||||
server_from_client = None
|
||||
self.server_from_client, server_from_client = server_from_client, self.server_from_client
|
||||
|
||||
def after_setup():
|
||||
self.stream_manager.node.protocol.routing_table.get_peers = lambda: [server_from_client]
|
||||
|
||||
def check_post(event):
|
||||
self.assertEqual(event['event'], 'Time To First Bytes')
|
||||
total_duration = event['properties']['total_duration']
|
||||
resolve_duration = event['properties']['resolve_duration']
|
||||
head_blob_duration = event['properties']['head_blob_duration']
|
||||
sd_blob_duration = event['properties']['sd_blob_duration']
|
||||
|
||||
self.assertEqual(event['event'], 'Time To First Bytes')
|
||||
self.assertEqual(event['properties']['tried_peers_count'], 1)
|
||||
self.assertEqual(event['properties']['active_peer_count'], 1)
|
||||
self.assertEqual(event['properties']['use_fixed_peers'], True)
|
||||
self.assertEqual(event['properties']['added_fixed_peers'], True)
|
||||
self.assertEqual(event['properties']['fixed_peer_delay'], self.client_config.fixed_peer_delay)
|
||||
self.assertGreaterEqual(total_duration, resolve_duration + head_blob_duration + sd_blob_duration)
|
||||
|
||||
await self._test_time_to_first_bytes(check_post, after_setup=after_setup)
|
||||
|
||||
async def test_override_fixed_peer_delay_dht_disabled(self):
|
||||
self.client_config.reflector_servers = [(self.server_from_client.address, self.server_from_client.tcp_port - 1)]
|
||||
self.client_config.components_to_skip = ['dht', 'hash_announcer']
|
||||
self.client_config.fixed_peer_delay = 9001.0
|
||||
self.server_from_client = None
|
||||
|
||||
def check_post(event):
|
||||
total_duration = event['properties']['total_duration']
|
||||
resolve_duration = event['properties']['resolve_duration']
|
||||
head_blob_duration = event['properties']['head_blob_duration']
|
||||
sd_blob_duration = event['properties']['sd_blob_duration']
|
||||
|
||||
self.assertEqual(event['event'], 'Time To First Bytes')
|
||||
self.assertEqual(event['properties']['tried_peers_count'], 1)
|
||||
self.assertEqual(event['properties']['active_peer_count'], 1)
|
||||
self.assertEqual(event['properties']['use_fixed_peers'], True)
|
||||
self.assertEqual(event['properties']['added_fixed_peers'], True)
|
||||
self.assertEqual(event['properties']['fixed_peer_delay'], 0.0)
|
||||
self.assertGreaterEqual(total_duration, resolve_duration + head_blob_duration + sd_blob_duration)
|
||||
|
||||
start = self.loop.time()
|
||||
await self._test_time_to_first_bytes(check_post)
|
||||
self.assertTrue(self.loop.time() - start < 3)
|
||||
|
||||
async def test_no_peers_timeout(self):
|
||||
# FIXME: the download should ideally fail right away if there are no peers
|
||||
# to initialize the shortlist and fixed peers are disabled
|
||||
self.server_from_client = None
|
||||
self.client_config.download_timeout = 3.0
|
||||
|
||||
def check_post(event):
|
||||
self.assertEqual(event['event'], 'Time To First Bytes')
|
||||
self.assertEqual(event['properties']['error'], 'DownloadSDTimeout')
|
||||
self.assertEqual(event['properties']['tried_peers_count'], None)
|
||||
self.assertEqual(event['properties']['active_peer_count'], None)
|
||||
self.assertEqual(event['properties']['use_fixed_peers'], False)
|
||||
self.assertEqual(event['properties']['added_fixed_peers'], False)
|
||||
self.assertEqual(event['properties']['fixed_peer_delay'], None)
|
||||
|
||||
start = self.loop.time()
|
||||
await self._test_time_to_first_bytes(check_post, DownloadSDTimeout)
|
||||
duration = self.loop.time() - start
|
||||
self.assertTrue(4.0 >= duration >= 3.0)
|
||||
|
||||
async def test_download_stop_resume_delete(self):
|
||||
await self.setup_stream_manager()
|
||||
received = []
|
||||
expected_events = ['Time To First Bytes', 'Download Finished']
|
||||
|
||||
async def check_post(event):
|
||||
received.append(event['event'])
|
||||
|
||||
self.stream_manager.analytics_manager._post = check_post
|
||||
|
||||
self.assertSetEqual(self.stream_manager.streams, set())
|
||||
stream = await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager)
|
||||
stream_hash = stream.stream_hash
|
||||
self.assertSetEqual(self.stream_manager.streams, {stream})
|
||||
self.assertTrue(stream.running)
|
||||
self.assertFalse(stream.finished)
|
||||
self.assertTrue(os.path.isfile(os.path.join(self.client_dir, "test_file")))
|
||||
stored_status = await self.client_storage.run_and_return_one_or_none(
|
||||
"select status from file where stream_hash=?", stream_hash
|
||||
)
|
||||
self.assertEqual(stored_status, "running")
|
||||
|
||||
await self.stream_manager.stop_stream(stream)
|
||||
|
||||
self.assertFalse(stream.finished)
|
||||
self.assertFalse(stream.running)
|
||||
self.assertFalse(os.path.isfile(os.path.join(self.client_dir, "test_file")))
|
||||
stored_status = await self.client_storage.run_and_return_one_or_none(
|
||||
"select status from file where stream_hash=?", stream_hash
|
||||
)
|
||||
self.assertEqual(stored_status, "stopped")
|
||||
|
||||
await self.stream_manager.start_stream(stream)
|
||||
await stream.downloader.stream_finished_event.wait()
|
||||
await asyncio.sleep(0, loop=self.loop)
|
||||
self.assertTrue(stream.finished)
|
||||
self.assertFalse(stream.running)
|
||||
self.assertTrue(os.path.isfile(os.path.join(self.client_dir, "test_file")))
|
||||
stored_status = await self.client_storage.run_and_return_one_or_none(
|
||||
"select status from file where stream_hash=?", stream_hash
|
||||
)
|
||||
self.assertEqual(stored_status, "finished")
|
||||
|
||||
await self.stream_manager.delete_stream(stream, True)
|
||||
self.assertSetEqual(self.stream_manager.streams, set())
|
||||
self.assertFalse(os.path.isfile(os.path.join(self.client_dir, "test_file")))
|
||||
stored_status = await self.client_storage.run_and_return_one_or_none(
|
||||
"select status from file where stream_hash=?", stream_hash
|
||||
)
|
||||
self.assertEqual(stored_status, None)
|
||||
self.assertListEqual(expected_events, received)
|
||||
|
||||
async def _test_download_error_on_start(self, expected_error, timeout=None):
|
||||
with self.assertRaises(expected_error):
|
||||
await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager, timeout=timeout)
|
||||
|
||||
async def _test_download_error_analytics_on_start(self, expected_error, timeout=None):
|
||||
received = []
|
||||
|
||||
async def check_post(event):
|
||||
self.assertEqual("Time To First Bytes", event['event'])
|
||||
received.append(event['properties']['error'])
|
||||
|
||||
self.stream_manager.analytics_manager._post = check_post
|
||||
await self._test_download_error_on_start(expected_error, timeout)
|
||||
await asyncio.sleep(0, loop=self.loop)
|
||||
self.assertListEqual([expected_error.__name__], received)
|
||||
|
||||
async def test_insufficient_funds(self):
|
||||
fee = {
|
||||
'currency': 'LBC',
|
||||
'amount': 11.0,
|
||||
'address': 'bYFeMtSL7ARuG1iMpjFyrnTe4oJHSAVNXF',
|
||||
'version': '_0_0_1'
|
||||
}
|
||||
await self.setup_stream_manager(10.0, fee)
|
||||
await self._test_download_error_on_start(InsufficientFundsError)
|
||||
|
||||
async def test_fee_above_max_allowed(self):
|
||||
fee = {
|
||||
'currency': 'USD',
|
||||
'amount': 51.0,
|
||||
'address': 'bYFeMtSL7ARuG1iMpjFyrnTe4oJHSAVNXF',
|
||||
'version': '_0_0_1'
|
||||
}
|
||||
await self.setup_stream_manager(1000000.0, fee)
|
||||
await self._test_download_error_on_start(KeyFeeAboveMaxAllowed)
|
||||
|
||||
async def test_resolve_error(self):
|
||||
await self.setup_stream_manager()
|
||||
self.uri = "fake"
|
||||
await self._test_download_error_on_start(ResolveError)
|
||||
|
||||
async def test_download_sd_timeout(self):
|
||||
self.server.stop_server()
|
||||
await self.setup_stream_manager()
|
||||
await self._test_download_error_analytics_on_start(DownloadSDTimeout, timeout=1)
|
||||
|
||||
async def test_download_data_timeout(self):
|
||||
await self.setup_stream_manager()
|
||||
with open(os.path.join(self.server_dir, self.sd_hash), 'r') as sdf:
|
||||
head_blob_hash = json.loads(sdf.read())['blobs'][0]['blob_hash']
|
||||
self.server_blob_manager.delete_blob(head_blob_hash)
|
||||
await self._test_download_error_analytics_on_start(DownloadDataTimeout, timeout=1)
|
||||
|
||||
async def test_download_then_recover_stream_on_startup(self, old_sort=False):
|
||||
expected_analytics_events = [
|
||||
'Time To First Bytes',
|
||||
'Download Finished'
|
||||
]
|
||||
received_events = []
|
||||
|
||||
async def check_post(event):
|
||||
received_events.append(event['event'])
|
||||
|
||||
await self.setup_stream_manager(old_sort=old_sort)
|
||||
self.stream_manager.analytics_manager._post = check_post
|
||||
|
||||
self.assertSetEqual(self.stream_manager.streams, set())
|
||||
stream = await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager)
|
||||
await stream.downloader.stream_finished_event.wait()
|
||||
await asyncio.sleep(0, loop=self.loop)
|
||||
self.stream_manager.stop()
|
||||
self.client_blob_manager.stop()
|
||||
os.remove(os.path.join(self.client_blob_manager.blob_dir, stream.sd_hash))
|
||||
for blob in stream.descriptor.blobs[:-1]:
|
||||
os.remove(os.path.join(self.client_blob_manager.blob_dir, blob.blob_hash))
|
||||
await self.client_blob_manager.setup()
|
||||
await self.stream_manager.start()
|
||||
self.assertEqual(1, len(self.stream_manager.streams))
|
||||
self.assertEqual(stream.sd_hash, list(self.stream_manager.streams)[0].sd_hash)
|
||||
self.assertEqual('stopped', list(self.stream_manager.streams)[0].status)
|
||||
|
||||
sd_blob = self.client_blob_manager.get_blob(stream.sd_hash)
|
||||
self.assertTrue(sd_blob.file_exists)
|
||||
self.assertTrue(sd_blob.get_is_verified())
|
||||
self.assertListEqual(expected_analytics_events, received_events)
|
||||
|
||||
def test_download_then_recover_old_sort_stream_on_startup(self):
|
||||
return self.test_download_then_recover_stream_on_startup(old_sort=True)
|
||||
@@ -73,6 +73,7 @@ class TestAccount(AsyncioTestCase):
|
||||
def test_load_and_save_account(self):
|
||||
account_data = {
|
||||
'name': 'Main Account',
|
||||
'modified_on': 123.456,
|
||||
'seed':
|
||||
"carbon smart garage balance margin twelve chest sword toast envelope bottom stomac"
|
||||
"h absent",
|
||||
|
||||
@@ -12,5 +12,4 @@ setenv =
|
||||
TORBA_LEDGER=lbrynet.extras.wallet
|
||||
commands =
|
||||
orchstr8 download
|
||||
coverage run -p --source={envsitepackagesdir}/lbrynet -m unittest discover -vv integration.wallet
|
||||
coverage run -p --source={envsitepackagesdir}/lbrynet -m unittest discover -vv integration.cli
|
||||
coverage run -p --source={envsitepackagesdir}/lbrynet -m unittest discover -vv integration
|
||||
Reference in new issue
Block a user