Compare commits
274
Commits
v0.35.0rc2
...
misc_fixes
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
27afc759ff | ||
|
|
7abb28cb3e | ||
|
|
7d2f592bb5 | ||
|
|
1a182bc9bd | ||
|
|
324cbdcdb2 | ||
|
|
6c0c0ee322 | ||
|
|
1e54271d8a | ||
|
|
54b9ed6cb2 | ||
|
|
558a02d187 | ||
|
|
3567b331c0 | ||
|
|
9e4cf3c995 | ||
|
|
db1d547b19 | ||
|
|
c31efd780b | ||
|
|
7cdb7987e3 | ||
|
|
ec07037e88 | ||
|
|
9bb7b0571e | ||
|
|
d98162a118 | ||
|
|
c5ae59bff7 | ||
|
|
efbf2f49a9 | ||
|
|
77c69f661d | ||
|
|
2e09d2adfa | ||
|
|
8620a0e424 | ||
|
|
e2c123456b | ||
|
|
c9be79f062 | ||
|
|
a2a1ef34ab | ||
|
|
761a2b235c | ||
|
|
3ca2fee592 | ||
|
|
a480b2d25f | ||
|
|
d7032b12d7 | ||
|
|
84381ff76c | ||
|
|
007dd43861 | ||
|
|
1bc4e4b702 | ||
|
|
b91f27219d | ||
|
|
dc4560cc9a | ||
|
|
84b471d486 | ||
|
|
c6a3c05a0c | ||
|
|
dd665a758d | ||
|
|
32dc1297ec | ||
|
|
bef244ba77 | ||
|
|
3c231d6215 | ||
|
|
a212cf6ba4 | ||
|
|
bc060f1378 | ||
|
|
22c701fd50 | ||
|
|
4f0e8fce6e | ||
|
|
f642cfe9dd | ||
|
|
24e073680b | ||
|
|
b2f63a1545 | ||
|
|
f506b3e6d4 | ||
|
|
f7412514b3 | ||
|
|
bd6a609b30 | ||
|
|
4e32b69d1d | ||
|
|
1116c7f29e | ||
|
|
d80c671cf2 | ||
|
|
14b12cbea2 | ||
|
|
1f7feafb67 | ||
|
|
cbe689ea7f | ||
|
|
9099ee2e8e | ||
|
|
b134e0c9c9 | ||
|
|
f8c0e80cfc | ||
|
|
1f266ebdad | ||
|
|
b09e1f8825 | ||
|
|
c496278a5b | ||
|
|
cacc65d45e | ||
|
|
2247b354b3 | ||
|
|
43111f0497 | ||
|
|
6c142b4477 | ||
|
|
e8683a24bd | ||
|
|
a4d17d1705 | ||
|
|
a030e13548 | ||
|
|
32b8601c38 | ||
|
|
f5bd28bfdb | ||
|
|
6cbbe23c7c | ||
|
|
d7c2e4e9b3 | ||
|
|
d61ddbb950 | ||
|
|
cf7bb6a391 | ||
|
|
68d7328918 | ||
|
|
ff2985cc80 | ||
|
|
ac69fcbc5f | ||
|
|
609cf42868 | ||
|
|
e43b29fcd1 | ||
|
|
6594b7c50c | ||
|
|
971f2a1cef | ||
|
|
fc88261cc6 | ||
|
|
895b3992c9 | ||
|
|
41951d8ec2 | ||
|
|
ae2121c5c4 | ||
|
|
771fe935f4 | ||
|
|
60f7766cf7 | ||
|
|
63bd983012 | ||
|
|
7ffce1040c | ||
|
|
ac6835fa60 | ||
|
|
3cf5c536c0 | ||
|
|
85cc1a4763 | ||
|
|
180cfcfb72 | ||
|
|
864c1dd901 | ||
|
|
aeaa7ba986 | ||
|
|
b8325c1f89 | ||
|
|
a36936b5ff | ||
|
|
7cf245a0e9 | ||
|
|
9f83cce574 | ||
|
|
30b113d64a | ||
|
|
ccd2023d37 | ||
|
|
3127cd11cf | ||
|
|
6e483cba8d | ||
|
|
46c03c0cb2 | ||
|
|
c2ca0a9467 | ||
|
|
b350e74aca | ||
|
|
de9473f75a | ||
|
|
bfc107fdb3 | ||
|
|
6a2efa54ab | ||
|
|
72f0e1ca14 | ||
|
|
9a1262d557 | ||
|
|
706a8e52dc | ||
|
|
f3c8776878 | ||
|
|
539ea04039 | ||
|
|
8f96b07559 | ||
|
|
0a1e97fe23 | ||
|
|
da637b9c0e | ||
|
|
3618de1227 | ||
|
|
3cb8a63bf9 | ||
|
|
bad99de5b3 | ||
|
|
3d1efbc9bb | ||
|
|
7bdef3146b | ||
|
|
2145bfcabf | ||
|
|
b88832d0a9 | ||
|
|
db08882f20 | ||
|
|
6f52d36b22 | ||
|
|
056a629e87 | ||
|
|
1ffb99f9f9 | ||
|
|
a8e612773d | ||
|
|
67088b2ca2 | ||
|
|
dab49e812f | ||
|
|
0f3f38d8a3 | ||
|
|
7ba433c2e5 | ||
|
|
bb7649e8ad | ||
|
|
832537a5cf | ||
|
|
3234d70270 | ||
|
|
432fe444f8 | ||
|
|
861c25716e | ||
|
|
9ba6653475 | ||
|
|
8107e97304 | ||
|
|
e2663f6d57 | ||
|
|
35479dcbd2 | ||
|
|
3a997277aa | ||
|
|
a7d94f4514 | ||
|
|
a0e3338177 | ||
|
|
d5a353066c | ||
|
|
bc29fc4465 | ||
|
|
987f175fcc | ||
|
|
5b1398b588 | ||
|
|
873b58efc9 | ||
|
|
01a81fceed | ||
|
|
a14b0aa573 | ||
|
|
a1abb7c8fb | ||
|
|
46b166952b | ||
|
|
1375153fda | ||
|
|
1a14157d37 | ||
|
|
40b9fa38b0 | ||
|
|
4d0e9e0428 | ||
|
|
3a53ef5690 | ||
|
|
33a68b5cef | ||
|
|
63bf084a6a | ||
|
|
a882f2e8c5 | ||
|
|
119e0ec408 | ||
|
|
7d782d516a | ||
|
|
d3eac48035 | ||
|
|
60c9ae64b4 | ||
|
|
2d314dce60 | ||
|
|
287b89db66 | ||
|
|
7a160f7335 | ||
|
|
04ddee7d89 | ||
|
|
3a916a8e8e | ||
|
|
f125468ebf | ||
|
|
f0e17cff9a | ||
|
|
93267efe0b | ||
|
|
c663e5a3cf | ||
|
|
676f0015aa | ||
|
|
d44a79ada2 | ||
|
|
d7ed8ffaa6 | ||
|
|
b230d8fbcb | ||
|
|
7d2428fd17 | ||
|
|
4157045a41 | ||
|
|
58987857cf | ||
|
|
fa8858325c | ||
|
|
3357a419ab | ||
|
|
a782adea75 | ||
|
|
e8991e6d52 | ||
|
|
13bd81c2b6 | ||
|
|
21241b98a4 | ||
|
|
bf467e06c5 | ||
|
|
886ed6415a | ||
|
|
97db241109 | ||
|
|
216379918b | ||
|
|
fcab4eb9bd | ||
|
|
19f300cc45 | ||
|
|
26fec098e4 | ||
|
|
7824de655e | ||
|
|
6ef1b1dd59 | ||
|
|
9fae8f5153 | ||
|
|
f660691e78 | ||
|
|
bf60683ba6 | ||
|
|
68215699f7 | ||
|
|
1644ffbf22 | ||
|
|
71f5061848 | ||
|
|
61cadb5cbe | ||
|
|
2034104cfc | ||
|
|
633f725634 | ||
|
|
8bf2e41c95 | ||
|
|
fa9a4a75c6 | ||
|
|
90bef98bc3 | ||
|
|
35c43647aa | ||
|
|
449a583286 | ||
|
|
b1f813782a | ||
|
|
60080cbd19 | ||
|
|
665dba62c7 | ||
|
|
76dea9e89b | ||
|
|
ebc9ed01c5 | ||
|
|
55116763e9 | ||
|
|
e1a6b4e3d6 | ||
|
|
911ca8c37e | ||
|
|
7f5deaf6c8 | ||
|
|
0200756218 | ||
|
|
a63442f843 | ||
|
|
74b3da79aa | ||
|
|
b6a4589da5 | ||
|
|
fbbb0a9931 | ||
|
|
a94d65d522 | ||
|
|
fa2ead002e | ||
|
|
8dfee4867a | ||
|
|
6614ab8e79 | ||
|
|
d32c1dfbb3 | ||
|
|
9f42b958fa | ||
|
|
574a912f7d | ||
|
|
2cfcc3cd45 | ||
|
|
9af625e627 | ||
|
|
1871c94003 | ||
|
|
93220c3874 | ||
|
|
362ed3f1c8 | ||
|
|
62a182b8bb | ||
|
|
6f76d9e24a | ||
|
|
55e83fe05c | ||
|
|
2f0c7d6a3f | ||
|
|
59e1df9efa | ||
|
|
fd56dc68cd | ||
|
|
a0a810edc3 | ||
|
|
c1d3b5984d | ||
|
|
1fb719dd61 | ||
|
|
b87316fb02 | ||
|
|
65f3d78084 | ||
|
|
47bc1526f7 | ||
|
|
3acabec4bb | ||
|
|
36a62d3756 | ||
|
|
2e22473ce3 | ||
|
|
0b0fd64642 | ||
|
|
ca17d8597d | ||
|
|
d16e2716c8 | ||
|
|
89c80cedfd | ||
|
|
021e30ffde | ||
|
|
eb2cd8fec1 | ||
|
|
6356818557 | ||
|
|
8bf724a78b | ||
|
|
65d0a924c1 | ||
|
|
ef7bd00f22 | ||
|
|
3c1fdb5114 | ||
|
|
83411acfbd | ||
|
|
4230812f82 | ||
|
|
7b57ba96dd | ||
|
|
668840921f | ||
|
|
a3bd335f7f | ||
|
|
4e6bca1389 | ||
|
|
c4b15920ff | ||
|
|
2a55e583e7 | ||
|
|
2c4fde06c5 | ||
|
|
eca83d7f47 |
No files matched your search
@@ -1,3 +1,4 @@
|
||||
.DS_Store
|
||||
/build
|
||||
/dist
|
||||
/.tox
|
||||
@@ -8,3 +9,5 @@
|
||||
lbrynet.egg-info
|
||||
__pycache__
|
||||
_trial_temp/
|
||||
|
||||
/tests/integration/files
|
||||
+12
-1
@@ -35,6 +35,17 @@ jobs:
|
||||
- coverage combine tests/
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- name: "Run Examples"
|
||||
install:
|
||||
- pip install coverage
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install -e .
|
||||
script:
|
||||
- HOME=/tmp coverage run -p --source=lbrynet scripts/generate_json_api.py
|
||||
after_success:
|
||||
- coverage combine
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- stage: build
|
||||
name: "Windows"
|
||||
language: generic
|
||||
@@ -69,7 +80,7 @@ jobs:
|
||||
install:
|
||||
- pip3 install pyinstaller
|
||||
- pip3 install git+https://github.com/lbryio/torba.git
|
||||
- python scripts/set_build.py
|
||||
- python3 scripts/set_build.py
|
||||
- pip3 install -e .
|
||||
script:
|
||||
- pyinstaller -F -n lbrynet lbrynet/extras/cli.py
|
||||
|
||||
+1
-1
@@ -1,3 +1,3 @@
|
||||
## Contributing to LBRY
|
||||
|
||||
https://lbry.io/faq/contributing
|
||||
https://lbry.tech/contribute
|
||||
+11
-12
@@ -13,15 +13,6 @@ Running `lbrynet` from source requires Python 3.6 or higher (3.7 is preferred).
|
||||
|
||||
After installing python 3, you'll need to install some additional libraries depending on your operating system.
|
||||
|
||||
### Virtualenv
|
||||
|
||||
Once python 3 is installed run `python3 -m pip install virtualenv` to install virtualenv.
|
||||
|
||||
### Windows
|
||||
|
||||
Windows users will need to install `Visual C++ Build Tools`, which can be installed by [Microsoft Build Tools](Microsoft Build Tools 2015)
|
||||
|
||||
|
||||
### macOS
|
||||
|
||||
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
|
||||
@@ -29,15 +20,17 @@ macOS users will need to install [xcode command line tools](https://developer.xa
|
||||
Remaining dependencies can then be installed by running:
|
||||
|
||||
```
|
||||
brew install python3 protobuf
|
||||
brew install python protobuf
|
||||
```
|
||||
|
||||
Assistance installing Python3: https://docs.python-guide.org/starting/install3/osx/
|
||||
|
||||
### Linux
|
||||
|
||||
On Ubuntu (we recommend 18.04), install the following:
|
||||
|
||||
```
|
||||
sudo apt-get install build-essential python3.7 python3.7-dev git python-virtualenv libssl-dev python-protobuf
|
||||
sudo apt-get install build-essential python3.7 python3.7-dev git python3-venv libssl-dev python-protobuf
|
||||
```
|
||||
|
||||
On Raspbian, you will also need to install `python-pyparsing`.
|
||||
@@ -52,8 +45,14 @@ To install:
|
||||
git clone https://github.com/lbryio/lbry.git
|
||||
cd lbry
|
||||
|
||||
virtualenv lbry-venv --python=python3.7
|
||||
Creating venv:
|
||||
python -m venv lbry-venv
|
||||
|
||||
Activating lbry-venv on Linux/Mac:
|
||||
source lbry-venv/bin/activate
|
||||
|
||||
Activating lbry-venv on Windows:
|
||||
lbry-venv\Scripts\activate
|
||||
|
||||
python --version # Python 2 is not supported. Make sure you're on Python 3.7
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ Installing from source is also relatively painless. Full instructions are in [IN
|
||||
|
||||
## Contributing
|
||||
|
||||
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.io/faq/contributing) link.
|
||||
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.tech/contribute) link.
|
||||
|
||||
## License
|
||||
|
||||
@@ -41,11 +41,11 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
|
||||
|
||||
## Security
|
||||
|
||||
We take security seriously. Please contact security@lbry.io regarding any security issues. [Our GPG key is here](https://lbry.io/faq/gpg-key) if you need it.
|
||||
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.io/faq/gpg-key) if you need it.
|
||||
|
||||
## Contact
|
||||
|
||||
The primary contact for this project is [@eukreign](mailto:lex@lbry.io).
|
||||
The primary contact for this project is [@eukreign](mailto:lex@lbry.com).
|
||||
|
||||
## Additional information and links
|
||||
|
||||
|
||||
+2634
-1332
File diff suppressed because it is too large.
Load diff
@@ -12,12 +12,13 @@ blockchain_name: lbrycrd_main
|
||||
data_dir: /home/lbry/.lbrynet
|
||||
download_directory: /home/lbry/downloads
|
||||
|
||||
delete_blobs_on_remove: True
|
||||
save_blobs: true
|
||||
save_files: false
|
||||
dht_node_port: 4444
|
||||
peer_port: 3333
|
||||
use_upnp: True
|
||||
use_upnp: true
|
||||
|
||||
#components_to_skip:
|
||||
# - peer_protocol_server
|
||||
# - hash_announcer
|
||||
# - blob_server
|
||||
# - dht
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
__name__ = "lbrynet"
|
||||
__version__ = "0.35.0rc2"
|
||||
__version__ = "0.37.0rc6"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
+270
-132
@@ -4,6 +4,8 @@ import asyncio
|
||||
import binascii
|
||||
import logging
|
||||
import typing
|
||||
import contextlib
|
||||
from io import BytesIO
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, modes
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from cryptography.hazmat.primitives.padding import PKCS7
|
||||
@@ -21,10 +23,6 @@ log = logging.getLogger(__name__)
|
||||
_hexmatch = re.compile("^[a-f,0-9]+$")
|
||||
|
||||
|
||||
def is_valid_hashcharacter(char: str) -> bool:
|
||||
return len(char) == 1 and _hexmatch.match(char)
|
||||
|
||||
|
||||
def is_valid_blobhash(blobhash: str) -> bool:
|
||||
"""Checks whether the blobhash is the correct length and contains only
|
||||
valid characters (0-9, a-f)
|
||||
@@ -46,143 +44,74 @@ def encrypt_blob_bytes(key: bytes, iv: bytes, unencrypted: bytes) -> typing.Tupl
|
||||
return encrypted, digest.hexdigest()
|
||||
|
||||
|
||||
class BlobFile:
|
||||
"""
|
||||
A chunk of data available on the network which is specified by a hashsum
|
||||
def decrypt_blob_bytes(data: bytes, length: int, key: bytes, iv: bytes) -> bytes:
|
||||
if len(data) != length:
|
||||
raise ValueError("unexpected length")
|
||||
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
|
||||
unpadder = PKCS7(AES.block_size).unpadder()
|
||||
decryptor = cipher.decryptor()
|
||||
return unpadder.update(decryptor.update(data) + decryptor.finalize()) + unpadder.finalize()
|
||||
|
||||
This class is used to create blobs on the local filesystem
|
||||
when we already know the blob hash before hand (i.e., when downloading blobs)
|
||||
Also can be used for reading from blobs on the local filesystem
|
||||
"""
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, blob_hash: str,
|
||||
length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['BlobFile'], typing.Awaitable]] = None):
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise InvalidBlobHashError(blob_hash)
|
||||
class AbstractBlob:
|
||||
"""
|
||||
A chunk of data (up to 2MB) available on the network which is specified by a sha384 hash
|
||||
|
||||
This class is non-io specific
|
||||
"""
|
||||
__slots__ = [
|
||||
'loop',
|
||||
'blob_hash',
|
||||
'length',
|
||||
'blob_completed_callback',
|
||||
'blob_directory',
|
||||
'writers',
|
||||
'verified',
|
||||
'writing',
|
||||
'readers'
|
||||
]
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None):
|
||||
self.loop = loop
|
||||
self.blob_hash = blob_hash
|
||||
self.length = length
|
||||
self.blob_dir = blob_dir
|
||||
self.file_path = os.path.join(blob_dir, self.blob_hash)
|
||||
self.writers: typing.List[HashBlobWriter] = []
|
||||
|
||||
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||
self.finished_writing = asyncio.Event(loop=loop)
|
||||
self.blob_write_lock = asyncio.Lock(loop=loop)
|
||||
if self.file_exists:
|
||||
length = int(os.stat(os.path.join(blob_dir, blob_hash)).st_size)
|
||||
self.length = length
|
||||
self.verified.set()
|
||||
self.finished_writing.set()
|
||||
self.saved_verified_blob = False
|
||||
self.blob_completed_callback = blob_completed_callback
|
||||
self.blob_directory = blob_directory
|
||||
self.writers: typing.Dict[typing.Tuple[typing.Optional[str], typing.Optional[int]], HashBlobWriter] = {}
|
||||
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||
self.writing: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||
self.readers: typing.List[typing.BinaryIO] = []
|
||||
|
||||
@property
|
||||
def file_exists(self):
|
||||
return os.path.isfile(self.file_path)
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise InvalidBlobHashError(blob_hash)
|
||||
|
||||
def writer_finished(self, writer: HashBlobWriter):
|
||||
def callback(finished: asyncio.Future):
|
||||
def __del__(self):
|
||||
if self.writers or self.readers:
|
||||
log.warning("%s not closed before being garbage collected", self.blob_hash)
|
||||
self.close()
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
raise NotImplementedError()
|
||||
|
||||
@contextlib.contextmanager
|
||||
def reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
if not self.is_readable():
|
||||
raise OSError(f"{str(type(self))} not readable, {len(self.readers)} readers {len(self.writers)} writers")
|
||||
with self._reader_context() as reader:
|
||||
try:
|
||||
error = finished.exception()
|
||||
except Exception as err:
|
||||
error = err
|
||||
if writer in self.writers: # remove this download attempt
|
||||
self.writers.remove(writer)
|
||||
if not error: # the blob downloaded, cancel all the other download attempts and set the result
|
||||
while self.writers:
|
||||
other = self.writers.pop()
|
||||
other.finished.cancel()
|
||||
t = self.loop.create_task(self.save_verified_blob(writer, finished.result()))
|
||||
t.add_done_callback(lambda *_: self.finished_writing.set())
|
||||
return
|
||||
if isinstance(error, (InvalidBlobHashError, InvalidDataError)):
|
||||
log.debug("writer error downloading %s: %s", self.blob_hash[:8], str(error))
|
||||
elif not isinstance(error, (DownloadCancelledError, asyncio.CancelledError, asyncio.TimeoutError)):
|
||||
log.exception("something else")
|
||||
raise error
|
||||
return callback
|
||||
self.readers.append(reader)
|
||||
yield reader
|
||||
finally:
|
||||
if reader in self.readers:
|
||||
self.readers.remove(reader)
|
||||
|
||||
async def save_verified_blob(self, writer, verified_bytes: bytes):
|
||||
def _save_verified():
|
||||
# log.debug(f"write blob file {self.blob_hash[:8]} from {writer.peer.address}")
|
||||
if not self.saved_verified_blob and not os.path.isfile(self.file_path):
|
||||
if self.get_length() == len(verified_bytes):
|
||||
with open(self.file_path, 'wb') as write_handle:
|
||||
write_handle.write(verified_bytes)
|
||||
self.saved_verified_blob = True
|
||||
else:
|
||||
raise Exception("length mismatch")
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
raise NotImplementedError()
|
||||
|
||||
async with self.blob_write_lock:
|
||||
if self.verified.is_set():
|
||||
return
|
||||
await self.loop.run_in_executor(None, _save_verified)
|
||||
if self.blob_completed_callback:
|
||||
await self.blob_completed_callback(self)
|
||||
self.verified.set()
|
||||
|
||||
def open_for_writing(self) -> HashBlobWriter:
|
||||
if self.file_exists:
|
||||
raise OSError(f"File already exists '{self.file_path}'")
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
|
||||
self.writers.append(writer)
|
||||
fut.add_done_callback(self.writer_finished(writer))
|
||||
return writer
|
||||
|
||||
async def sendfile(self, writer: asyncio.StreamWriter) -> int:
|
||||
"""
|
||||
Read and send the file to the writer and return the number of bytes sent
|
||||
"""
|
||||
|
||||
with open(self.file_path, 'rb') as handle:
|
||||
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
|
||||
|
||||
def close(self):
|
||||
while self.writers:
|
||||
self.writers.pop().finished.cancel()
|
||||
|
||||
def delete(self):
|
||||
self.close()
|
||||
self.saved_verified_blob = False
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
self.verified.clear()
|
||||
self.finished_writing.clear()
|
||||
self.length = None
|
||||
|
||||
def decrypt(self, key: bytes, iv: bytes) -> bytes:
|
||||
"""
|
||||
Decrypt a BlobFile to plaintext bytes
|
||||
"""
|
||||
|
||||
with open(self.file_path, "rb") as f:
|
||||
buff = f.read()
|
||||
if len(buff) != self.length:
|
||||
raise ValueError("unexpected length")
|
||||
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
|
||||
unpadder = PKCS7(AES.block_size).unpadder()
|
||||
decryptor = cipher.decryptor()
|
||||
return unpadder.update(decryptor.update(buff) + decryptor.finalize()) + unpadder.finalize()
|
||||
|
||||
@classmethod
|
||||
async def create_from_unencrypted(cls, loop: asyncio.BaseEventLoop, blob_dir: str, key: bytes,
|
||||
iv: bytes, unencrypted: bytes, blob_num: int) -> BlobInfo:
|
||||
"""
|
||||
Create an encrypted BlobFile from plaintext bytes
|
||||
"""
|
||||
|
||||
blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted)
|
||||
length = len(blob_bytes)
|
||||
blob = cls(loop, blob_dir, blob_hash, length)
|
||||
writer = blob.open_for_writing()
|
||||
writer.write(blob_bytes)
|
||||
await blob.verified.wait()
|
||||
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash)
|
||||
|
||||
def set_length(self, length):
|
||||
def set_length(self, length) -> None:
|
||||
if self.length is not None and length == self.length:
|
||||
return
|
||||
if self.length is None and 0 <= length <= MAX_BLOB_SIZE:
|
||||
@@ -190,8 +119,217 @@ class BlobFile:
|
||||
return
|
||||
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s", self.length, length)
|
||||
|
||||
def get_length(self):
|
||||
def get_length(self) -> typing.Optional[int]:
|
||||
return self.length
|
||||
|
||||
def get_is_verified(self):
|
||||
def get_is_verified(self) -> bool:
|
||||
return self.verified.is_set()
|
||||
|
||||
def is_readable(self) -> bool:
|
||||
return self.verified.is_set()
|
||||
|
||||
def is_writeable(self) -> bool:
|
||||
return not self.writing.is_set()
|
||||
|
||||
def write_blob(self, blob_bytes: bytes):
|
||||
if not self.is_writeable():
|
||||
raise OSError("cannot open blob for writing")
|
||||
try:
|
||||
self.writing.set()
|
||||
self._write_blob(blob_bytes)
|
||||
finally:
|
||||
self.writing.clear()
|
||||
|
||||
def close(self):
|
||||
while self.writers:
|
||||
peer, writer = self.writers.popitem()
|
||||
if writer and writer.finished and not writer.finished.done() and not self.loop.is_closed():
|
||||
writer.finished.cancel()
|
||||
while self.readers:
|
||||
reader = self.readers.pop()
|
||||
if reader:
|
||||
reader.close()
|
||||
|
||||
def delete(self):
|
||||
self.close()
|
||||
self.verified.clear()
|
||||
self.length = None
|
||||
|
||||
async def sendfile(self, writer: asyncio.StreamWriter) -> int:
|
||||
"""
|
||||
Read and send the file to the writer and return the number of bytes sent
|
||||
"""
|
||||
|
||||
if not self.is_readable():
|
||||
raise OSError('blob files cannot be read')
|
||||
with self.reader_context() as handle:
|
||||
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
|
||||
|
||||
def decrypt(self, key: bytes, iv: bytes) -> bytes:
|
||||
"""
|
||||
Decrypt a BlobFile to plaintext bytes
|
||||
"""
|
||||
|
||||
with self.reader_context() as reader:
|
||||
return decrypt_blob_bytes(reader.read(), self.length, key, iv)
|
||||
|
||||
@classmethod
|
||||
async def create_from_unencrypted(
|
||||
cls, loop: asyncio.BaseEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
|
||||
unencrypted: bytes, blob_num: int,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None) -> BlobInfo:
|
||||
"""
|
||||
Create an encrypted BlobFile from plaintext bytes
|
||||
"""
|
||||
|
||||
blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted)
|
||||
length = len(blob_bytes)
|
||||
blob = cls(loop, blob_hash, length, blob_completed_callback, blob_dir)
|
||||
writer = blob.get_blob_writer()
|
||||
writer.write(blob_bytes)
|
||||
await blob.verified.wait()
|
||||
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash)
|
||||
|
||||
def save_verified_blob(self, verified_bytes: bytes):
|
||||
if self.verified.is_set():
|
||||
return
|
||||
if self.is_writeable():
|
||||
self._write_blob(verified_bytes)
|
||||
self.verified.set()
|
||||
if self.blob_completed_callback:
|
||||
self.blob_completed_callback(self)
|
||||
|
||||
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
|
||||
peer_port: typing.Optional[int] = None) -> HashBlobWriter:
|
||||
if (peer_address, peer_port) in self.writers and not self.writers[(peer_address, peer_port)].closed():
|
||||
raise OSError(f"attempted to download blob twice from {peer_address}:{peer_port}")
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
|
||||
self.writers[(peer_address, peer_port)] = writer
|
||||
|
||||
def remove_writer(_):
|
||||
if (peer_address, peer_port) in self.writers:
|
||||
del self.writers[(peer_address, peer_port)]
|
||||
|
||||
fut.add_done_callback(remove_writer)
|
||||
|
||||
def writer_finished_callback(finished: asyncio.Future):
|
||||
try:
|
||||
err = finished.exception()
|
||||
if err:
|
||||
raise err
|
||||
verified_bytes = finished.result()
|
||||
while self.writers:
|
||||
_, other = self.writers.popitem()
|
||||
if other is not writer:
|
||||
other.close_handle()
|
||||
self.save_verified_blob(verified_bytes)
|
||||
except (InvalidBlobHashError, InvalidDataError) as error:
|
||||
log.warning("writer error downloading %s: %s", self.blob_hash[:8], str(error))
|
||||
except (DownloadCancelledError, asyncio.CancelledError, asyncio.TimeoutError):
|
||||
pass
|
||||
|
||||
fut.add_done_callback(writer_finished_callback)
|
||||
return writer
|
||||
|
||||
|
||||
class BlobBuffer(AbstractBlob):
|
||||
"""
|
||||
An in-memory only blob
|
||||
"""
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None):
|
||||
self._verified_bytes: typing.Optional[BytesIO] = None
|
||||
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
if not self.is_readable():
|
||||
raise OSError("cannot open blob for reading")
|
||||
try:
|
||||
yield self._verified_bytes
|
||||
finally:
|
||||
if self._verified_bytes:
|
||||
self._verified_bytes.close()
|
||||
self._verified_bytes = None
|
||||
self.verified.clear()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
if self._verified_bytes:
|
||||
raise OSError("already have bytes for blob")
|
||||
self._verified_bytes = BytesIO(blob_bytes)
|
||||
|
||||
def delete(self):
|
||||
if self._verified_bytes:
|
||||
self._verified_bytes.close()
|
||||
self._verified_bytes = None
|
||||
return super().delete()
|
||||
|
||||
def __del__(self):
|
||||
super().__del__()
|
||||
if self._verified_bytes:
|
||||
self.delete()
|
||||
|
||||
|
||||
class BlobFile(AbstractBlob):
|
||||
"""
|
||||
A blob existing on the local file system
|
||||
"""
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None):
|
||||
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
|
||||
if not blob_directory or not os.path.isdir(blob_directory):
|
||||
raise OSError(f"invalid blob directory '{blob_directory}'")
|
||||
self.file_path = os.path.join(self.blob_directory, self.blob_hash)
|
||||
if self.file_exists:
|
||||
file_size = int(os.stat(self.file_path).st_size)
|
||||
if length and length != file_size:
|
||||
log.warning("expected %s to be %s bytes, file has %s", self.blob_hash, length, file_size)
|
||||
self.delete()
|
||||
else:
|
||||
self.length = file_size
|
||||
self.verified.set()
|
||||
|
||||
@property
|
||||
def file_exists(self):
|
||||
return os.path.isfile(self.file_path)
|
||||
|
||||
def is_writeable(self) -> bool:
|
||||
return super().is_writeable() and not os.path.isfile(self.file_path)
|
||||
|
||||
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
|
||||
peer_port: typing.Optional[str] = None) -> HashBlobWriter:
|
||||
if self.file_exists:
|
||||
raise OSError(f"File already exists '{self.file_path}'")
|
||||
return super().get_blob_writer(peer_address, peer_port)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
handle = open(self.file_path, 'rb')
|
||||
try:
|
||||
yield handle
|
||||
finally:
|
||||
handle.close()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
with open(self.file_path, 'wb') as f:
|
||||
f.write(blob_bytes)
|
||||
|
||||
def delete(self):
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
return super().delete()
|
||||
|
||||
@classmethod
|
||||
async def create_from_unencrypted(
|
||||
cls, loop: asyncio.BaseEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
|
||||
unencrypted: bytes, blob_num: int,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'],
|
||||
asyncio.Task]] = None) -> BlobInfo:
|
||||
if not blob_dir or not os.path.isdir(blob_dir):
|
||||
raise OSError(f"cannot create blob in directory: '{blob_dir}'")
|
||||
return await super().create_from_unencrypted(
|
||||
loop, blob_dir, key, iv, unencrypted, blob_num, blob_completed_callback
|
||||
)
|
||||
@@ -2,6 +2,13 @@ import typing
|
||||
|
||||
|
||||
class BlobInfo:
|
||||
__slots__ = [
|
||||
'blob_hash',
|
||||
'blob_num',
|
||||
'length',
|
||||
'iv',
|
||||
]
|
||||
|
||||
def __init__(self, blob_num: int, length: int, iv: str, blob_hash: typing.Optional[str] = None):
|
||||
self.blob_hash = blob_hash
|
||||
self.blob_num = blob_num
|
||||
|
||||
@@ -2,18 +2,19 @@ import os
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_file import BlobFile, is_valid_blobhash
|
||||
from lbrynet.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.dht.protocol.data_store import DictDataStore
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobFileManager:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, storage: SQLiteStorage,
|
||||
class BlobManager:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, storage: 'SQLiteStorage', config: 'Config',
|
||||
node_data_store: typing.Optional['DictDataStore'] = None):
|
||||
"""
|
||||
This class stores blobs on the hard disk
|
||||
@@ -27,16 +28,59 @@ class BlobFileManager:
|
||||
self._node_data_store = node_data_store
|
||||
self.completed_blob_hashes: typing.Set[str] = set() if not self._node_data_store\
|
||||
else self._node_data_store.completed_blobs
|
||||
self.blobs: typing.Dict[str, BlobFile] = {}
|
||||
self.blobs: typing.Dict[str, AbstractBlob] = {}
|
||||
self.config = config
|
||||
|
||||
def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None):
|
||||
if self.config.save_blobs:
|
||||
return BlobFile(
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
||||
)
|
||||
else:
|
||||
if is_valid_blobhash(blob_hash) and os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
return BlobFile(
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
||||
)
|
||||
return BlobBuffer(
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
||||
)
|
||||
|
||||
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
|
||||
if blob_hash in self.blobs:
|
||||
if self.config.save_blobs and isinstance(self.blobs[blob_hash], BlobBuffer):
|
||||
buffer = self.blobs.pop(blob_hash)
|
||||
if blob_hash in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.remove(blob_hash)
|
||||
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
|
||||
if buffer.is_readable():
|
||||
with buffer.reader_context() as reader:
|
||||
self.blobs[blob_hash].write_blob(reader.read())
|
||||
if length and self.blobs[blob_hash].length is None:
|
||||
self.blobs[blob_hash].set_length(length)
|
||||
else:
|
||||
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
|
||||
return self.blobs[blob_hash]
|
||||
|
||||
def is_blob_verified(self, blob_hash: str, length: typing.Optional[int] = None) -> bool:
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise ValueError(blob_hash)
|
||||
if blob_hash in self.blobs:
|
||||
return self.blobs[blob_hash].get_is_verified()
|
||||
if not os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
return False
|
||||
return self._get_blob(blob_hash, length).get_is_verified()
|
||||
|
||||
async def setup(self) -> bool:
|
||||
def get_files_in_blob_dir() -> typing.Set[str]:
|
||||
if not self.blob_dir:
|
||||
return set()
|
||||
return {
|
||||
item.name for item in os.scandir(self.blob_dir) if is_valid_blobhash(item.name)
|
||||
}
|
||||
|
||||
in_blobfiles_dir = await self.loop.run_in_executor(None, get_files_in_blob_dir)
|
||||
self.completed_blob_hashes.update(await self.storage.sync_missing_blobs(in_blobfiles_dir))
|
||||
to_add = await self.storage.sync_missing_blobs(in_blobfiles_dir)
|
||||
if to_add:
|
||||
self.completed_blob_hashes.update(to_add)
|
||||
return True
|
||||
|
||||
def stop(self):
|
||||
@@ -45,37 +89,31 @@ class BlobFileManager:
|
||||
blob.close()
|
||||
self.completed_blob_hashes.clear()
|
||||
|
||||
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
|
||||
if blob_hash in self.blobs:
|
||||
if length and self.blobs[blob_hash].length is None:
|
||||
self.blobs[blob_hash].set_length(length)
|
||||
else:
|
||||
self.blobs[blob_hash] = BlobFile(self.loop, self.blob_dir, blob_hash, length, self.blob_completed)
|
||||
return self.blobs[blob_hash]
|
||||
|
||||
def get_stream_descriptor(self, sd_hash):
|
||||
return StreamDescriptor.from_stream_descriptor_blob(self.loop, self.blob_dir, self.get_blob(sd_hash))
|
||||
|
||||
async def blob_completed(self, blob: BlobFile):
|
||||
def blob_completed(self, blob: AbstractBlob) -> asyncio.Task:
|
||||
if blob.blob_hash is None:
|
||||
raise Exception("Blob hash is None")
|
||||
if not blob.length:
|
||||
raise Exception("Blob has a length of 0")
|
||||
if blob.blob_hash not in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.add(blob.blob_hash)
|
||||
await self.storage.add_completed_blob(blob.blob_hash, blob.length)
|
||||
if isinstance(blob, BlobFile):
|
||||
if blob.blob_hash not in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.add(blob.blob_hash)
|
||||
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=True))
|
||||
else:
|
||||
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=False))
|
||||
|
||||
def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]:
|
||||
"""Returns of the blobhashes_to_check, which are valid"""
|
||||
blobs = [self.get_blob(b) for b in blob_hashes]
|
||||
return [blob.blob_hash for blob in blobs if blob.get_is_verified()]
|
||||
return [blob_hash for blob_hash in blob_hashes if self.is_blob_verified(blob_hash)]
|
||||
|
||||
def delete_blob(self, blob_hash: str):
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise Exception("invalid blob hash to delete")
|
||||
|
||||
if blob_hash not in self.blobs:
|
||||
if os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
if self.blob_dir and os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
os.remove(os.path.join(self.blob_dir, blob_hash))
|
||||
else:
|
||||
self.blobs.pop(blob_hash).delete()
|
||||
|
||||
@@ -44,16 +44,15 @@ class HashBlobWriter:
|
||||
self._hashsum.update(data)
|
||||
self.len_so_far += len(data)
|
||||
if self.len_so_far > expected_length:
|
||||
self.close_handle()
|
||||
self.finished.set_exception(InvalidDataError(
|
||||
f'Length so far is greater than the expected length. {self.len_so_far} to {expected_length}'
|
||||
))
|
||||
self.close_handle()
|
||||
return
|
||||
self.buffer.write(data)
|
||||
if self.len_so_far == expected_length:
|
||||
blob_hash = self.calculate_blob_hash()
|
||||
if blob_hash != self.expected_blob_hash:
|
||||
self.close_handle()
|
||||
self.finished.set_exception(InvalidBlobHashError(
|
||||
f"blob hash is {blob_hash} vs expected {self.expected_blob_hash}"
|
||||
))
|
||||
@@ -62,6 +61,8 @@ class HashBlobWriter:
|
||||
self.close_handle()
|
||||
|
||||
def close_handle(self):
|
||||
if not self.finished.done():
|
||||
self.finished.cancel()
|
||||
if self.buffer is not None:
|
||||
self.buffer.close()
|
||||
self.buffer = None
|
||||
@@ -4,8 +4,9 @@ import typing
|
||||
import binascii
|
||||
from lbrynet.error import InvalidBlobHashError, InvalidDataError
|
||||
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest
|
||||
from lbrynet.utils import cache_concurrent
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from lbrynet.blob.blob_file import AbstractBlob
|
||||
from lbrynet.blob.writer import HashBlobWriter
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -17,15 +18,18 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
self.peer_port: typing.Optional[int] = None
|
||||
self.peer_address: typing.Optional[str] = None
|
||||
self.peer_timeout = peer_timeout
|
||||
self.transport: asyncio.Transport = None
|
||||
self.transport: typing.Optional[asyncio.Transport] = None
|
||||
|
||||
self.writer: 'HashBlobWriter' = None
|
||||
self.blob: 'BlobFile' = None
|
||||
self.writer: typing.Optional['HashBlobWriter'] = None
|
||||
self.blob: typing.Optional['AbstractBlob'] = None
|
||||
|
||||
self._blob_bytes_received = 0
|
||||
self._response_fut: asyncio.Future = None
|
||||
self._response_fut: typing.Optional[asyncio.Future] = None
|
||||
self.buf = b''
|
||||
|
||||
# this is here to handle the race when the downloader is closed right as response_fut gets a result
|
||||
self.closed = asyncio.Event(loop=self.loop)
|
||||
|
||||
def data_received(self, data: bytes):
|
||||
log.debug("%s:%d -- got %s bytes -- %s bytes on buffer -- %s blob bytes received",
|
||||
self.peer_address, self.peer_port, len(data), len(self.buf), self._blob_bytes_received)
|
||||
@@ -35,6 +39,9 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return
|
||||
if not self._response_fut:
|
||||
log.warning("Protocol received data before expected, probable race on keep alive. Closing transport.")
|
||||
return self.close()
|
||||
if self._blob_bytes_received and not self.writer.closed():
|
||||
return self._write(data)
|
||||
|
||||
@@ -63,8 +70,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
# write blob bytes if we're writing a blob and have blob bytes to write
|
||||
self._write(response.blob_data)
|
||||
|
||||
|
||||
def _write(self, data):
|
||||
def _write(self, data: bytes):
|
||||
if len(data) > (self.blob.get_length() - self._blob_bytes_received):
|
||||
data = data[:(self.blob.get_length() - self._blob_bytes_received)]
|
||||
log.warning("got more than asked from %s:%d, probable sendfile bug", self.peer_address, self.peer_port)
|
||||
@@ -87,6 +93,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
:return: download success (bool), keep connection (bool)
|
||||
"""
|
||||
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
|
||||
blob_hash = self.blob.blob_hash
|
||||
try:
|
||||
msg = request.serialize()
|
||||
log.debug("send request to %s:%i -> %s", self.peer_address, self.peer_port, msg.decode())
|
||||
@@ -95,6 +102,10 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
availability_response = response.get_availability_response()
|
||||
price_response = response.get_price_response()
|
||||
blob_response = response.get_blob_response()
|
||||
if self.closed.is_set():
|
||||
msg = f"cancelled blob request for {blob_hash} immediately after we got a response"
|
||||
log.warning(msg)
|
||||
raise asyncio.CancelledError(msg)
|
||||
if (not blob_response or blob_response.error) and\
|
||||
(not availability_response or not availability_response.available_blobs):
|
||||
log.warning("%s not in availability response from %s:%i", self.blob.blob_hash, self.peer_address,
|
||||
@@ -133,6 +144,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
return self._blob_bytes_received, self.close()
|
||||
|
||||
def close(self):
|
||||
self.closed.set()
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
if self.writer and not self.writer.closed():
|
||||
@@ -145,18 +157,21 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
self.transport = None
|
||||
self.buf = b''
|
||||
|
||||
async def download_blob(self, blob: 'BlobFile') -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
if blob.get_is_verified() or blob.file_exists or blob.blob_write_lock.locked():
|
||||
async def download_blob(self, blob: 'AbstractBlob') -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
self.closed.clear()
|
||||
blob_hash = blob.blob_hash
|
||||
if blob.get_is_verified() or not blob.is_writeable():
|
||||
return 0, self.transport
|
||||
try:
|
||||
self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0
|
||||
self._blob_bytes_received = 0
|
||||
self.blob, self.writer = blob, blob.get_blob_writer(self.peer_address, self.peer_port)
|
||||
self._response_fut = asyncio.Future(loop=self.loop)
|
||||
return await self._download_blob()
|
||||
except OSError as e:
|
||||
log.error("race happened downloading from %s:%i", self.peer_address, self.peer_port)
|
||||
# i'm not sure how to fix this race condition - jack
|
||||
log.exception(e)
|
||||
return self._blob_bytes_received, self.transport
|
||||
log.warning("race happened downloading %s from %s:%i", blob_hash, self.peer_address, self.peer_port)
|
||||
# return self._blob_bytes_received, self.transport
|
||||
raise
|
||||
except asyncio.TimeoutError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
@@ -165,6 +180,10 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
except asyncio.CancelledError:
|
||||
self.close()
|
||||
raise
|
||||
finally:
|
||||
if self.writer and not self.writer.closed():
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
|
||||
def connection_made(self, transport: asyncio.Transport):
|
||||
self.transport = transport
|
||||
@@ -177,9 +196,10 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
self.close()
|
||||
|
||||
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', address: str, tcp_port: int,
|
||||
@cache_concurrent
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'AbstractBlob', address: str, tcp_port: int,
|
||||
peer_connect_timeout: float, blob_download_timeout: float,
|
||||
connected_transport: asyncio.Transport = None)\
|
||||
connected_transport: asyncio.Transport = None, connection_id: int = 0)\
|
||||
-> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
"""
|
||||
Returns [<downloaded blob>, <keep connection>]
|
||||
@@ -196,7 +216,7 @@ async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', address: s
|
||||
if not connected_transport:
|
||||
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
|
||||
peer_connect_timeout, loop=loop)
|
||||
if blob.get_is_verified() or blob.file_exists:
|
||||
if blob.get_is_verified() or not blob.is_writeable():
|
||||
# file exists but not verified means someone is writing right now, give it time, come back later
|
||||
return 0, connected_transport
|
||||
return await protocol.download_blob(blob)
|
||||
|
||||
@@ -1,21 +1,23 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet.utils import drain_tasks
|
||||
from lbrynet.utils import cache_concurrent
|
||||
from lbrynet.blob_exchange.client import request_blob
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
from lbrynet.blob.blob_file import AbstractBlob
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobDownloader:
|
||||
BAN_TIME = 10.0 # fixme: when connection manager gets implemented, move it out from here
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager',
|
||||
BAN_FACTOR = 2.0 # fixme: when connection manager gets implemented, move it out from here
|
||||
BAN_CAP = 30.0*60.0 # maximum time a node remains banned before being retried
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
peer_queue: asyncio.Queue):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
@@ -24,21 +26,16 @@ class BlobDownloader:
|
||||
self.active_connections: typing.Dict['KademliaPeer', asyncio.Task] = {} # active request_blob calls
|
||||
self.ignored: typing.Dict['KademliaPeer', int] = {}
|
||||
self.scores: typing.Dict['KademliaPeer', int] = {}
|
||||
self.failures: typing.Dict['KademliaPeer', int] = {}
|
||||
self.connections: typing.Dict['KademliaPeer', asyncio.Transport] = {}
|
||||
self.time_since_last_blob = loop.time()
|
||||
self.is_running = asyncio.Event(loop=self.loop)
|
||||
|
||||
def should_race_continue(self, blob: 'BlobFile'):
|
||||
def should_race_continue(self, blob: 'AbstractBlob'):
|
||||
if len(self.active_connections) >= self.config.max_connections_per_download:
|
||||
return False
|
||||
# if a peer won 3 or more blob races and is active as a downloader, stop the race so bandwidth improves
|
||||
# the safe net side is that any failure will reset the peer score, triggering the race back
|
||||
# TODO: this is a good idea for low bandwidth, but doesnt play nice on high bandwidth
|
||||
# for peer, task in self.active_connections.items():
|
||||
# if self.scores.get(peer, 0) >= 0 and self.rounds_won.get(peer, 0) >= 3 and not task.done():
|
||||
# return False
|
||||
return not (blob.get_is_verified() or blob.file_exists)
|
||||
return not (blob.get_is_verified() or not blob.is_writeable())
|
||||
|
||||
async def request_blob_from_peer(self, blob: 'BlobFile', peer: 'KademliaPeer'):
|
||||
async def request_blob_from_peer(self, blob: 'AbstractBlob', peer: 'KademliaPeer', connection_id: int = 0):
|
||||
if blob.get_is_verified():
|
||||
return
|
||||
self.scores[peer] = self.scores.get(peer, 0) - 1 # starts losing score, to account for cancelled ones
|
||||
@@ -46,92 +43,80 @@ class BlobDownloader:
|
||||
start = self.loop.time()
|
||||
bytes_received, transport = await request_blob(
|
||||
self.loop, blob, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
|
||||
self.config.blob_download_timeout, connected_transport=transport
|
||||
self.config.blob_download_timeout, connected_transport=transport, connection_id=connection_id
|
||||
)
|
||||
if bytes_received == blob.get_length():
|
||||
self.time_since_last_blob = self.loop.time()
|
||||
if not transport and peer not in self.ignored:
|
||||
self.ignored[peer] = self.loop.time()
|
||||
log.debug("drop peer %s:%i", peer.address, peer.tcp_port)
|
||||
self.failures[peer] = self.failures.get(peer, 0) + 1
|
||||
if peer in self.connections:
|
||||
del self.connections[peer]
|
||||
elif transport:
|
||||
log.debug("keep peer %s:%i", peer.address, peer.tcp_port)
|
||||
self.failures[peer] = 0
|
||||
self.connections[peer] = transport
|
||||
rough_speed = (bytes_received / (self.loop.time() - start)) if bytes_received else 0
|
||||
self.scores[peer] = rough_speed
|
||||
elapsed = self.loop.time() - start
|
||||
self.scores[peer] = bytes_received / elapsed if bytes_received and elapsed else 0
|
||||
|
||||
async def new_peer_or_finished(self, blob: 'BlobFile'):
|
||||
async def get_and_re_add_peers():
|
||||
try:
|
||||
new_peers = await asyncio.wait_for(self.peer_queue.get(), timeout=1.0)
|
||||
self.peer_queue.put_nowait(new_peers)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
tasks = [self.loop.create_task(get_and_re_add_peers()), self.loop.create_task(blob.verified.wait())]
|
||||
active_tasks = list(self.active_connections.values())
|
||||
try:
|
||||
await asyncio.wait(tasks + active_tasks, loop=self.loop, return_when='FIRST_COMPLETED')
|
||||
finally:
|
||||
drain_tasks(tasks)
|
||||
async def new_peer_or_finished(self):
|
||||
active_tasks = list(self.active_connections.values()) + [asyncio.sleep(1)]
|
||||
await asyncio.wait(active_tasks, loop=self.loop, return_when='FIRST_COMPLETED')
|
||||
|
||||
def cleanup_active(self):
|
||||
to_remove = [peer for (peer, task) in self.active_connections.items() if task.done()]
|
||||
for peer in to_remove:
|
||||
del self.active_connections[peer]
|
||||
self.clearbanned()
|
||||
|
||||
def clearbanned(self):
|
||||
now = self.loop.time()
|
||||
if now - self.time_since_last_blob > 60.0:
|
||||
return
|
||||
forgiven = [banned_peer for banned_peer, when in self.ignored.items() if now - when > self.BAN_TIME]
|
||||
self.peer_queue.put_nowait(forgiven)
|
||||
for banned_peer in forgiven:
|
||||
self.ignored.pop(banned_peer)
|
||||
self.ignored = dict((
|
||||
(peer, when) for (peer, when) in self.ignored.items()
|
||||
if (now - when) < min(self.BAN_CAP, (self.failures.get(peer, 0) ** self.BAN_FACTOR))
|
||||
))
|
||||
|
||||
async def download_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
|
||||
@cache_concurrent
|
||||
async def download_blob(self, blob_hash: str, length: typing.Optional[int] = None,
|
||||
connection_id: int = 0) -> 'AbstractBlob':
|
||||
blob = self.blob_manager.get_blob(blob_hash, length)
|
||||
if blob.get_is_verified():
|
||||
return blob
|
||||
self.is_running.set()
|
||||
try:
|
||||
while not blob.get_is_verified():
|
||||
batch: typing.List['KademliaPeer'] = []
|
||||
while not blob.get_is_verified() and self.is_running.is_set():
|
||||
batch: typing.Set['KademliaPeer'] = set()
|
||||
while not self.peer_queue.empty():
|
||||
batch.extend(self.peer_queue.get_nowait())
|
||||
batch.sort(key=lambda peer: self.scores.get(peer, 0), reverse=True)
|
||||
batch.update(self.peer_queue.get_nowait())
|
||||
if batch:
|
||||
self.peer_queue.put_nowait(list(batch))
|
||||
log.debug(
|
||||
"running, %d peers, %d ignored, %d active",
|
||||
len(batch), len(self.ignored), len(self.active_connections)
|
||||
)
|
||||
for peer in batch:
|
||||
for peer in sorted(batch, key=lambda peer: self.scores.get(peer, 0), reverse=True):
|
||||
if not self.should_race_continue(blob):
|
||||
break
|
||||
if peer not in self.active_connections and peer not in self.ignored:
|
||||
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
|
||||
t = self.loop.create_task(self.request_blob_from_peer(blob, peer))
|
||||
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id))
|
||||
self.active_connections[peer] = t
|
||||
await self.new_peer_or_finished(blob)
|
||||
await self.new_peer_or_finished()
|
||||
self.cleanup_active()
|
||||
if batch:
|
||||
self.peer_queue.put_nowait(set(batch).difference(self.ignored))
|
||||
else:
|
||||
self.clearbanned()
|
||||
blob.close()
|
||||
log.debug("downloaded %s", blob_hash[:8])
|
||||
return blob
|
||||
finally:
|
||||
while self.active_connections:
|
||||
self.active_connections.popitem()[1].cancel()
|
||||
blob.close()
|
||||
|
||||
def close(self):
|
||||
self.scores.clear()
|
||||
self.ignored.clear()
|
||||
self.is_running.clear()
|
||||
for transport in self.connections.values():
|
||||
transport.close()
|
||||
|
||||
|
||||
async def download_blob(loop, config: 'Config', blob_manager: 'BlobFileManager', node: 'Node',
|
||||
blob_hash: str) -> 'BlobFile':
|
||||
async def download_blob(loop, config: 'Config', blob_manager: 'BlobManager', node: 'Node',
|
||||
blob_hash: str) -> 'AbstractBlob':
|
||||
search_queue = asyncio.Queue(loop=loop, maxsize=config.max_connections_per_download)
|
||||
search_queue.put_nowait(blob_hash)
|
||||
peer_queue, accumulate_task = node.accumulate_peers(search_queue)
|
||||
|
||||
@@ -8,13 +8,13 @@ from lbrynet.blob_exchange.serialization import BlobAvailabilityResponse, BlobPr
|
||||
BlobPaymentAddressResponse
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobServerProtocol(asyncio.Protocol):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', lbrycrd_address: str):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str):
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
@@ -94,7 +94,7 @@ class BlobServerProtocol(asyncio.Protocol):
|
||||
|
||||
|
||||
class BlobServer:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', lbrycrd_address: str):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str):
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
|
||||
+8
-2
@@ -480,10 +480,12 @@ class Config(CLIConfig):
|
||||
# protocol timeouts
|
||||
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
|
||||
blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
|
||||
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 2.0)
|
||||
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0)
|
||||
node_rpc_timeout = Float("Timeout when making a DHT request", constants.rpc_timeout)
|
||||
|
||||
# blob announcement and download
|
||||
save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True)
|
||||
|
||||
announce_head_and_sd_only = Toggle(
|
||||
"Announce only the descriptor and first (rather than all) data blob for a stream to the DHT", True,
|
||||
previous_names=['announce_head_blobs_only']
|
||||
@@ -493,7 +495,7 @@ class Config(CLIConfig):
|
||||
previous_names=['concurrent_announcers']
|
||||
)
|
||||
max_connections_per_download = Integer(
|
||||
"Maximum number of peers to connect to while downloading a blob", 8,
|
||||
"Maximum number of peers to connect to while downloading a blob", 4,
|
||||
previous_names=['max_connections_per_stream']
|
||||
)
|
||||
fixed_peer_delay = Float(
|
||||
@@ -528,12 +530,16 @@ class Config(CLIConfig):
|
||||
('lbrynet4.lbry.io', 4444) # ASIA
|
||||
])
|
||||
|
||||
# comment server uses a single string because requests are made to the /api resource
|
||||
comment_server = String("Server to store user-generated metadata.", "http://comments.lbry.com:2903/api")
|
||||
|
||||
# blockchain
|
||||
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
|
||||
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
|
||||
cache_time = Integer("Time to cache resolved claims", 150) # TODO: use this
|
||||
|
||||
# daemon
|
||||
save_files = Toggle("Save downloaded files when calling `get` by default", True)
|
||||
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
|
||||
share_usage_data = Toggle(
|
||||
"Whether to share usage stats and diagnostic info with LBRY.", True,
|
||||
|
||||
@@ -13,55 +13,44 @@ class BlobAnnouncer:
|
||||
self.loop = loop
|
||||
self.node = node
|
||||
self.storage = storage
|
||||
self.pending_call: asyncio.Handle = None
|
||||
self.announce_task: asyncio.Task = None
|
||||
self.running = False
|
||||
self.announce_queue: typing.List[str] = []
|
||||
|
||||
async def _announce(self, batch_size: typing.Optional[int] = 10):
|
||||
if not batch_size:
|
||||
return
|
||||
if not self.node.joined.is_set():
|
||||
await self.node.joined.wait()
|
||||
blob_hashes = await self.storage.get_blobs_to_announce()
|
||||
if blob_hashes:
|
||||
self.announce_queue.extend(blob_hashes)
|
||||
log.info("%i blobs to announce", len(blob_hashes))
|
||||
batch = []
|
||||
while len(self.announce_queue):
|
||||
cnt = 0
|
||||
announced = []
|
||||
while self.announce_queue and cnt < batch_size:
|
||||
blob_hash = self.announce_queue.pop()
|
||||
announced.append(blob_hash)
|
||||
batch.append(self.node.announce_blob(blob_hash))
|
||||
cnt += 1
|
||||
to_await = []
|
||||
while batch:
|
||||
to_await.append(batch.pop())
|
||||
if to_await:
|
||||
await asyncio.gather(*tuple(to_await), loop=self.loop)
|
||||
await self.storage.update_last_announced_blobs(announced)
|
||||
log.info("announced %i blobs", len(announced))
|
||||
if self.running:
|
||||
self.pending_call = self.loop.call_later(60, self.announce, batch_size)
|
||||
async def _submit_announcement(self, blob_hash):
|
||||
try:
|
||||
peers = len(await self.node.announce_blob(blob_hash))
|
||||
if peers > 4:
|
||||
return blob_hash
|
||||
else:
|
||||
log.warning("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError):
|
||||
raise err
|
||||
log.warning("error announcing %s: %s", blob_hash[:8], str(err))
|
||||
|
||||
def announce(self, batch_size: typing.Optional[int] = 10):
|
||||
self.announce_task = self.loop.create_task(self._announce(batch_size))
|
||||
|
||||
async def _announce(self, batch_size: typing.Optional[int] = 10):
|
||||
while batch_size:
|
||||
if not self.node.joined.is_set():
|
||||
await self.node.joined.wait()
|
||||
self.announce_queue.extend(await self.storage.get_blobs_to_announce())
|
||||
log.debug("announcer task wake up, %d blobs to announce", len(self.announce_queue))
|
||||
while len(self.announce_queue):
|
||||
log.info("%i blobs to announce", len(self.announce_queue))
|
||||
announced = await asyncio.gather(*[
|
||||
self._submit_announcement(
|
||||
self.announce_queue.pop()) for _ in range(batch_size) if self.announce_queue
|
||||
], loop=self.loop)
|
||||
announced = list(filter(None, announced))
|
||||
if announced:
|
||||
await self.storage.update_last_announced_blobs(announced)
|
||||
log.info("announced %i blobs", len(announced))
|
||||
await asyncio.sleep(60)
|
||||
|
||||
def start(self, batch_size: typing.Optional[int] = 10):
|
||||
if self.running:
|
||||
raise Exception("already running")
|
||||
self.running = True
|
||||
self.announce(batch_size)
|
||||
assert not self.announce_task or self.announce_task.done(), "already running"
|
||||
self.announce_task = self.loop.create_task(self._announce(batch_size))
|
||||
|
||||
def stop(self):
|
||||
self.running = False
|
||||
if self.pending_call:
|
||||
if not self.pending_call.cancelled():
|
||||
self.pending_call.cancel()
|
||||
self.pending_call = None
|
||||
if self.announce_task:
|
||||
if not (self.announce_task.done() or self.announce_task.cancelled()):
|
||||
self.announce_task.cancel()
|
||||
self.announce_task = None
|
||||
if self.announce_task and not self.announce_task.done():
|
||||
self.announce_task.cancel()
|
||||
@@ -201,7 +201,6 @@ class Node:
|
||||
try:
|
||||
async with peer_generator as junction:
|
||||
yield junction
|
||||
await peer_generator.finished.wait()
|
||||
finally:
|
||||
if add_hashes_task and not (add_hashes_task.done() or add_hashes_task.cancelled()):
|
||||
add_hashes_task.cancel()
|
||||
|
||||
@@ -135,6 +135,15 @@ class PeerManager:
|
||||
|
||||
|
||||
class KademliaPeer:
|
||||
__slots__ = [
|
||||
'loop',
|
||||
'_node_id',
|
||||
'address',
|
||||
'udp_port',
|
||||
'tcp_port',
|
||||
'protocol_version',
|
||||
]
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, address: str, node_id: typing.Optional[bytes] = None,
|
||||
udp_port: typing.Optional[int] = None, tcp_port: typing.Optional[int] = None):
|
||||
if node_id is not None:
|
||||
|
||||
@@ -26,12 +26,11 @@ class AsyncGeneratorJunction:
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, queue: typing.Optional[asyncio.Queue] = None):
|
||||
self.loop = loop
|
||||
self.__iterator_queue = asyncio.Queue(loop=loop)
|
||||
self.result_queue = queue or asyncio.Queue(loop=loop)
|
||||
self.tasks: typing.List[asyncio.Task] = []
|
||||
self.running_iterators: typing.Dict[typing.AsyncGenerator, bool] = {}
|
||||
self.generator_queue: asyncio.Queue = asyncio.Queue(loop=self.loop)
|
||||
self.can_iterate = asyncio.Event(loop=self.loop)
|
||||
self.finished = asyncio.Event(loop=self.loop)
|
||||
|
||||
@property
|
||||
def running(self):
|
||||
@@ -42,15 +41,16 @@ class AsyncGeneratorJunction:
|
||||
try:
|
||||
async for item in iterator:
|
||||
self.result_queue.put_nowait(item)
|
||||
self.__iterator_queue.put_nowait(item)
|
||||
finally:
|
||||
self.running_iterators[iterator] = False
|
||||
if not self.running:
|
||||
self.__iterator_queue.put_nowait(StopAsyncIteration)
|
||||
|
||||
while True:
|
||||
async_gen: typing.Union[typing.AsyncGenerator, 'AsyncGeneratorType'] = await self.generator_queue.get()
|
||||
self.running_iterators[async_gen] = True
|
||||
self.tasks.append(self.loop.create_task(iterate(async_gen)))
|
||||
if not self.can_iterate.is_set():
|
||||
self.can_iterate.set()
|
||||
|
||||
def add_generator(self, async_gen: typing.Union[typing.AsyncGenerator, 'AsyncGeneratorType']):
|
||||
"""
|
||||
@@ -62,14 +62,10 @@ class AsyncGeneratorJunction:
|
||||
return self
|
||||
|
||||
async def __anext__(self):
|
||||
if not self.can_iterate.is_set():
|
||||
await self.can_iterate.wait()
|
||||
if not self.running:
|
||||
raise StopAsyncIteration()
|
||||
try:
|
||||
return await self.result_queue.get()
|
||||
finally:
|
||||
self.awaiting = None
|
||||
result = await self.__iterator_queue.get()
|
||||
if result is StopAsyncIteration:
|
||||
raise result
|
||||
return result
|
||||
|
||||
def aclose(self):
|
||||
async def _aclose():
|
||||
@@ -80,8 +76,6 @@ class AsyncGeneratorJunction:
|
||||
self.running_iterators[iterator] = False
|
||||
drain_tasks(self.tasks)
|
||||
raise StopAsyncIteration()
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return self.loop.create_task(_aclose())
|
||||
|
||||
async def __aenter__(self):
|
||||
|
||||
@@ -5,7 +5,7 @@ from itertools import chain
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.error import RemoteException
|
||||
from lbrynet.dht.error import RemoteException, TransportNotConnected
|
||||
from lbrynet.dht.protocol.distance import Distance
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -169,6 +169,8 @@ class IterativeFinder:
|
||||
log.warning(str(err))
|
||||
self.active.discard(peer)
|
||||
return
|
||||
except TransportNotConnected:
|
||||
return self.aclose()
|
||||
except RemoteException:
|
||||
return
|
||||
return await self._handle_probe_result(peer, response)
|
||||
@@ -215,7 +217,7 @@ class IterativeFinder:
|
||||
await self._search_round()
|
||||
if self.running:
|
||||
self.delayed_calls.append(self.loop.call_later(delay, self._search))
|
||||
except (asyncio.CancelledError, StopAsyncIteration):
|
||||
except (asyncio.CancelledError, StopAsyncIteration, TransportNotConnected):
|
||||
if self.running:
|
||||
self.loop.call_soon(self.aclose)
|
||||
|
||||
|
||||
@@ -524,6 +524,10 @@ class KademliaProtocol(DatagramProtocol):
|
||||
response = await asyncio.wait_for(response_fut, self.rpc_timeout)
|
||||
self.peer_manager.report_last_replied(peer.address, peer.udp_port)
|
||||
return response
|
||||
except asyncio.CancelledError:
|
||||
if not response_fut.done():
|
||||
response_fut.cancel()
|
||||
raise
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
if self.peer_manager.peer_is_good(peer) is False:
|
||||
@@ -540,7 +544,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
|
||||
async def _send(self, peer: 'KademliaPeer', message: typing.Union[RequestDatagram, ResponseDatagram,
|
||||
ErrorDatagram]):
|
||||
if not self.transport:
|
||||
if not self.transport or self.transport.is_closing():
|
||||
raise TransportNotConnected()
|
||||
|
||||
data = message.bencode()
|
||||
|
||||
@@ -32,6 +32,12 @@ class DownloadDataTimeout(Exception):
|
||||
self.download = download
|
||||
|
||||
|
||||
class ResolveTimeout(Exception):
|
||||
def __init__(self, uri):
|
||||
super().__init__(f'Failed to resolve "{uri}" within the timeout')
|
||||
self.uri = uri
|
||||
|
||||
|
||||
class RequestCanceledError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ def display(data):
|
||||
print(json.dumps(data, indent=2))
|
||||
|
||||
|
||||
async def execute_command(conf, method, params):
|
||||
async def execute_command(conf, method, params, callback=display):
|
||||
async with aiohttp.ClientSession() as session:
|
||||
try:
|
||||
message = {'method': method, 'params': params}
|
||||
@@ -33,12 +33,9 @@ async def execute_command(conf, method, params):
|
||||
try:
|
||||
data = await resp.json()
|
||||
if 'result' in data:
|
||||
display(data['result'])
|
||||
return callback(data['result'])
|
||||
elif 'error' in data:
|
||||
if 'message' in data['error']:
|
||||
display(data['error']['message'])
|
||||
else:
|
||||
display(data['error'])
|
||||
return callback(data['error'])
|
||||
except Exception as e:
|
||||
log.exception('Could not process response from server:', exc_info=e)
|
||||
except aiohttp.ClientConnectionError:
|
||||
|
||||
@@ -16,7 +16,7 @@ from lbrynet import utils
|
||||
from lbrynet.conf import HEADERS_FILE_SHA256_CHECKSUM
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.blob_announcer import BlobAnnouncer
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
from lbrynet.blob_exchange.server import BlobServer
|
||||
from lbrynet.stream.stream_manager import StreamManager
|
||||
from lbrynet.extras.daemon.Component import Component
|
||||
@@ -63,7 +63,7 @@ class DatabaseComponent(Component):
|
||||
|
||||
@staticmethod
|
||||
def get_current_db_revision():
|
||||
return 9
|
||||
return 11
|
||||
|
||||
@property
|
||||
def revision_filename(self):
|
||||
@@ -278,10 +278,10 @@ class BlobComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.blob_manager: BlobFileManager = None
|
||||
self.blob_manager: BlobManager = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[BlobFileManager]:
|
||||
def component(self) -> typing.Optional[BlobManager]:
|
||||
return self.blob_manager
|
||||
|
||||
async def start(self):
|
||||
@@ -291,8 +291,10 @@ class BlobComponent(Component):
|
||||
dht_node: Node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
if dht_node:
|
||||
data_store = dht_node.protocol.data_store
|
||||
self.blob_manager = BlobFileManager(asyncio.get_event_loop(), os.path.join(self.conf.data_dir, "blobfiles"),
|
||||
storage, data_store)
|
||||
blob_dir = os.path.join(self.conf.data_dir, 'blobfiles')
|
||||
if not os.path.isdir(blob_dir):
|
||||
os.mkdir(blob_dir)
|
||||
self.blob_manager = BlobManager(asyncio.get_event_loop(), blob_dir, storage, self.conf, data_store)
|
||||
return await self.blob_manager.setup()
|
||||
|
||||
async def stop(self):
|
||||
@@ -451,7 +453,7 @@ class PeerProtocolServerComponent(Component):
|
||||
async def start(self):
|
||||
log.info("start blob server")
|
||||
upnp = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
blob_manager: BlobFileManager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
blob_manager: BlobManager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
wallet: LbryWalletManager = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
peer_port = self.conf.tcp_port
|
||||
address = await wallet.get_unused_address()
|
||||
@@ -485,7 +487,7 @@ class UPnPComponent(Component):
|
||||
while True:
|
||||
if now:
|
||||
await self._maintain_redirects()
|
||||
await asyncio.sleep(360)
|
||||
await asyncio.sleep(360, loop=self.component_manager.loop)
|
||||
|
||||
async def _maintain_redirects(self):
|
||||
# setup the gateway if necessary
|
||||
@@ -528,7 +530,7 @@ class UPnPComponent(Component):
|
||||
self.upnp_redirects.update(upnp_redirects)
|
||||
except (asyncio.TimeoutError, UPnPError):
|
||||
self.upnp = None
|
||||
return self._maintain_redirects()
|
||||
return
|
||||
elif self.upnp: # check existing redirects are still active
|
||||
found = set()
|
||||
mappings = await self.upnp.get_redirects()
|
||||
|
||||
+803
-360
File diff suppressed because it is too large.
Load diff
@@ -1,27 +1,6 @@
|
||||
import aiohttp
|
||||
import logging
|
||||
from lbrynet.conf import Config
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
USER_AGENT = "AuthServiceProxy/0.1"
|
||||
TWISTED_SECURE_SESSION = "TWISTED_SECURE_SESSION"
|
||||
TWISTED_SESSION = "TWISTED_SESSION"
|
||||
LBRY_SECRET = "LBRY_SECRET"
|
||||
HTTP_TIMEOUT = 30
|
||||
from lbrynet.extras.cli import execute_command
|
||||
|
||||
|
||||
class JSONRPCException(Exception):
|
||||
def __init__(self, rpc_error):
|
||||
super().__init__()
|
||||
self.error = rpc_error
|
||||
|
||||
|
||||
async def daemon_rpc(conf: Config, method: str, *args, **kwargs):
|
||||
async with aiohttp.ClientSession() as session:
|
||||
message = {'method': method, 'params': [args, kwargs]}
|
||||
async with session.get(conf.api_connection_url, json=message) as resp:
|
||||
data = await resp.json()
|
||||
if 'result' in data:
|
||||
return data['result']
|
||||
elif 'error' in data:
|
||||
raise JSONRPCException(data['error'])
|
||||
def daemon_rpc(conf: Config, method: str, **kwargs):
|
||||
return execute_command(conf, method, kwargs, callback=lambda data: data)
|
||||
@@ -0,0 +1,31 @@
|
||||
import logging
|
||||
import aiohttp
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def rpc_body(method: str, rpc_id: any, **params) -> dict:
|
||||
return {'jsonrpc': '2.0', 'id': rpc_id, 'method': method, 'params': {**params}}
|
||||
|
||||
|
||||
async def jsonrpc_post(url: str, method: str, **params) -> any:
|
||||
clean = params.pop('clean', True)
|
||||
response = (await jsonrpc_batch(url, [rpc_body(method, 1, **params)]))[0]
|
||||
if clean:
|
||||
if 'error' in response:
|
||||
return response['error']
|
||||
return response['result']
|
||||
else:
|
||||
return response
|
||||
|
||||
|
||||
async def jsonrpc_batch(url: str, calls: list, batch_size: int = 50, clean: bool = False) -> list:
|
||||
headers = {'Content-Type': 'application/json'}
|
||||
complete = []
|
||||
batch_size = max(batch_size, 50)
|
||||
for i in range(0, len(calls), batch_size):
|
||||
async with aiohttp.request('POST', url, headers=headers, json=calls[i:i+batch_size]) as response:
|
||||
complete += await response.json()
|
||||
if clean:
|
||||
complete = [body['result'] if 'result' in body else None for body in complete]
|
||||
return complete
|
||||
@@ -5,34 +5,116 @@ from datetime import datetime
|
||||
from json import JSONEncoder
|
||||
from ecdsa import BadSignatureError
|
||||
from lbrynet.schema.claim import Claim
|
||||
from lbrynet.wallet.ledger import MainNetLedger
|
||||
from lbrynet.wallet.ledger import MainNetLedger, Account
|
||||
from lbrynet.wallet.transaction import Transaction, Output
|
||||
from lbrynet.wallet.dewies import dewies_to_lbc
|
||||
from lbrynet.stream.managed_stream import ManagedStream
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def encode_txo_doc():
|
||||
return {
|
||||
'txid': "hash of transaction in hex",
|
||||
'nout': "position in the transaction",
|
||||
'height': "block where transaction was recorded",
|
||||
'amount': "value of the txo as a decimal",
|
||||
'address': "address of who can spend the txo",
|
||||
'confirmations': "number of confirmed blocks",
|
||||
'is_change': "payment to change address, only available when it can be determined",
|
||||
'is_mine': "payment to one of your accounts, only available when it can be determined",
|
||||
'type': "one of 'claim', 'support' or 'payment'",
|
||||
'name': "when type is 'claim' or 'support', this is the claim name",
|
||||
'claim_id': "when type is 'claim' or 'support', this is the claim id",
|
||||
'claim_op': "when type is 'claim', this determines if it is 'create' or 'update'",
|
||||
'value': "when type is 'claim' or 'support' with payload, this is the decoded protobuf payload",
|
||||
'value_type': "determines the type of the 'value' field: 'channel', 'stream', etc",
|
||||
'protobuf': "hex encoded raw protobuf version of 'value' field",
|
||||
'permanent_url': "when type is 'claim' or 'support', this is the long permanent claim URL",
|
||||
'signing_channel': "for signed claims only, metadata of signing channel",
|
||||
'is_channel_signature_valid': "for signed claims only, whether signature is valid",
|
||||
}
|
||||
|
||||
|
||||
def encode_tx_doc():
|
||||
return {
|
||||
'txid': "hash of transaction in hex",
|
||||
'height': "block where transaction was recorded",
|
||||
'inputs': [encode_txo_doc()],
|
||||
'outputs': [encode_txo_doc()],
|
||||
'total_input': "sum of inputs as a decimal",
|
||||
'total_output': "sum of outputs, sans fee, as a decimal",
|
||||
'total_fee': "fee amount",
|
||||
'hex': "entire transaction encoded in hex",
|
||||
}
|
||||
|
||||
|
||||
def encode_account_doc():
|
||||
return {
|
||||
'id': 'account_id',
|
||||
'is_default': 'this account is used by default',
|
||||
'ledger': 'name of crypto currency and network',
|
||||
'name': 'optional account name',
|
||||
'seed': 'human friendly words from which account can be recreated',
|
||||
'encrypted': 'if account is encrypted',
|
||||
'private_key': 'extended private key',
|
||||
'public_key': 'extended public key',
|
||||
'address_generator': 'settings for generating addresses',
|
||||
'modified_on': 'date of last modification to account settings'
|
||||
}
|
||||
|
||||
|
||||
def encode_file_doc():
|
||||
return {
|
||||
'completed': '(bool) true if download is completed',
|
||||
'file_name': '(str) name of file',
|
||||
'download_directory': '(str) download directory',
|
||||
'points_paid': '(float) credit paid to download file',
|
||||
'stopped': '(bool) true if download is stopped',
|
||||
'stream_hash': '(str) stream hash of file',
|
||||
'stream_name': '(str) stream name',
|
||||
'suggested_file_name': '(str) suggested file name',
|
||||
'sd_hash': '(str) sd hash of file',
|
||||
'download_path': '(str) download path of file',
|
||||
'mime_type': '(str) mime type of file',
|
||||
'key': '(str) key attached to file',
|
||||
'total_bytes_lower_bound': '(int) lower bound file size in bytes',
|
||||
'total_bytes': '(int) file upper bound size in bytes',
|
||||
'written_bytes': '(int) written size in bytes',
|
||||
'blobs_completed': '(int) number of fully downloaded blobs',
|
||||
'blobs_in_stream': '(int) total blobs on stream',
|
||||
'blobs_remaining': '(int) total blobs remaining to download',
|
||||
'status': '(str) downloader status',
|
||||
'claim_id': '(str) None if claim is not found else the claim id',
|
||||
'txid': '(str) None if claim is not found else the transaction id',
|
||||
'nout': '(int) None if claim is not found else the transaction output index',
|
||||
'outpoint': '(str) None if claim is not found else the tx and output',
|
||||
'metadata': '(dict) None if claim is not found else the claim metadata',
|
||||
'channel_claim_id': '(str) None if claim is not found or not signed',
|
||||
'channel_name': '(str) None if claim is not found or not signed',
|
||||
'claim_name': '(str) None if claim is not found else the claim name'
|
||||
}
|
||||
|
||||
|
||||
class JSONResponseEncoder(JSONEncoder):
|
||||
|
||||
def __init__(self, *args, ledger: MainNetLedger, **kwargs):
|
||||
def __init__(self, *args, ledger: MainNetLedger, include_protobuf=False, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.ledger = ledger
|
||||
self.include_protobuf = include_protobuf
|
||||
|
||||
def default(self, obj): # pylint: disable=method-hidden
|
||||
if isinstance(obj, Account):
|
||||
return self.encode_account(obj)
|
||||
if isinstance(obj, ManagedStream):
|
||||
return self.encode_file(obj)
|
||||
if isinstance(obj, Transaction):
|
||||
return self.encode_transaction(obj)
|
||||
if isinstance(obj, Output):
|
||||
return self.encode_output(obj)
|
||||
if isinstance(obj, Claim):
|
||||
claim_dict = obj.to_dict()
|
||||
if obj.is_stream:
|
||||
claim_dict['stream']['sd_hash'] = obj.stream.sd_hash
|
||||
if 'fee' in claim_dict['stream'] and 'address' in claim_dict['stream']['fee']:
|
||||
claim_dict['stream']['fee']['address'] = obj.stream.fee.address
|
||||
elif obj.is_channel:
|
||||
claim_dict['channel']['public_key'] = obj.channel.public_key
|
||||
return claim_dict
|
||||
return self.encode_claim(obj)
|
||||
if isinstance(obj, datetime):
|
||||
return obj.strftime("%Y%m%dT%H:%M:%S")
|
||||
if isinstance(obj, Decimal):
|
||||
@@ -59,50 +141,56 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
output = {
|
||||
'txid': txo.tx_ref.id,
|
||||
'nout': txo.position,
|
||||
'height': tx_height,
|
||||
'amount': dewies_to_lbc(txo.amount),
|
||||
'address': txo.get_address(self.ledger),
|
||||
'height': tx_height,
|
||||
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height
|
||||
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
|
||||
'timestamp': self.ledger.headers[tx_height]['timestamp'] if tx_height > 0 else None
|
||||
}
|
||||
if txo.is_change is not None:
|
||||
output['is_change'] = txo.is_change
|
||||
if txo.is_my_account is not None:
|
||||
output['is_mine'] = txo.is_my_account
|
||||
|
||||
if txo.script.is_claim_name:
|
||||
output['type'] = 'claim'
|
||||
output['claim_op'] = 'create'
|
||||
elif txo.script.is_update_claim:
|
||||
output['type'] = 'claim'
|
||||
output['claim_op'] = 'update'
|
||||
elif txo.script.is_support_claim:
|
||||
output['type'] = 'support'
|
||||
else:
|
||||
output['type'] = 'payment'
|
||||
|
||||
if txo.script.is_claim_involved:
|
||||
output.update({
|
||||
'name': txo.claim_name,
|
||||
'claim_id': txo.claim_id,
|
||||
'permanent_url': txo.permanent_url,
|
||||
})
|
||||
|
||||
if txo.script.is_claim_name or txo.script.is_update_claim:
|
||||
claim = txo.claim
|
||||
output['value'] = claim
|
||||
if claim.is_signed:
|
||||
output['valid_signature'] = None
|
||||
if check_signature and txo.channel is not None:
|
||||
output['channel_name'] = txo.channel.claim_name
|
||||
output['value'] = txo.claim
|
||||
output['value_type'] = txo.claim.claim_type
|
||||
if self.include_protobuf:
|
||||
output['protobuf'] = hexlify(txo.claim.to_bytes())
|
||||
if txo.channel is not None:
|
||||
output['signing_channel'] = {
|
||||
'name': txo.channel.claim_name,
|
||||
'claim_id': txo.channel.claim_id,
|
||||
'value': txo.channel.claim
|
||||
}
|
||||
if check_signature and txo.claim.is_signed:
|
||||
output['is_channel_signature_valid'] = False
|
||||
try:
|
||||
output['valid_signature'] = txo.is_signed_by(txo.channel, self.ledger)
|
||||
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
|
||||
except BadSignatureError:
|
||||
output['valid_signature'] = False
|
||||
pass
|
||||
except ValueError:
|
||||
log.exception(
|
||||
'txo.id: %s, txo.channel.id:%s, output: %s',
|
||||
txo.id, txo.channel.id, output
|
||||
)
|
||||
output['valid_signature'] = False
|
||||
|
||||
if txo.script.is_claim_name:
|
||||
output['type'] = 'claim'
|
||||
elif txo.script.is_update_claim:
|
||||
output['type'] = 'update'
|
||||
elif txo.script.is_support_claim:
|
||||
output['type'] = 'support'
|
||||
else:
|
||||
output['type'] = 'basic'
|
||||
|
||||
return output
|
||||
|
||||
def encode_input(self, txi):
|
||||
@@ -110,3 +198,25 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
'txid': txi.txo_ref.tx_ref.id,
|
||||
'nout': txi.txo_ref.position
|
||||
}
|
||||
|
||||
def encode_account(self, account):
|
||||
result = account.to_dict()
|
||||
result['id'] = account.id
|
||||
result.pop('certificates', None)
|
||||
result['is_default'] = self.ledger.accounts[0] == account
|
||||
return result
|
||||
|
||||
def encode_file(self, managed_stream):
|
||||
file = managed_stream.as_dict()
|
||||
tx_height = managed_stream.stream_claim_info.height
|
||||
best_height = self.ledger.headers.height
|
||||
file.update({
|
||||
'height': tx_height,
|
||||
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
|
||||
'timestamp': self.ledger.headers[tx_height]['timestamp'] if tx_height > 0 else None
|
||||
})
|
||||
return file
|
||||
|
||||
@staticmethod
|
||||
def encode_claim(claim):
|
||||
return getattr(claim, claim.claim_type).to_dict()
|
||||
@@ -22,6 +22,10 @@ def migrate_db(conf, start, end):
|
||||
from .migrate7to8 import do_migration
|
||||
elif current == 8:
|
||||
from .migrate8to9 import do_migration
|
||||
elif current == 9:
|
||||
from .migrate9to10 import do_migration
|
||||
elif current == 10:
|
||||
from .migrate10to11 import do_migration
|
||||
else:
|
||||
raise Exception("DB migration of version {} to {} is not available".format(current,
|
||||
current+1))
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
import sqlite3
|
||||
import os
|
||||
import binascii
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
current_columns = []
|
||||
for col_info in cursor.execute("pragma table_info('file');").fetchall():
|
||||
current_columns.append(col_info[1])
|
||||
if 'content_fee' in current_columns or 'saved_file' in current_columns:
|
||||
connection.close()
|
||||
print("already migrated")
|
||||
return
|
||||
|
||||
cursor.execute(
|
||||
"pragma foreign_keys=off;"
|
||||
)
|
||||
|
||||
cursor.execute("""
|
||||
create table if not exists new_file (
|
||||
stream_hash text primary key not null references stream,
|
||||
file_name text,
|
||||
download_directory text,
|
||||
blob_data_rate real not null,
|
||||
status text not null,
|
||||
saved_file integer not null,
|
||||
content_fee text
|
||||
);
|
||||
""")
|
||||
for (stream_hash, file_name, download_dir, data_rate, status) in cursor.execute("select * from file").fetchall():
|
||||
saved_file = 0
|
||||
if download_dir != '{stream}' and file_name != '{stream}':
|
||||
try:
|
||||
if os.path.isfile(os.path.join(binascii.unhexlify(download_dir).decode(),
|
||||
binascii.unhexlify(file_name).decode())):
|
||||
saved_file = 1
|
||||
else:
|
||||
download_dir, file_name = None, None
|
||||
except Exception:
|
||||
download_dir, file_name = None, None
|
||||
else:
|
||||
download_dir, file_name = None, None
|
||||
cursor.execute(
|
||||
"insert into new_file values (?, ?, ?, ?, ?, ?, NULL)",
|
||||
(stream_hash, file_name, download_dir, data_rate, status, saved_file)
|
||||
)
|
||||
cursor.execute("drop table file")
|
||||
cursor.execute("alter table new_file rename to file")
|
||||
connection.commit()
|
||||
connection.close()
|
||||
@@ -0,0 +1,20 @@
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
query = "select stream_hash, sd_hash from main.stream"
|
||||
for stream_hash, sd_hash in cursor.execute(query).fetchall():
|
||||
head_blob_hash = cursor.execute(
|
||||
"select blob_hash from stream_blob where position = 0 and stream_hash = ?",
|
||||
(stream_hash,)
|
||||
).fetchone()
|
||||
if not head_blob_hash:
|
||||
continue
|
||||
cursor.execute("update blob set should_announce=1 where blob_hash in (?, ?)", (sd_hash, head_blob_hash[0],))
|
||||
connection.commit()
|
||||
connection.close()
|
||||
@@ -8,6 +8,7 @@ import time
|
||||
from torba.client.basedatabase import SQLiteMixin
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
||||
from lbrynet.wallet.transaction import Transaction
|
||||
from lbrynet.schema.claim import Claim
|
||||
from lbrynet.dht.constants import data_expiration
|
||||
from lbrynet.blob.blob_info import BlobInfo
|
||||
@@ -54,10 +55,6 @@ class StoredStreamClaim:
|
||||
def nout(self) -> typing.Optional[int]:
|
||||
return None if not self.outpoint else int(self.outpoint.split(":")[1])
|
||||
|
||||
@property
|
||||
def metadata(self) -> typing.Optional[typing.Dict]:
|
||||
return None if not self.claim else self.claim.claim_dict['stream']['metadata']
|
||||
|
||||
def as_dict(self) -> typing.Dict:
|
||||
return {
|
||||
"name": self.claim_name,
|
||||
@@ -118,8 +115,8 @@ def _batched_select(transaction, query, parameters, batch_size=900):
|
||||
def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
|
||||
files = []
|
||||
signed_claims = {}
|
||||
for (rowid, stream_hash, file_name, download_dir, data_rate, status, _, sd_hash, stream_key,
|
||||
stream_name, suggested_file_name, *claim_args) in _batched_select(
|
||||
for (rowid, stream_hash, file_name, download_dir, data_rate, status, saved_file, raw_content_fee, _,
|
||||
sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in _batched_select(
|
||||
transaction, "select file.rowid, file.*, stream.*, c.* "
|
||||
"from file inner join stream on file.stream_hash=stream.stream_hash "
|
||||
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
|
||||
@@ -145,7 +142,11 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
|
||||
"key": stream_key,
|
||||
"stream_name": stream_name, # hex
|
||||
"suggested_file_name": suggested_file_name, # hex
|
||||
"claim": claim
|
||||
"claim": claim,
|
||||
"saved_file": bool(saved_file),
|
||||
"content_fee": None if not raw_content_fee else Transaction(
|
||||
binascii.unhexlify(raw_content_fee)
|
||||
)
|
||||
}
|
||||
)
|
||||
for claim_name, claim_id in _batched_select(
|
||||
@@ -157,21 +158,12 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
|
||||
|
||||
|
||||
def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'):
|
||||
# add the head blob and set it to be announced
|
||||
transaction.execute(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
sd_blob.blob_hash, sd_blob.length, 0, 1, "pending", 0, 0,
|
||||
descriptor.blobs[0].blob_hash, descriptor.blobs[0].length, 0, 1, "pending", 0, 0
|
||||
)
|
||||
# add all blobs, except the last one, which is empty
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
[(blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
|
||||
for blob in (descriptor.blobs[:-1] if len(descriptor.blobs) > 1 else descriptor.blobs) + [sd_blob]]
|
||||
)
|
||||
# add the rest of the blobs with announcement off
|
||||
if len(descriptor.blobs) > 2:
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
[(blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
|
||||
for blob in descriptor.blobs[1:-1]]
|
||||
)
|
||||
# associate the blobs to the stream
|
||||
transaction.execute("insert or ignore into stream values (?, ?, ?, ?, ?)",
|
||||
(descriptor.stream_hash, sd_blob.blob_hash, descriptor.key,
|
||||
@@ -183,6 +175,11 @@ def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descripto
|
||||
[(descriptor.stream_hash, blob.blob_hash, blob.blob_num, blob.iv)
|
||||
for blob in descriptor.blobs]
|
||||
)
|
||||
# ensure should_announce is set regardless if insert was ignored
|
||||
transaction.execute(
|
||||
"update blob set should_announce=1 where blob_hash in (?, ?)",
|
||||
(sd_blob.blob_hash, descriptor.blobs[0].blob_hash,)
|
||||
)
|
||||
|
||||
|
||||
def delete_stream(transaction: sqlite3.Connection, descriptor: 'StreamDescriptor'):
|
||||
@@ -195,13 +192,21 @@ def delete_stream(transaction: sqlite3.Connection, descriptor: 'StreamDescriptor
|
||||
transaction.executemany("delete from blob where blob_hash=?", blob_hashes)
|
||||
|
||||
|
||||
def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: str, download_directory: str,
|
||||
data_payment_rate: float, status: str) -> int:
|
||||
def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: typing.Optional[str],
|
||||
download_directory: typing.Optional[str], data_payment_rate: float, status: str,
|
||||
content_fee: typing.Optional[Transaction]) -> int:
|
||||
if not file_name and not download_directory:
|
||||
encoded_file_name, encoded_download_dir = None, None
|
||||
else:
|
||||
encoded_file_name = binascii.hexlify(file_name.encode()).decode()
|
||||
encoded_download_dir = binascii.hexlify(download_directory.encode()).decode()
|
||||
transaction.execute(
|
||||
"insert or replace into file values (?, ?, ?, ?, ?)",
|
||||
(stream_hash, binascii.hexlify(file_name.encode()).decode(),
|
||||
binascii.hexlify(download_directory.encode()).decode(), data_payment_rate, status)
|
||||
"insert or replace into file values (?, ?, ?, ?, ?, ?, ?)",
|
||||
(stream_hash, encoded_file_name, encoded_download_dir, data_payment_rate, status,
|
||||
1 if (file_name and download_directory and os.path.isfile(os.path.join(download_directory, file_name))) else 0,
|
||||
None if not content_fee else binascii.hexlify(content_fee.raw).decode())
|
||||
)
|
||||
|
||||
return transaction.execute("select rowid from file where stream_hash=?", (stream_hash, )).fetchone()[0]
|
||||
|
||||
|
||||
@@ -250,10 +255,12 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
create table if not exists file (
|
||||
stream_hash text primary key not null references stream,
|
||||
file_name text not null,
|
||||
download_directory text not null,
|
||||
file_name text,
|
||||
download_directory text,
|
||||
blob_data_rate real not null,
|
||||
status text not null
|
||||
status text not null,
|
||||
saved_file integer not null,
|
||||
content_fee text
|
||||
);
|
||||
|
||||
create table if not exists content_claim (
|
||||
@@ -296,27 +303,28 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
# # # # # # # # # blob functions # # # # # # # # #
|
||||
|
||||
def add_completed_blob(self, blob_hash: str, length: int):
|
||||
def _add_blob(transaction: sqlite3.Connection):
|
||||
transaction.execute(
|
||||
async def add_blobs(self, *blob_hashes_and_lengths: typing.Tuple[str, int], finished=False):
|
||||
def _add_blobs(transaction: sqlite3.Connection):
|
||||
transaction.executemany(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
|
||||
(blob_hash, length, 0, 0, "pending", 0, 0)
|
||||
[
|
||||
(blob_hash, length, 0, 0, "pending" if not finished else "finished", 0, 0)
|
||||
for blob_hash, length in blob_hashes_and_lengths
|
||||
]
|
||||
)
|
||||
transaction.execute(
|
||||
"update blob set status='finished' where blob.blob_hash=?", (blob_hash, )
|
||||
)
|
||||
return self.db.run(_add_blob)
|
||||
if finished:
|
||||
transaction.executemany(
|
||||
"update blob set status='finished' where blob.blob_hash=?", [
|
||||
(blob_hash, ) for blob_hash, _ in blob_hashes_and_lengths
|
||||
]
|
||||
)
|
||||
return await self.db.run(_add_blobs)
|
||||
|
||||
def get_blob_status(self, blob_hash: str):
|
||||
return self.run_and_return_one_or_none(
|
||||
"select status from blob where blob_hash=?", blob_hash
|
||||
)
|
||||
|
||||
def add_known_blob(self, blob_hash: str, length: int):
|
||||
return self.db.execute(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)", (blob_hash, length, 0, 0, "pending", 0, 0)
|
||||
)
|
||||
|
||||
def should_announce(self, blob_hash: str):
|
||||
return self.run_and_return_one_or_none(
|
||||
"select should_announce from blob where blob_hash=?", blob_hash
|
||||
@@ -419,6 +427,26 @@ class SQLiteStorage(SQLiteMixin):
|
||||
}
|
||||
return self.db.run(_sync_blobs)
|
||||
|
||||
def sync_files_to_blobs(self):
|
||||
def _sync_blobs(transaction: sqlite3.Connection):
|
||||
transaction.executemany(
|
||||
"update file set status='stopped' where stream_hash=?",
|
||||
transaction.execute(
|
||||
"select distinct sb.stream_hash from stream_blob sb "
|
||||
"inner join blob b on b.blob_hash=sb.blob_hash and b.status=='pending'"
|
||||
).fetchall()
|
||||
)
|
||||
return self.db.run(_sync_blobs)
|
||||
|
||||
def set_files_as_streaming(self, stream_hashes: typing.List[str]):
|
||||
def _set_streaming(transaction: sqlite3.Connection):
|
||||
transaction.executemany(
|
||||
"update file set file_name=null, download_directory=null where stream_hash=?",
|
||||
[(stream_hash, ) for stream_hash in stream_hashes]
|
||||
)
|
||||
|
||||
return self.db.run(_set_streaming)
|
||||
|
||||
# # # # # # # # # stream functions # # # # # # # # #
|
||||
|
||||
async def stream_exists(self, sd_hash: str) -> bool:
|
||||
@@ -481,20 +509,53 @@ class SQLiteStorage(SQLiteMixin):
|
||||
"select stream_hash from stream where sd_hash = ?", sd_blob_hash
|
||||
)
|
||||
|
||||
def get_stream_info_for_sd_hash(self, sd_blob_hash):
|
||||
return self.run_and_return_one_or_none(
|
||||
"select stream_hash, stream_name, suggested_filename, stream_key from stream where sd_hash = ?",
|
||||
sd_blob_hash
|
||||
)
|
||||
|
||||
def delete_stream(self, descriptor: 'StreamDescriptor'):
|
||||
return self.db.run_with_foreign_keys_disabled(delete_stream, descriptor)
|
||||
|
||||
# # # # # # # # # file stuff # # # # # # # # #
|
||||
|
||||
def save_downloaded_file(self, stream_hash, file_name, download_directory,
|
||||
data_payment_rate) -> typing.Awaitable[int]:
|
||||
def save_downloaded_file(self, stream_hash: str, file_name: typing.Optional[str],
|
||||
download_directory: typing.Optional[str], data_payment_rate: float,
|
||||
content_fee: typing.Optional[Transaction] = None) -> typing.Awaitable[int]:
|
||||
return self.save_published_file(
|
||||
stream_hash, file_name, download_directory, data_payment_rate, status="running"
|
||||
stream_hash, file_name, download_directory, data_payment_rate, status="running",
|
||||
content_fee=content_fee
|
||||
)
|
||||
|
||||
def save_published_file(self, stream_hash: str, file_name: str, download_directory: str, data_payment_rate: float,
|
||||
status="finished") -> typing.Awaitable[int]:
|
||||
return self.db.run(store_file, stream_hash, file_name, download_directory, data_payment_rate, status)
|
||||
def save_published_file(self, stream_hash: str, file_name: typing.Optional[str],
|
||||
download_directory: typing.Optional[str], data_payment_rate: float,
|
||||
status: str = "finished",
|
||||
content_fee: typing.Optional[Transaction] = None) -> typing.Awaitable[int]:
|
||||
return self.db.run(store_file, stream_hash, file_name, download_directory, data_payment_rate, status,
|
||||
content_fee)
|
||||
|
||||
async def update_manually_removed_files_since_last_run(self):
|
||||
"""
|
||||
Update files that have been removed from the downloads directory since the last run
|
||||
"""
|
||||
def update_manually_removed_files(transaction: sqlite3.Connection):
|
||||
removed = []
|
||||
for (stream_hash, download_directory, file_name) in transaction.execute(
|
||||
"select stream_hash, download_directory, file_name from file where saved_file=1"
|
||||
).fetchall():
|
||||
if download_directory and file_name and os.path.isfile(
|
||||
os.path.join(binascii.unhexlify(download_directory).decode(),
|
||||
binascii.unhexlify(file_name).decode())):
|
||||
continue
|
||||
else:
|
||||
removed.append((stream_hash,))
|
||||
if removed:
|
||||
transaction.executemany(
|
||||
"update file set file_name=null, download_directory=null, saved_file=0 where stream_hash=?",
|
||||
removed
|
||||
)
|
||||
return await self.db.run(update_manually_removed_files)
|
||||
|
||||
def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]:
|
||||
return self.db.run(get_all_lbry_files)
|
||||
@@ -503,24 +564,45 @@ class SQLiteStorage(SQLiteMixin):
|
||||
log.debug("update file status %s -> %s", stream_hash, new_status)
|
||||
return self.db.execute("update file set status=? where stream_hash=?", (new_status, stream_hash))
|
||||
|
||||
def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: str, file_name: str):
|
||||
return self.db.execute("update file set download_directory=?, file_name=? where stream_hash=?", (
|
||||
binascii.hexlify(download_dir.encode()).decode(), binascii.hexlify(file_name.encode()).decode(),
|
||||
stream_hash
|
||||
async def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: typing.Optional[str],
|
||||
file_name: typing.Optional[str]):
|
||||
if not file_name or not download_dir:
|
||||
encoded_file_name, encoded_download_dir = None, None
|
||||
else:
|
||||
encoded_file_name = binascii.hexlify(file_name.encode()).decode()
|
||||
encoded_download_dir = binascii.hexlify(download_dir.encode()).decode()
|
||||
return await self.db.execute("update file set download_directory=?, file_name=? where stream_hash=?", (
|
||||
encoded_download_dir, encoded_file_name, stream_hash,
|
||||
))
|
||||
|
||||
async def recover_streams(self, descriptors_and_sds: typing.List[typing.Tuple['StreamDescriptor', 'BlobFile']],
|
||||
async def save_content_fee(self, stream_hash: str, content_fee: Transaction):
|
||||
return await self.db.execute("update file set content_fee=? where stream_hash=?", (
|
||||
binascii.hexlify(content_fee.raw), stream_hash,
|
||||
))
|
||||
|
||||
async def set_saved_file(self, stream_hash: str):
|
||||
return await self.db.execute("update file set saved_file=1 where stream_hash=?", (
|
||||
stream_hash,
|
||||
))
|
||||
|
||||
async def clear_saved_file(self, stream_hash: str):
|
||||
return await self.db.execute("update file set saved_file=0 where stream_hash=?", (
|
||||
stream_hash,
|
||||
))
|
||||
|
||||
async def recover_streams(self, descriptors_and_sds: typing.List[typing.Tuple['StreamDescriptor', 'BlobFile',
|
||||
typing.Optional[Transaction]]],
|
||||
download_directory: str):
|
||||
def _recover(transaction: sqlite3.Connection):
|
||||
stream_hashes = [d.stream_hash for d, s in descriptors_and_sds]
|
||||
for descriptor, sd_blob in descriptors_and_sds:
|
||||
stream_hashes = [x[0].stream_hash for x in descriptors_and_sds]
|
||||
for descriptor, sd_blob, content_fee in descriptors_and_sds:
|
||||
content_claim = transaction.execute(
|
||||
"select * from content_claim where stream_hash=?", (descriptor.stream_hash, )
|
||||
).fetchone()
|
||||
delete_stream(transaction, descriptor) # this will also delete the content claim
|
||||
store_stream(transaction, sd_blob, descriptor)
|
||||
store_file(transaction, descriptor.stream_hash, os.path.basename(descriptor.suggested_file_name),
|
||||
download_directory, 0.0, 'stopped')
|
||||
download_directory, 0.0, 'stopped', content_fee=content_fee)
|
||||
if content_claim:
|
||||
transaction.execute("insert or ignore into content_claim values (?, ?)", content_claim)
|
||||
transaction.executemany(
|
||||
@@ -539,16 +621,18 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
# # # # # # # # # support functions # # # # # # # # #
|
||||
|
||||
def save_supports(self, claim_id, supports):
|
||||
def save_supports(self, claim_id_to_supports: dict):
|
||||
# TODO: add 'address' to support items returned for a claim from lbrycrdd and lbryum-server
|
||||
def _save_support(transaction):
|
||||
transaction.execute("delete from support where claim_id=?", (claim_id,))
|
||||
for support in supports:
|
||||
transaction.execute(
|
||||
"insert into support values (?, ?, ?, ?)",
|
||||
("%s:%i" % (support['txid'], support['nout']), claim_id, lbc_to_dewies(support['amount']),
|
||||
support.get('address', ""))
|
||||
)
|
||||
bind = "({})".format(','.join(['?'] * len(claim_id_to_supports)))
|
||||
transaction.execute(f"delete from support where claim_id in {bind}", list(claim_id_to_supports.keys()))
|
||||
for claim_id, supports in claim_id_to_supports.items():
|
||||
for support in supports:
|
||||
transaction.execute(
|
||||
"insert into support values (?, ?, ?, ?)",
|
||||
("%s:%i" % (support['txid'], support['nout']), claim_id, lbc_to_dewies(support['amount']),
|
||||
support.get('address', ""))
|
||||
)
|
||||
return self.db.run(_save_support)
|
||||
|
||||
def get_supports(self, *claim_ids):
|
||||
@@ -576,7 +660,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
# # # # # # # # # claim functions # # # # # # # # #
|
||||
|
||||
async def save_claims(self, claim_infos):
|
||||
support_callbacks = []
|
||||
claim_id_to_supports = {}
|
||||
update_file_callbacks = []
|
||||
|
||||
def _save_claims(transaction):
|
||||
@@ -591,7 +675,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
sequence = claim_info['claim_sequence']
|
||||
certificate_id = claim_info['value'].signing_channel_id
|
||||
try:
|
||||
source_hash = claim_info['value'].stream.sd_hash
|
||||
source_hash = claim_info['value'].stream.source.sd_hash
|
||||
except (AttributeError, ValueError):
|
||||
source_hash = None
|
||||
serialized = binascii.hexlify(claim_info['value'].to_bytes())
|
||||
@@ -602,7 +686,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
# if this response doesn't have support info don't overwrite the existing
|
||||
# support info
|
||||
if 'supports' in claim_info:
|
||||
support_callbacks.append((claim_id, claim_info['supports']))
|
||||
claim_id_to_supports[claim_id] = claim_info['supports']
|
||||
if not source_hash:
|
||||
continue
|
||||
stream_hash = transaction.execute(
|
||||
@@ -632,23 +716,20 @@ class SQLiteStorage(SQLiteMixin):
|
||||
await self.db.run(_save_claims)
|
||||
if update_file_callbacks:
|
||||
await asyncio.wait(update_file_callbacks)
|
||||
if support_callbacks:
|
||||
await asyncio.wait([
|
||||
self.save_supports(*args) for args in support_callbacks
|
||||
])
|
||||
if claim_id_to_supports:
|
||||
await self.save_supports(claim_id_to_supports)
|
||||
|
||||
def save_claims_for_resolve(self, claim_infos):
|
||||
to_save = []
|
||||
to_save = {}
|
||||
for info in claim_infos:
|
||||
if 'value' in info:
|
||||
if info['value']:
|
||||
to_save.append(info)
|
||||
to_save[info['claim_id']] = info
|
||||
else:
|
||||
if 'certificate' in info and info['certificate']['value']:
|
||||
to_save.append(info['certificate'])
|
||||
if 'claim' in info and info['claim']['value']:
|
||||
to_save.append(info['claim'])
|
||||
return self.save_claims(to_save)
|
||||
for key in ('certificate', 'claim'):
|
||||
if info.get(key, {}).get('value'):
|
||||
to_save[info[key]['claim_id']] = info[key]
|
||||
return self.save_claims(to_save.values())
|
||||
|
||||
@staticmethod
|
||||
def _save_content_claim(transaction, claim_outpoint, stream_hash):
|
||||
@@ -671,7 +752,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
if not known_sd_hash:
|
||||
raise Exception("stream not found")
|
||||
# check the claim contains the same sd hash
|
||||
if known_sd_hash[0] != claim.stream.sd_hash:
|
||||
if known_sd_hash[0] != claim.stream.source.sd_hash:
|
||||
raise Exception("stream mismatch")
|
||||
|
||||
# if there is a current claim associated to the file, check that the new claim is an update to it
|
||||
|
||||
@@ -0,0 +1,521 @@
|
||||
import json
|
||||
import logging
|
||||
import os.path
|
||||
import hashlib
|
||||
from typing import Tuple, List
|
||||
from string import ascii_letters
|
||||
from decimal import Decimal, ROUND_UP
|
||||
from binascii import hexlify, unhexlify
|
||||
from google.protobuf.json_format import MessageToDict
|
||||
|
||||
from torba.client.hash import Base58
|
||||
from torba.client.constants import COIN
|
||||
|
||||
from lbrynet.schema.mime_types import guess_media_type
|
||||
from lbrynet.schema.base import Metadata, BaseMessageList
|
||||
from lbrynet.schema.types.v2.claim_pb2 import (
|
||||
Fee as FeeMessage,
|
||||
Location as LocationMessage,
|
||||
Language as LanguageMessage
|
||||
)
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def calculate_sha384_file_hash(file_path):
|
||||
sha384 = hashlib.sha384()
|
||||
with open(file_path, 'rb') as f:
|
||||
for chunk in iter(lambda: f.read(128 * sha384.block_size), b''):
|
||||
sha384.update(chunk)
|
||||
return sha384.digest()
|
||||
|
||||
|
||||
class Dimmensional(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def width(self) -> int:
|
||||
return self.message.width
|
||||
|
||||
@width.setter
|
||||
def width(self, width: int):
|
||||
self.message.width = width
|
||||
|
||||
@property
|
||||
def height(self) -> int:
|
||||
return self.message.height
|
||||
|
||||
@height.setter
|
||||
def height(self, height: int):
|
||||
self.message.height = height
|
||||
|
||||
@property
|
||||
def dimensions(self) -> Tuple[int, int]:
|
||||
return self.width, self.height
|
||||
|
||||
@dimensions.setter
|
||||
def dimensions(self, dimensions: Tuple[int, int]):
|
||||
self.message.width, self.message.height = dimensions
|
||||
|
||||
def _extract(self, file_metadata, field):
|
||||
try:
|
||||
setattr(self, field, file_metadata.getValues(field)[0])
|
||||
except:
|
||||
log.exception(f'Could not extract {field} from file metadata.')
|
||||
|
||||
def update(self, file_metadata=None, height=None, width=None):
|
||||
if height is not None:
|
||||
self.height = height
|
||||
elif file_metadata:
|
||||
self._extract(file_metadata, 'height')
|
||||
|
||||
if width is not None:
|
||||
self.width = width
|
||||
elif file_metadata:
|
||||
self._extract(file_metadata, 'width')
|
||||
|
||||
|
||||
class Playable(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def duration(self) -> int:
|
||||
return self.message.duration
|
||||
|
||||
@duration.setter
|
||||
def duration(self, duration: int):
|
||||
self.message.duration = duration
|
||||
|
||||
def update(self, file_metadata=None, duration=None):
|
||||
if duration is not None:
|
||||
self.duration = duration
|
||||
elif file_metadata:
|
||||
try:
|
||||
self.duration = file_metadata.getValues('duration')[0].seconds
|
||||
except:
|
||||
log.exception('Could not extract duration from file metadata.')
|
||||
|
||||
|
||||
class Image(Dimmensional):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
|
||||
class Audio(Playable):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
|
||||
class Video(Dimmensional, Playable):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def update(self, file_metadata=None, height=None, width=None, duration=None):
|
||||
Dimmensional.update(self, file_metadata, height, width)
|
||||
Playable.update(self, file_metadata, duration)
|
||||
|
||||
|
||||
class Source(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def update(self, file_path=None):
|
||||
if file_path is not None:
|
||||
self.name = os.path.basename(file_path)
|
||||
self.media_type, stream_type = guess_media_type(file_path)
|
||||
if not os.path.isfile(file_path):
|
||||
raise Exception(f"File does not exist: {file_path}")
|
||||
self.size = os.path.getsize(file_path)
|
||||
if self.size == 0:
|
||||
raise Exception(f"Cannot publish empty file: {file_path}")
|
||||
self.file_hash_bytes = calculate_sha384_file_hash(file_path)
|
||||
return stream_type
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self.message.name
|
||||
|
||||
@name.setter
|
||||
def name(self, name: str):
|
||||
self.message.name = name
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
return self.message.size
|
||||
|
||||
@size.setter
|
||||
def size(self, size: int):
|
||||
self.message.size = size
|
||||
|
||||
@property
|
||||
def media_type(self) -> str:
|
||||
return self.message.media_type
|
||||
|
||||
@media_type.setter
|
||||
def media_type(self, media_type: str):
|
||||
self.message.media_type = media_type
|
||||
|
||||
@property
|
||||
def file_hash(self) -> str:
|
||||
return hexlify(self.message.hash).decode()
|
||||
|
||||
@file_hash.setter
|
||||
def file_hash(self, file_hash: str):
|
||||
self.message.hash = unhexlify(file_hash.encode())
|
||||
|
||||
@property
|
||||
def file_hash_bytes(self) -> bytes:
|
||||
return self.message.hash
|
||||
|
||||
@file_hash_bytes.setter
|
||||
def file_hash_bytes(self, file_hash_bytes: bytes):
|
||||
self.message.hash = file_hash_bytes
|
||||
|
||||
@property
|
||||
def sd_hash(self) -> str:
|
||||
return hexlify(self.message.sd_hash).decode()
|
||||
|
||||
@sd_hash.setter
|
||||
def sd_hash(self, sd_hash: str):
|
||||
self.message.sd_hash = unhexlify(sd_hash.encode())
|
||||
|
||||
@property
|
||||
def sd_hash_bytes(self) -> bytes:
|
||||
return self.message.sd_hash
|
||||
|
||||
@sd_hash_bytes.setter
|
||||
def sd_hash_bytes(self, sd_hash: bytes):
|
||||
self.message.sd_hash = sd_hash
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
return self.message.url
|
||||
|
||||
@url.setter
|
||||
def url(self, url: str):
|
||||
self.message.url = url
|
||||
|
||||
|
||||
class Fee(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def update(self, address: str = None, currency: str = None, amount=None):
|
||||
if address is not None:
|
||||
self.address = address
|
||||
if currency is not None and amount is not None:
|
||||
currency = currency.lower()
|
||||
assert currency in ('lbc', 'btc', 'usd'), f'Unknown currency type: {currency}'
|
||||
setattr(self, currency, Decimal(amount))
|
||||
|
||||
@property
|
||||
def currency(self) -> str:
|
||||
return FeeMessage.Currency.Name(self.message.currency)
|
||||
|
||||
@property
|
||||
def address(self) -> str:
|
||||
return Base58.encode(self.message.address)
|
||||
|
||||
@address.setter
|
||||
def address(self, address: str):
|
||||
self.message.address = Base58.decode(address)
|
||||
|
||||
@property
|
||||
def address_bytes(self) -> bytes:
|
||||
return self.message.address
|
||||
|
||||
@address_bytes.setter
|
||||
def address_bytes(self, address: bytes):
|
||||
self.message.address = address
|
||||
|
||||
@property
|
||||
def amount(self) -> Decimal:
|
||||
if self.currency == 'LBC':
|
||||
return self.lbc
|
||||
if self.currency == 'BTC':
|
||||
return self.btc
|
||||
if self.currency == 'USD':
|
||||
return self.usd
|
||||
|
||||
DEWIES = Decimal(COIN)
|
||||
|
||||
@property
|
||||
def lbc(self) -> Decimal:
|
||||
if self.message.currency != FeeMessage.LBC:
|
||||
raise ValueError('LBC can only be returned for LBC fees.')
|
||||
return Decimal(self.message.amount / self.DEWIES)
|
||||
|
||||
@lbc.setter
|
||||
def lbc(self, amount: Decimal):
|
||||
self.dewies = int(amount * self.DEWIES)
|
||||
|
||||
@property
|
||||
def dewies(self) -> int:
|
||||
if self.message.currency != FeeMessage.LBC:
|
||||
raise ValueError('Dewies can only be returned for LBC fees.')
|
||||
return self.message.amount
|
||||
|
||||
@dewies.setter
|
||||
def dewies(self, amount: int):
|
||||
self.message.amount = amount
|
||||
self.message.currency = FeeMessage.LBC
|
||||
|
||||
SATOSHIES = Decimal(COIN)
|
||||
|
||||
@property
|
||||
def btc(self) -> Decimal:
|
||||
if self.message.currency != FeeMessage.BTC:
|
||||
raise ValueError('BTC can only be returned for BTC fees.')
|
||||
return Decimal(self.message.amount / self.SATOSHIES)
|
||||
|
||||
@btc.setter
|
||||
def btc(self, amount: Decimal):
|
||||
self.satoshis = int(amount * self.SATOSHIES)
|
||||
|
||||
@property
|
||||
def satoshis(self) -> int:
|
||||
if self.message.currency != FeeMessage.BTC:
|
||||
raise ValueError('Satoshies can only be returned for BTC fees.')
|
||||
return self.message.amount
|
||||
|
||||
@satoshis.setter
|
||||
def satoshis(self, amount: int):
|
||||
self.message.amount = amount
|
||||
self.message.currency = FeeMessage.BTC
|
||||
|
||||
PENNIES = Decimal('100.0')
|
||||
PENNY = Decimal('0.01')
|
||||
|
||||
@property
|
||||
def usd(self) -> Decimal:
|
||||
if self.message.currency != FeeMessage.USD:
|
||||
raise ValueError('USD can only be returned for USD fees.')
|
||||
return Decimal(self.message.amount / self.PENNIES)
|
||||
|
||||
@usd.setter
|
||||
def usd(self, amount: Decimal):
|
||||
self.pennies = int(amount.quantize(self.PENNY, ROUND_UP) * self.PENNIES)
|
||||
|
||||
@property
|
||||
def pennies(self) -> int:
|
||||
if self.message.currency != FeeMessage.USD:
|
||||
raise ValueError('Pennies can only be returned for USD fees.')
|
||||
return self.message.amount
|
||||
|
||||
@pennies.setter
|
||||
def pennies(self, amount: int):
|
||||
self.message.amount = amount
|
||||
self.message.currency = FeeMessage.USD
|
||||
|
||||
|
||||
class ClaimReference(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def claim_id(self) -> str:
|
||||
return hexlify(self.claim_hash[::-1]).decode()
|
||||
|
||||
@claim_id.setter
|
||||
def claim_id(self, claim_id: str):
|
||||
self.claim_hash = unhexlify(claim_id)[::-1]
|
||||
|
||||
@property
|
||||
def claim_hash(self) -> bytes:
|
||||
return self.message.claim_hash
|
||||
|
||||
@claim_hash.setter
|
||||
def claim_hash(self, claim_hash: bytes):
|
||||
self.message.claim_hash = claim_hash
|
||||
|
||||
|
||||
class ClaimList(BaseMessageList[ClaimReference]):
|
||||
|
||||
__slots__ = ()
|
||||
item_class = ClaimReference
|
||||
|
||||
@property
|
||||
def _message(self):
|
||||
return self.message.claim_references
|
||||
|
||||
def append(self, value):
|
||||
self.add().claim_id = value
|
||||
|
||||
@property
|
||||
def ids(self) -> List[str]:
|
||||
return [c.claim_id for c in self]
|
||||
|
||||
|
||||
class Language(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def langtag(self) -> str:
|
||||
langtag = []
|
||||
if self.language:
|
||||
langtag.append(self.language)
|
||||
if self.script:
|
||||
langtag.append(self.script)
|
||||
if self.region:
|
||||
langtag.append(self.region)
|
||||
return '-'.join(langtag)
|
||||
|
||||
@langtag.setter
|
||||
def langtag(self, langtag: str):
|
||||
parts = langtag.split('-')
|
||||
self.language = parts.pop(0)
|
||||
if parts and len(parts[0]) == 4:
|
||||
self.script = parts.pop(0)
|
||||
if parts and len(parts[0]) == 2:
|
||||
self.region = parts.pop(0)
|
||||
assert not parts, f"Failed to parse language tag: {langtag}"
|
||||
|
||||
@property
|
||||
def language(self) -> str:
|
||||
if self.message.language:
|
||||
return LanguageMessage.Language.Name(self.message.language)
|
||||
|
||||
@language.setter
|
||||
def language(self, language: str):
|
||||
self.message.language = LanguageMessage.Language.Value(language)
|
||||
|
||||
@property
|
||||
def script(self) -> str:
|
||||
if self.message.script:
|
||||
return LanguageMessage.Script.Name(self.message.script)
|
||||
|
||||
@script.setter
|
||||
def script(self, script: str):
|
||||
self.message.script = LanguageMessage.Script.Value(script)
|
||||
|
||||
@property
|
||||
def region(self) -> str:
|
||||
if self.message.region:
|
||||
return LocationMessage.Country.Name(self.message.region)
|
||||
|
||||
@region.setter
|
||||
def region(self, region: str):
|
||||
self.message.region = LocationMessage.Country.Value(region)
|
||||
|
||||
|
||||
class LanguageList(BaseMessageList[Language]):
|
||||
__slots__ = ()
|
||||
item_class = Language
|
||||
|
||||
def append(self, value: str):
|
||||
self.add().langtag = value
|
||||
|
||||
|
||||
class Location(Metadata):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def from_value(self, value):
|
||||
if isinstance(value, str) and value.startswith('{'):
|
||||
value = json.loads(value)
|
||||
|
||||
if isinstance(value, dict):
|
||||
for key, val in value.items():
|
||||
setattr(self, key, val)
|
||||
|
||||
elif isinstance(value, str):
|
||||
parts = value.split(':')
|
||||
if len(parts) > 2 or (parts[0] and parts[0][0] in ascii_letters):
|
||||
country = parts and parts.pop(0)
|
||||
if country:
|
||||
self.country = country
|
||||
state = parts and parts.pop(0)
|
||||
if state:
|
||||
self.state = state
|
||||
city = parts and parts.pop(0)
|
||||
if city:
|
||||
self.city = city
|
||||
code = parts and parts.pop(0)
|
||||
if code:
|
||||
self.code = code
|
||||
latitude = parts and parts.pop(0)
|
||||
if latitude:
|
||||
self.latitude = latitude
|
||||
longitude = parts and parts.pop(0)
|
||||
if longitude:
|
||||
self.longitude = longitude
|
||||
|
||||
else:
|
||||
raise ValueError(f'Could not parse country value: {value}')
|
||||
|
||||
def to_dict(self):
|
||||
d = MessageToDict(self.message)
|
||||
if self.message.longitude:
|
||||
d['longitude'] = self.longitude
|
||||
if self.message.latitude:
|
||||
d['latitude'] = self.latitude
|
||||
return d
|
||||
|
||||
@property
|
||||
def country(self) -> str:
|
||||
if self.message.country:
|
||||
return LocationMessage.Country.Name(self.message.country)
|
||||
|
||||
@country.setter
|
||||
def country(self, country: str):
|
||||
self.message.country = LocationMessage.Country.Value(country)
|
||||
|
||||
@property
|
||||
def state(self) -> str:
|
||||
return self.message.state
|
||||
|
||||
@state.setter
|
||||
def state(self, state: str):
|
||||
self.message.state = state
|
||||
|
||||
@property
|
||||
def city(self) -> str:
|
||||
return self.message.city
|
||||
|
||||
@city.setter
|
||||
def city(self, city: str):
|
||||
self.message.city = city
|
||||
|
||||
@property
|
||||
def code(self) -> str:
|
||||
return self.message.code
|
||||
|
||||
@code.setter
|
||||
def code(self, code: str):
|
||||
self.message.code = code
|
||||
|
||||
GPS_PRECISION = Decimal('10000000')
|
||||
|
||||
@property
|
||||
def latitude(self) -> str:
|
||||
if self.message.latitude:
|
||||
return str(Decimal(self.message.latitude) / self.GPS_PRECISION)
|
||||
|
||||
@latitude.setter
|
||||
def latitude(self, latitude: str):
|
||||
latitude = Decimal(latitude)
|
||||
assert -90 <= latitude <= 90, "Latitude must be between -90 and 90 degrees."
|
||||
self.message.latitude = int(latitude * self.GPS_PRECISION)
|
||||
|
||||
@property
|
||||
def longitude(self) -> str:
|
||||
if self.message.longitude:
|
||||
return str(Decimal(self.message.longitude) / self.GPS_PRECISION)
|
||||
|
||||
@longitude.setter
|
||||
def longitude(self, longitude: str):
|
||||
longitude = Decimal(longitude)
|
||||
assert -180 <= longitude <= 180, "Longitude must be between -180 and 180 degrees."
|
||||
self.message.longitude = int(longitude * self.GPS_PRECISION)
|
||||
|
||||
|
||||
class LocationList(BaseMessageList[Location]):
|
||||
__slots__ = ()
|
||||
item_class = Location
|
||||
|
||||
def append(self, value):
|
||||
self.add().from_value(value)
|
||||
+50
-3
@@ -1,4 +1,5 @@
|
||||
from binascii import hexlify, unhexlify
|
||||
from typing import List, Iterator, TypeVar, Generic
|
||||
|
||||
from google.protobuf.message import DecodeError
|
||||
from google.protobuf.json_format import MessageToDict
|
||||
@@ -21,9 +22,10 @@ class Signable:
|
||||
self.unsigned_payload = None
|
||||
self.signing_channel_hash = None
|
||||
|
||||
@property
|
||||
def is_undetermined(self):
|
||||
return self.message.WhichOneof('type') is None
|
||||
def clear_signature(self):
|
||||
self.signature = None
|
||||
self.unsigned_payload = None
|
||||
self.signing_channel_hash = None
|
||||
|
||||
@property
|
||||
def signing_channel_id(self):
|
||||
@@ -72,3 +74,48 @@ class Signable:
|
||||
|
||||
def __bytes__(self):
|
||||
return self.to_bytes()
|
||||
|
||||
|
||||
class Metadata:
|
||||
|
||||
__slots__ = 'message',
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
|
||||
|
||||
I = TypeVar('I')
|
||||
|
||||
|
||||
class BaseMessageList(Metadata, Generic[I]):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
item_class = None
|
||||
|
||||
@property
|
||||
def _message(self):
|
||||
return self.message
|
||||
|
||||
def add(self) -> I:
|
||||
return self.item_class(self._message.add())
|
||||
|
||||
def extend(self, values: List[str]):
|
||||
for value in values:
|
||||
self.append(value)
|
||||
|
||||
def append(self, value: str):
|
||||
raise NotImplemented
|
||||
|
||||
def __len__(self):
|
||||
return len(self._message)
|
||||
|
||||
def __iter__(self) -> Iterator[I]:
|
||||
for item in self._message:
|
||||
yield self.item_class(item)
|
||||
|
||||
def __getitem__(self, item) -> I:
|
||||
return self.item_class(self._message[item])
|
||||
|
||||
def __delitem__(self, key):
|
||||
del self._message[key]
|
||||
+266
-408
@@ -1,74 +1,85 @@
|
||||
import os.path
|
||||
from typing import List, Tuple
|
||||
from decimal import Decimal
|
||||
import logging
|
||||
from typing import List
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
from google.protobuf.json_format import MessageToDict
|
||||
from google.protobuf.message import DecodeError
|
||||
from hachoir.core.log import log as hachoir_log
|
||||
from hachoir.parser import createParser as binary_file_parser
|
||||
from hachoir.metadata import extractMetadata as binary_file_metadata
|
||||
from hachoir.core.log import log as hachoir_log
|
||||
|
||||
from torba.client.hash import Base58
|
||||
from torba.client.constants import COIN
|
||||
|
||||
from lbrynet.schema import compat
|
||||
from lbrynet.schema.base import Signable
|
||||
from lbrynet.schema.mime_types import guess_media_type
|
||||
from lbrynet.schema.types.v2.claim_pb2 import (
|
||||
Claim as ClaimMessage,
|
||||
Fee as FeeMessage,
|
||||
Location as LocationMessage,
|
||||
Language as LanguageMessage
|
||||
from lbrynet.schema.mime_types import guess_media_type, guess_stream_type
|
||||
from lbrynet.schema.attrs import (
|
||||
Source, Playable, Dimmensional, Fee, Image, Video, Audio,
|
||||
LanguageList, LocationList, ClaimList, ClaimReference
|
||||
)
|
||||
from lbrynet.schema.types.v2.claim_pb2 import Claim as ClaimMessage
|
||||
|
||||
|
||||
hachoir_log.use_print = False
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Claim(Signable):
|
||||
|
||||
STREAM = 'stream'
|
||||
CHANNEL = 'channel'
|
||||
COLLECTION = 'collection'
|
||||
REPOST = 'repost'
|
||||
|
||||
__slots__ = 'version',
|
||||
|
||||
message_class = ClaimMessage
|
||||
|
||||
def __init__(self, claim_message=None):
|
||||
super().__init__(claim_message)
|
||||
def __init__(self, message=None):
|
||||
super().__init__(message)
|
||||
self.version = 2
|
||||
|
||||
@property
|
||||
def claim_type(self) -> str:
|
||||
return self.message.WhichOneof('type')
|
||||
|
||||
def get_message(self, type_name):
|
||||
message = getattr(self.message, type_name)
|
||||
if self.claim_type is None:
|
||||
message.SetInParent()
|
||||
if self.claim_type != type_name:
|
||||
raise ValueError(f'Claim is not a {type_name}.')
|
||||
return message
|
||||
|
||||
@property
|
||||
def is_stream(self):
|
||||
return self.message.WhichOneof('type') == 'stream'
|
||||
|
||||
@property
|
||||
def is_channel(self):
|
||||
return self.message.WhichOneof('type') == 'channel'
|
||||
|
||||
@property
|
||||
def stream_message(self):
|
||||
if self.is_undetermined:
|
||||
self.message.stream.SetInParent()
|
||||
if not self.is_stream:
|
||||
raise ValueError('Claim is not a stream.')
|
||||
return self.message.stream
|
||||
return self.claim_type == self.STREAM
|
||||
|
||||
@property
|
||||
def stream(self) -> 'Stream':
|
||||
return Stream(self)
|
||||
|
||||
@property
|
||||
def channel_message(self):
|
||||
if self.is_undetermined:
|
||||
self.message.channel.SetInParent()
|
||||
if not self.is_channel:
|
||||
raise ValueError('Claim is not a channel.')
|
||||
return self.message.channel
|
||||
def is_channel(self):
|
||||
return self.claim_type == self.CHANNEL
|
||||
|
||||
@property
|
||||
def channel(self) -> 'Channel':
|
||||
return Channel(self)
|
||||
|
||||
def to_dict(self):
|
||||
return MessageToDict(self.message, preserving_proto_field_name=True)
|
||||
@property
|
||||
def is_repost(self):
|
||||
return self.claim_type == self.REPOST
|
||||
|
||||
@property
|
||||
def repost(self) -> 'Repost':
|
||||
return Repost(self)
|
||||
|
||||
@property
|
||||
def is_collection(self):
|
||||
return self.claim_type == self.COLLECTION
|
||||
|
||||
@property
|
||||
def collection(self) -> 'Collection':
|
||||
return Collection(self)
|
||||
|
||||
@classmethod
|
||||
def from_bytes(cls, data: bytes) -> 'Claim':
|
||||
@@ -87,394 +98,158 @@ class Claim(Signable):
|
||||
return claim
|
||||
|
||||
|
||||
class Dimmensional:
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def width(self) -> int:
|
||||
return self.message.width
|
||||
|
||||
@width.setter
|
||||
def width(self, width: int):
|
||||
self.message.width = width
|
||||
|
||||
@property
|
||||
def height(self) -> int:
|
||||
return self.message.height
|
||||
|
||||
@height.setter
|
||||
def height(self, height: int):
|
||||
self.message.height = height
|
||||
|
||||
@property
|
||||
def dimensions(self) -> Tuple[int, int]:
|
||||
return self.width, self.height
|
||||
|
||||
@dimensions.setter
|
||||
def dimensions(self, dimensions: Tuple[int, int]):
|
||||
self.message.width, self.message.height = dimensions
|
||||
|
||||
|
||||
class Playable:
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def duration(self) -> int:
|
||||
return self.message.duration
|
||||
|
||||
@duration.setter
|
||||
def duration(self, duration: int):
|
||||
self.message.duration = duration
|
||||
|
||||
def set_duration_from_path(self, file_path):
|
||||
try:
|
||||
file_metadata = binary_file_metadata(binary_file_parser(file_path))
|
||||
self.duration = file_metadata.getValues('duration')[0].seconds
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
class Image(Dimmensional):
|
||||
|
||||
__slots__ = 'message',
|
||||
|
||||
def __init__(self, image_message):
|
||||
self.message = image_message
|
||||
|
||||
|
||||
class Video(Dimmensional, Playable):
|
||||
|
||||
__slots__ = 'message',
|
||||
|
||||
def __init__(self, video_message):
|
||||
self.message = video_message
|
||||
|
||||
|
||||
class Audio(Playable):
|
||||
|
||||
__slots__ = 'message',
|
||||
|
||||
def __init__(self, audio_message):
|
||||
self.message = audio_message
|
||||
|
||||
|
||||
class File:
|
||||
|
||||
__slots__ = '_file',
|
||||
|
||||
def __init__(self, file_message):
|
||||
self._file = file_message
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._file.name
|
||||
|
||||
@name.setter
|
||||
def name(self, name: str):
|
||||
self._file.name = name
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
return self._file.size
|
||||
|
||||
@size.setter
|
||||
def size(self, size: int):
|
||||
self._file.size = size
|
||||
|
||||
|
||||
class Fee:
|
||||
|
||||
__slots__ = '_fee',
|
||||
|
||||
def __init__(self, fee_message):
|
||||
self._fee = fee_message
|
||||
|
||||
@property
|
||||
def currency(self) -> str:
|
||||
return FeeMessage.Currency.Name(self._fee.currency)
|
||||
|
||||
@property
|
||||
def address(self) -> str:
|
||||
return Base58.encode(self._fee.address)
|
||||
|
||||
@address.setter
|
||||
def address(self, address: str):
|
||||
self._fee.address = Base58.decode(address)
|
||||
|
||||
@property
|
||||
def address_bytes(self) -> bytes:
|
||||
return self._fee.address
|
||||
|
||||
@address_bytes.setter
|
||||
def address_bytes(self, address: bytes):
|
||||
self._fee.address = address
|
||||
|
||||
@property
|
||||
def amount(self) -> Decimal:
|
||||
if self.currency == 'LBC':
|
||||
return self.lbc
|
||||
if self.currency == 'USD':
|
||||
return self.usd
|
||||
|
||||
DEWIES = Decimal(COIN)
|
||||
|
||||
@property
|
||||
def lbc(self) -> Decimal:
|
||||
if self._fee.currency != FeeMessage.LBC:
|
||||
raise ValueError('LBC can only be returned for LBC fees.')
|
||||
return Decimal(self._fee.amount / self.DEWIES)
|
||||
|
||||
@lbc.setter
|
||||
def lbc(self, amount: Decimal):
|
||||
self.dewies = int(amount * self.DEWIES)
|
||||
|
||||
@property
|
||||
def dewies(self) -> int:
|
||||
if self._fee.currency != FeeMessage.LBC:
|
||||
raise ValueError('Dewies can only be returned for LBC fees.')
|
||||
return self._fee.amount
|
||||
|
||||
@dewies.setter
|
||||
def dewies(self, amount: int):
|
||||
self._fee.amount = amount
|
||||
self._fee.currency = FeeMessage.LBC
|
||||
|
||||
PENNIES = Decimal(100.0)
|
||||
|
||||
@property
|
||||
def usd(self) -> Decimal:
|
||||
if self._fee.currency != FeeMessage.USD:
|
||||
raise ValueError('USD can only be returned for USD fees.')
|
||||
return Decimal(self._fee.amount / self.PENNIES)
|
||||
|
||||
@usd.setter
|
||||
def usd(self, amount: Decimal):
|
||||
self.pennies = int(amount * self.PENNIES)
|
||||
|
||||
@property
|
||||
def pennies(self) -> int:
|
||||
if self._fee.currency != FeeMessage.USD:
|
||||
raise ValueError('Pennies can only be returned for USD fees.')
|
||||
return self._fee.amount
|
||||
|
||||
@pennies.setter
|
||||
def pennies(self, amount: int):
|
||||
self._fee.amount = amount
|
||||
self._fee.currency = FeeMessage.USD
|
||||
|
||||
|
||||
class BaseClaimSubType:
|
||||
class BaseClaim:
|
||||
|
||||
__slots__ = 'claim', 'message'
|
||||
|
||||
def __init__(self, claim: Claim):
|
||||
claim_type = None
|
||||
object_fields = 'thumbnail',
|
||||
repeat_fields = 'tags', 'languages', 'locations'
|
||||
|
||||
def __init__(self, claim: Claim = None):
|
||||
self.claim = claim or Claim()
|
||||
|
||||
@property
|
||||
def tags(self) -> List:
|
||||
return self.message.tags
|
||||
|
||||
@property
|
||||
def title(self) -> str:
|
||||
return self.message.title
|
||||
|
||||
@title.setter
|
||||
def title(self, title: str):
|
||||
self.message.title = title
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
return self.message.description
|
||||
|
||||
@description.setter
|
||||
def description(self, description: str):
|
||||
self.message.description = description
|
||||
|
||||
@property
|
||||
def thumbnail_url(self) -> str:
|
||||
return self.message.thumbnail_url
|
||||
|
||||
@thumbnail_url.setter
|
||||
def thumbnail_url(self, thumbnail_url: str):
|
||||
self.message.thumbnail_url = thumbnail_url
|
||||
|
||||
@property
|
||||
def language(self) -> str:
|
||||
if len(self.languages) > 0:
|
||||
return LanguageMessage.Language.Name(self.languages[0].language)
|
||||
|
||||
@language.setter
|
||||
def language(self, language: str):
|
||||
value = LanguageMessage.Language.Value(language)
|
||||
if len(self.languages) > 0:
|
||||
self.languages[0].language = value
|
||||
else:
|
||||
self.languages.add().language = value
|
||||
|
||||
@property
|
||||
def languages(self):
|
||||
return self.message.languages
|
||||
|
||||
@property
|
||||
def location_country(self) -> str:
|
||||
if len(self.locations) > 0:
|
||||
return LocationMessage.Country.Name(self.locations[0].country)
|
||||
|
||||
@location_country.setter
|
||||
def location_country(self, country: str):
|
||||
value = LocationMessage.Country.Value(country)
|
||||
if len(self.locations) > 0:
|
||||
self.locations[0].location = value
|
||||
else:
|
||||
self.locations.add().location = value
|
||||
|
||||
@property
|
||||
def locations(self):
|
||||
return self.message.locations
|
||||
self.message = self.claim.get_message(self.claim_type)
|
||||
|
||||
def to_dict(self):
|
||||
return MessageToDict(self.message, preserving_proto_field_name=True)
|
||||
claim = MessageToDict(self.claim.message, preserving_proto_field_name=True)
|
||||
claim.update(claim.pop(self.claim_type))
|
||||
if 'languages' in claim:
|
||||
claim['languages'] = self.langtags
|
||||
if 'locations' in claim:
|
||||
claim['locations'] = [l.to_dict() for l in self.locations]
|
||||
return claim
|
||||
|
||||
def update(self, tags=None, clear_tags=False, **kwargs):
|
||||
def update(self, **kwargs):
|
||||
for key in list(kwargs):
|
||||
for field in self.object_fields:
|
||||
if key.startswith(f'{field}_'):
|
||||
attr = getattr(self, field)
|
||||
setattr(attr, key[len(f'{field}_'):], kwargs.pop(key))
|
||||
continue
|
||||
|
||||
if clear_tags:
|
||||
self.message.ClearField('tags')
|
||||
|
||||
if tags is not None:
|
||||
if isinstance(tags, str):
|
||||
self.tags.append(tags)
|
||||
elif isinstance(tags, list):
|
||||
self.tags.extend(tags)
|
||||
else:
|
||||
raise ValueError(f"Unknown tag type: {tags}")
|
||||
for l in self.repeat_fields:
|
||||
field = getattr(self, l)
|
||||
if kwargs.pop(f'clear_{l}', False):
|
||||
del field[:]
|
||||
items = kwargs.pop(l, None)
|
||||
if items is not None:
|
||||
if isinstance(items, str):
|
||||
field.append(items)
|
||||
elif isinstance(items, list):
|
||||
field.extend(items)
|
||||
else:
|
||||
raise ValueError(f"Unknown {l} value: {items}")
|
||||
|
||||
for key, value in kwargs.items():
|
||||
setattr(self, key, value)
|
||||
|
||||
@property
|
||||
def title(self) -> str:
|
||||
return self.claim.message.title
|
||||
|
||||
class Channel(BaseClaimSubType):
|
||||
@title.setter
|
||||
def title(self, title: str):
|
||||
self.claim.message.title = title
|
||||
|
||||
@property
|
||||
def description(self) -> str:
|
||||
return self.claim.message.description
|
||||
|
||||
@description.setter
|
||||
def description(self, description: str):
|
||||
self.claim.message.description = description
|
||||
|
||||
@property
|
||||
def thumbnail(self) -> Source:
|
||||
return Source(self.claim.message.thumbnail)
|
||||
|
||||
@property
|
||||
def tags(self) -> List:
|
||||
return self.claim.message.tags
|
||||
|
||||
@property
|
||||
def languages(self) -> LanguageList:
|
||||
return LanguageList(self.claim.message.languages)
|
||||
|
||||
@property
|
||||
def langtags(self) -> List[str]:
|
||||
return [l.langtag for l in self.languages]
|
||||
|
||||
@property
|
||||
def locations(self) -> LocationList:
|
||||
return LocationList(self.claim.message.locations)
|
||||
|
||||
|
||||
class Stream(BaseClaim):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def __init__(self, claim: Claim = None):
|
||||
super().__init__(claim)
|
||||
self.message = self.claim.channel_message
|
||||
claim_type = Claim.STREAM
|
||||
|
||||
@property
|
||||
def public_key(self) -> str:
|
||||
return hexlify(self.message.public_key).decode()
|
||||
object_fields = BaseClaim.object_fields + ('source',)
|
||||
|
||||
@public_key.setter
|
||||
def public_key(self, sd_public_key: str):
|
||||
self.message.public_key = unhexlify(sd_public_key.encode())
|
||||
def to_dict(self):
|
||||
claim = super().to_dict()
|
||||
if 'source' in claim:
|
||||
if 'hash' in claim['source']:
|
||||
claim['source']['hash'] = self.source.file_hash
|
||||
if 'sd_hash' in claim['source']:
|
||||
claim['source']['sd_hash'] = self.source.sd_hash
|
||||
fee = claim.get('fee', {})
|
||||
if 'address' in fee:
|
||||
fee['address'] = self.fee.address
|
||||
if 'amount' in fee:
|
||||
fee['amount'] = str(self.fee.amount)
|
||||
stream_type = self.message.WhichOneof('type')
|
||||
if stream_type:
|
||||
claim['stream_type'] = stream_type
|
||||
return claim
|
||||
|
||||
@property
|
||||
def public_key_bytes(self) -> bytes:
|
||||
return self.message.public_key
|
||||
def update(self, file_path=None, height=None, width=None, duration=None, **kwargs):
|
||||
if kwargs.pop('clear_fee', False):
|
||||
self.message.ClearField('fee')
|
||||
else:
|
||||
self.fee.update(
|
||||
kwargs.pop('fee_address', None),
|
||||
kwargs.pop('fee_currency', None),
|
||||
kwargs.pop('fee_amount', None)
|
||||
)
|
||||
|
||||
@public_key_bytes.setter
|
||||
def public_key_bytes(self, public_key: bytes):
|
||||
self.message.public_key = public_key
|
||||
if 'sd_hash' in kwargs:
|
||||
self.source.sd_hash = kwargs.pop('sd_hash')
|
||||
if 'file_name' in kwargs:
|
||||
self.source.name = kwargs.pop('file_name')
|
||||
if 'file_hash' in kwargs:
|
||||
self.source.file_hash = kwargs.pop('file_hash')
|
||||
|
||||
@property
|
||||
def contact_email(self) -> str:
|
||||
return self.message.contact_email
|
||||
stream_type = None
|
||||
if file_path is not None:
|
||||
stream_type = self.source.update(file_path=file_path)
|
||||
elif self.source.name:
|
||||
self.source.media_type, stream_type = guess_media_type(self.source.name)
|
||||
elif self.source.media_type:
|
||||
stream_type = guess_stream_type(self.source.media_type)
|
||||
|
||||
@contact_email.setter
|
||||
def contact_email(self, contact_email: str):
|
||||
self.message.contact_email = contact_email
|
||||
if 'file_size' in kwargs:
|
||||
self.source.size = kwargs.pop('file_size')
|
||||
|
||||
@property
|
||||
def homepage_url(self) -> str:
|
||||
return self.message.homepage_url
|
||||
|
||||
@homepage_url.setter
|
||||
def homepage_url(self, homepage_url: str):
|
||||
self.message.homepage_url = homepage_url
|
||||
|
||||
@property
|
||||
def cover_url(self) -> str:
|
||||
return self.message.cover_url
|
||||
|
||||
@cover_url.setter
|
||||
def cover_url(self, cover_url: str):
|
||||
self.message.cover_url = cover_url
|
||||
|
||||
|
||||
class Stream(BaseClaimSubType):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def __init__(self, claim: Claim = None):
|
||||
super().__init__(claim)
|
||||
self.message = self.claim.stream_message
|
||||
|
||||
def update(
|
||||
self, file_path=None, stream_type=None,
|
||||
fee_currency=None, fee_amount=None, fee_address=None,
|
||||
**kwargs):
|
||||
|
||||
duration_was_not_set = True
|
||||
sub_types = ('image', 'video', 'audio')
|
||||
for key in list(kwargs.keys()):
|
||||
for sub_type in sub_types:
|
||||
if key.startswith(f'{sub_type}_'):
|
||||
stream_type = sub_type
|
||||
sub_obj = getattr(self, sub_type)
|
||||
sub_obj_attr = key[len(f'{sub_type}_'):]
|
||||
setattr(sub_obj, sub_obj_attr, kwargs.pop(key))
|
||||
if sub_obj_attr == 'duration':
|
||||
duration_was_not_set = False
|
||||
break
|
||||
|
||||
if stream_type is not None:
|
||||
if stream_type not in sub_types:
|
||||
raise Exception(
|
||||
f"stream_type of '{stream_type}' is not valid, must be one of: {sub_types}"
|
||||
)
|
||||
|
||||
sub_obj = getattr(self, stream_type)
|
||||
if duration_was_not_set and file_path and isinstance(sub_obj, Playable):
|
||||
sub_obj.set_duration_from_path(file_path)
|
||||
if stream_type in ('image', 'video', 'audio'):
|
||||
media = getattr(self, stream_type)
|
||||
media_args = {'file_metadata': None}
|
||||
if file_path is not None:
|
||||
try:
|
||||
media_args['file_metadata'] = binary_file_metadata(binary_file_parser(file_path))
|
||||
except:
|
||||
log.exception('Could not read file metadata.')
|
||||
if isinstance(media, Playable):
|
||||
media_args['duration'] = duration
|
||||
if isinstance(media, Dimmensional):
|
||||
media_args['height'] = height
|
||||
media_args['width'] = width
|
||||
media.update(**media_args)
|
||||
|
||||
super().update(**kwargs)
|
||||
|
||||
if file_path is not None:
|
||||
self.media_type = guess_media_type(file_path)
|
||||
if not os.path.isfile(file_path):
|
||||
raise Exception(f"File does not exist: {file_path}")
|
||||
self.file.size = os.path.getsize(file_path)
|
||||
if self.file.size == 0:
|
||||
raise Exception(f"Cannot publish empty file: {file_path}")
|
||||
|
||||
if fee_amount and fee_currency:
|
||||
if fee_address:
|
||||
self.fee.address = fee_address
|
||||
if fee_currency.lower() == 'lbc':
|
||||
self.fee.lbc = Decimal(fee_amount)
|
||||
elif fee_currency.lower() == 'usd':
|
||||
self.fee.usd = Decimal(fee_amount)
|
||||
else:
|
||||
raise Exception(f'Unknown currency type: {fee_currency}')
|
||||
|
||||
@property
|
||||
def sd_hash(self) -> str:
|
||||
return hexlify(self.message.sd_hash).decode()
|
||||
|
||||
@sd_hash.setter
|
||||
def sd_hash(self, sd_hash: str):
|
||||
self.message.sd_hash = unhexlify(sd_hash.encode())
|
||||
|
||||
@property
|
||||
def sd_hash_bytes(self) -> bytes:
|
||||
return self.message.sd_hash
|
||||
|
||||
@sd_hash_bytes.setter
|
||||
def sd_hash_bytes(self, sd_hash: bytes):
|
||||
self.message.sd_hash = sd_hash
|
||||
|
||||
@property
|
||||
def author(self) -> str:
|
||||
return self.message.author
|
||||
@@ -507,14 +282,6 @@ class Stream(BaseClaimSubType):
|
||||
def release_time(self, release_time: int):
|
||||
self.message.release_time = release_time
|
||||
|
||||
@property
|
||||
def media_type(self) -> str:
|
||||
return self.message.media_type
|
||||
|
||||
@media_type.setter
|
||||
def media_type(self, media_type: str):
|
||||
self.message.media_type = media_type
|
||||
|
||||
@property
|
||||
def fee(self) -> Fee:
|
||||
return Fee(self.message.fee)
|
||||
@@ -524,8 +291,12 @@ class Stream(BaseClaimSubType):
|
||||
return self.message.HasField('fee')
|
||||
|
||||
@property
|
||||
def file(self) -> File:
|
||||
return File(self.message.file)
|
||||
def source(self) -> Source:
|
||||
return Source(self.message.source)
|
||||
|
||||
@property
|
||||
def stream_type(self) -> str:
|
||||
return self.message.WhichOneof('type')
|
||||
|
||||
@property
|
||||
def image(self) -> Image:
|
||||
@@ -538,3 +309,90 @@ class Stream(BaseClaimSubType):
|
||||
@property
|
||||
def audio(self) -> Audio:
|
||||
return Audio(self.message.audio)
|
||||
|
||||
|
||||
class Channel(BaseClaim):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
claim_type = Claim.CHANNEL
|
||||
|
||||
object_fields = BaseClaim.object_fields + ('cover',)
|
||||
repeat_fields = BaseClaim.repeat_fields + ('featured',)
|
||||
|
||||
def to_dict(self):
|
||||
claim = super().to_dict()
|
||||
claim['public_key'] = self.public_key
|
||||
if 'featured' in claim:
|
||||
claim['featured'] = self.featured.ids
|
||||
return claim
|
||||
|
||||
@property
|
||||
def public_key(self) -> str:
|
||||
return hexlify(self.message.public_key).decode()
|
||||
|
||||
@public_key.setter
|
||||
def public_key(self, sd_public_key: str):
|
||||
self.message.public_key = unhexlify(sd_public_key.encode())
|
||||
|
||||
@property
|
||||
def public_key_bytes(self) -> bytes:
|
||||
return self.message.public_key
|
||||
|
||||
@public_key_bytes.setter
|
||||
def public_key_bytes(self, public_key: bytes):
|
||||
self.message.public_key = public_key
|
||||
|
||||
@property
|
||||
def email(self) -> str:
|
||||
return self.message.email
|
||||
|
||||
@email.setter
|
||||
def email(self, email: str):
|
||||
self.message.email = email
|
||||
|
||||
@property
|
||||
def website_url(self) -> str:
|
||||
return self.message.website_url
|
||||
|
||||
@website_url.setter
|
||||
def website_url(self, website_url: str):
|
||||
self.message.website_url = website_url
|
||||
|
||||
@property
|
||||
def cover(self) -> Source:
|
||||
return Source(self.message.cover)
|
||||
|
||||
@property
|
||||
def featured(self) -> ClaimList:
|
||||
return ClaimList(self.message.featured)
|
||||
|
||||
|
||||
class Repost(BaseClaim):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
claim_type = Claim.REPOST
|
||||
|
||||
@property
|
||||
def reference(self) -> ClaimReference:
|
||||
return ClaimReference(self.message)
|
||||
|
||||
|
||||
class Collection(BaseClaim):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
claim_type = Claim.COLLECTION
|
||||
|
||||
repeat_fields = BaseClaim.repeat_fields + ('claims',)
|
||||
|
||||
def to_dict(self):
|
||||
claim = super().to_dict()
|
||||
if 'claim_references' in claim:
|
||||
claim['claim_references'] = self.claims.ids
|
||||
return claim
|
||||
|
||||
@property
|
||||
def claims(self) -> ClaimList:
|
||||
return ClaimList(self.message)
|
||||
+27
-19
@@ -1,19 +1,26 @@
|
||||
import json
|
||||
from decimal import Decimal
|
||||
|
||||
from google.protobuf.message import DecodeError
|
||||
|
||||
from lbrynet.schema.types.v1.legacy_claim_pb2 import Claim as OldClaimMessage
|
||||
from lbrynet.schema.types.v1.metadata_pb2 import Metadata as MetadataMessage
|
||||
from lbrynet.schema.types.v1.certificate_pb2 import KeyType
|
||||
from lbrynet.schema.types.v1.fee_pb2 import Fee as FeeMessage
|
||||
|
||||
|
||||
def from_old_json_schema(claim, payload: bytes):
|
||||
value = json.loads(payload)
|
||||
try:
|
||||
value = json.loads(payload)
|
||||
except:
|
||||
raise DecodeError('Could not parse JSON.')
|
||||
stream = claim.stream
|
||||
stream.media_type = value.get('content_type', value.get('content-type', 'application/octet-stream'))
|
||||
stream.source.sd_hash = value['sources']['lbry_sd_hash']
|
||||
stream.source.media_type = value.get(
|
||||
'content_type', value.get('content-type', 'application/octet-stream'))
|
||||
stream.title = value.get('title', '')
|
||||
stream.description = value.get('description', '')
|
||||
stream.thumbnail_url = value.get('thumbnail', '')
|
||||
if value.get('thumbnail', ''):
|
||||
stream.thumbnail.url = value.get('thumbnail', '')
|
||||
stream.author = value.get('author', '')
|
||||
stream.license = value.get('license', '')
|
||||
stream.license_url = value.get('license_url', '')
|
||||
@@ -22,10 +29,9 @@ def from_old_json_schema(claim, payload: bytes):
|
||||
if language.lower() == 'english':
|
||||
language = 'en'
|
||||
try:
|
||||
stream.language = language
|
||||
stream.languages.append(language)
|
||||
except:
|
||||
pass
|
||||
stream.sd_hash = value['sources']['lbry_sd_hash']
|
||||
if value.get('nsfw', False):
|
||||
stream.tags.append('mature')
|
||||
if "fee" in value:
|
||||
@@ -35,8 +41,10 @@ def from_old_json_schema(claim, payload: bytes):
|
||||
stream.fee.lbc = Decimal(fee[currency]['amount'])
|
||||
elif currency == 'USD':
|
||||
stream.fee.usd = Decimal(fee[currency]['amount'])
|
||||
elif currency == 'BTC':
|
||||
stream.fee.btc = Decimal(fee[currency]['amount'])
|
||||
else:
|
||||
raise ValueError(f'Unknown currency: {currency}')
|
||||
raise DecodeError(f'Unknown currency: {currency}')
|
||||
stream.fee.address = fee[currency]['address']
|
||||
return claim
|
||||
|
||||
@@ -44,18 +52,21 @@ def from_old_json_schema(claim, payload: bytes):
|
||||
def from_types_v1(claim, payload: bytes):
|
||||
old = OldClaimMessage()
|
||||
old.ParseFromString(payload)
|
||||
if old.claimType == 1:
|
||||
if old.claimType == 2:
|
||||
channel = claim.channel
|
||||
channel.public_key_bytes = old.certificate.publicKey
|
||||
else:
|
||||
stream = claim.stream
|
||||
stream.title = old.stream.metadata.title
|
||||
stream.description = old.stream.metadata.description
|
||||
stream.author = old.stream.metadata.author
|
||||
stream.license = old.stream.metadata.license
|
||||
stream.license_url = old.stream.metadata.licenseUrl
|
||||
stream.thumbnail_url = old.stream.metadata.thumbnail
|
||||
stream.thumbnail.url = old.stream.metadata.thumbnail
|
||||
if old.stream.metadata.HasField('language'):
|
||||
stream.languages.add().language = old.stream.metadata.language
|
||||
stream.media_type = old.stream.source.contentType
|
||||
stream.sd_hash_bytes = old.stream.source.source
|
||||
stream.languages.add().message.language = old.stream.metadata.language
|
||||
stream.source.media_type = old.stream.source.contentType
|
||||
stream.source.sd_hash_bytes = old.stream.source.source
|
||||
if old.stream.metadata.nsfw:
|
||||
stream.tags.append('mature')
|
||||
if old.stream.metadata.HasField('fee'):
|
||||
@@ -66,18 +77,15 @@ def from_types_v1(claim, payload: bytes):
|
||||
stream.fee.lbc = Decimal(fee.amount)
|
||||
elif currency == 'USD':
|
||||
stream.fee.usd = Decimal(fee.amount)
|
||||
elif currency == 'BTC':
|
||||
stream.fee.btc = Decimal(fee.amount)
|
||||
else:
|
||||
raise ValueError(f'Unsupported currency: {currency}')
|
||||
raise DecodeError(f'Unsupported currency: {currency}')
|
||||
if old.HasField('publisherSignature'):
|
||||
sig = old.publisherSignature
|
||||
claim.signature = sig.signature
|
||||
claim.signature_type = KeyType.Name(sig.signatureType)
|
||||
claim.signing_channel_hash = sig.certificateId
|
||||
claim.signing_channel_hash = sig.certificateId[::-1]
|
||||
old.ClearField("publisherSignature")
|
||||
claim.unsigned_payload = old.SerializeToString()
|
||||
elif old.claimType == 2:
|
||||
channel = claim.channel
|
||||
channel.public_key_bytes = old.certificate.publicKey
|
||||
else:
|
||||
raise ValueError('claimType must be 1 for Streams and 2 for Channel')
|
||||
return claim
|
||||
+160
-146
@@ -1,160 +1,174 @@
|
||||
import os
|
||||
|
||||
|
||||
types_map = {
|
||||
# http://www.iana.org/assignments/media-types
|
||||
'.a': 'application/octet-stream',
|
||||
'.ai': 'application/postscript',
|
||||
'.aif': 'audio/x-aiff',
|
||||
'.aifc': 'audio/x-aiff',
|
||||
'.aiff': 'audio/x-aiff',
|
||||
'.au': 'audio/basic',
|
||||
'.avi': 'video/x-msvideo',
|
||||
'.bat': 'text/plain',
|
||||
'.bcpio': 'application/x-bcpio',
|
||||
'.bin': 'application/octet-stream',
|
||||
'.bmp': 'image/bmp',
|
||||
'.c': 'text/plain',
|
||||
'.cdf': 'application/x-netcdf',
|
||||
'.cpio': 'application/x-cpio',
|
||||
'.csh': 'application/x-csh',
|
||||
'.css': 'text/css',
|
||||
'.csv': 'text/csv',
|
||||
'.dll': 'application/octet-stream',
|
||||
'.doc': 'application/msword',
|
||||
'.dot': 'application/msword',
|
||||
'.dvi': 'application/x-dvi',
|
||||
'.eml': 'message/rfc822',
|
||||
'.eps': 'application/postscript',
|
||||
'.epub': 'application/epub+zip',
|
||||
'.etx': 'text/x-setext',
|
||||
'.exe': 'application/octet-stream',
|
||||
'.gif': 'image/gif',
|
||||
'.gtar': 'application/x-gtar',
|
||||
'.h': 'text/plain',
|
||||
'.hdf': 'application/x-hdf',
|
||||
'.htm': 'text/html',
|
||||
'.html': 'text/html',
|
||||
'.ico': 'image/vnd.microsoft.icon',
|
||||
'.ief': 'image/ief',
|
||||
'.iges': 'model/iges',
|
||||
'.jpe': 'image/jpeg',
|
||||
'.jpeg': 'image/jpeg',
|
||||
'.jpg': 'image/jpeg',
|
||||
'.js': 'application/javascript',
|
||||
'.json': 'application/json',
|
||||
'.ksh': 'text/plain',
|
||||
'.latex': 'application/x-latex',
|
||||
'.m1v': 'video/mpeg',
|
||||
'.m3u': 'application/vnd.apple.mpegurl',
|
||||
'.m3u8': 'application/vnd.apple.mpegurl',
|
||||
'.man': 'application/x-troff-man',
|
||||
'.markdown': 'text/markdown',
|
||||
'.md': 'text/markdown',
|
||||
'.me': 'application/x-troff-me',
|
||||
'.mht': 'message/rfc822',
|
||||
'.mhtml': 'message/rfc822',
|
||||
'.mif': 'application/x-mif',
|
||||
'.mov': 'video/quicktime',
|
||||
'.movie': 'video/x-sgi-movie',
|
||||
'.mp2': 'audio/mpeg',
|
||||
'.mp3': 'audio/mpeg',
|
||||
'.mp4': 'video/mp4',
|
||||
'.mpa': 'video/mpeg',
|
||||
'.mpe': 'video/mpeg',
|
||||
'.mpeg': 'video/mpeg',
|
||||
'.mpg': 'video/mpeg',
|
||||
'.ms': 'application/x-troff-ms',
|
||||
'.nc': 'application/x-netcdf',
|
||||
'.nws': 'message/rfc822',
|
||||
'.o': 'application/octet-stream',
|
||||
'.obj': 'application/octet-stream',
|
||||
'.oda': 'application/oda',
|
||||
'.p12': 'application/x-pkcs12',
|
||||
'.p7c': 'application/pkcs7-mime',
|
||||
'.pbm': 'image/x-portable-bitmap',
|
||||
'.pdf': 'application/pdf',
|
||||
'.pfx': 'application/x-pkcs12',
|
||||
'.pgm': 'image/x-portable-graymap',
|
||||
'.pl': 'text/plain',
|
||||
'.png': 'image/png',
|
||||
'.pnm': 'image/x-portable-anymap',
|
||||
'.pot': 'application/vnd.ms-powerpoint',
|
||||
'.ppa': 'application/vnd.ms-powerpoint',
|
||||
'.ppm': 'image/x-portable-pixmap',
|
||||
'.pps': 'application/vnd.ms-powerpoint',
|
||||
'.ppt': 'application/vnd.ms-powerpoint',
|
||||
'.ps': 'application/postscript',
|
||||
'.pwz': 'application/vnd.ms-powerpoint',
|
||||
'.py': 'text/x-python',
|
||||
'.pyc': 'application/x-python-code',
|
||||
'.pyo': 'application/x-python-code',
|
||||
'.qt': 'video/quicktime',
|
||||
'.ra': 'audio/x-pn-realaudio',
|
||||
'.ram': 'application/x-pn-realaudio',
|
||||
'.ras': 'image/x-cmu-raster',
|
||||
'.rdf': 'application/xml',
|
||||
'.rgb': 'image/x-rgb',
|
||||
'.roff': 'application/x-troff',
|
||||
'.rtx': 'text/richtext',
|
||||
'.sgm': 'text/x-sgml',
|
||||
'.sgml': 'text/x-sgml',
|
||||
'.sh': 'application/x-sh',
|
||||
'.shar': 'application/x-shar',
|
||||
'.snd': 'audio/basic',
|
||||
'.so': 'application/octet-stream',
|
||||
'.src': 'application/x-wais-source',
|
||||
'.stl': 'model/stl',
|
||||
'.sv4cpio': 'application/x-sv4cpio',
|
||||
'.sv4crc': 'application/x-sv4crc',
|
||||
'.svg': 'image/svg+xml',
|
||||
'.swf': 'application/x-shockwave-flash',
|
||||
'.t': 'application/x-troff',
|
||||
'.tar': 'application/x-tar',
|
||||
'.tcl': 'application/x-tcl',
|
||||
'.tex': 'application/x-tex',
|
||||
'.texi': 'application/x-texinfo',
|
||||
'.texinfo': 'application/x-texinfo',
|
||||
'.tif': 'image/tiff',
|
||||
'.tiff': 'image/tiff',
|
||||
'.tr': 'application/x-troff',
|
||||
'.tsv': 'text/tab-separated-values',
|
||||
'.txt': 'text/plain',
|
||||
'.ustar': 'application/x-ustar',
|
||||
'.vcf': 'text/x-vcard',
|
||||
'.wav': 'audio/x-wav',
|
||||
'.webm': 'video/webm',
|
||||
'.wiz': 'application/msword',
|
||||
'.wsdl': 'application/xml',
|
||||
'.xbm': 'image/x-xbitmap',
|
||||
'.xlb': 'application/vnd.ms-excel',
|
||||
'.xls': 'application/vnd.ms-excel',
|
||||
'.xml': 'text/xml',
|
||||
'.xpdl': 'application/xml',
|
||||
'.xpm': 'image/x-xpixmap',
|
||||
'.xsl': 'application/xml',
|
||||
'.xwd': 'image/x-xwindowdump',
|
||||
'.zip': 'application/zip',
|
||||
# Type mapping for automated metadata extraction (video, audio, image, document, binary, model)
|
||||
'.a': ('application/octet-stream', 'binary'),
|
||||
'.ai': ('application/postscript', 'image'),
|
||||
'.aif': ('audio/x-aiff', 'audio'),
|
||||
'.aifc': ('audio/x-aiff', 'audio'),
|
||||
'.aiff': ('audio/x-aiff', 'audio'),
|
||||
'.au': ('audio/basic', 'audio'),
|
||||
'.avi': ('video/x-msvideo', 'video'),
|
||||
'.bat': ('text/plain', 'document'),
|
||||
'.bcpio': ('application/x-bcpio', 'binary'),
|
||||
'.bin': ('application/octet-stream', 'binary'),
|
||||
'.bmp': ('image/bmp', 'image'),
|
||||
'.c': ('text/plain', 'document'),
|
||||
'.cdf': ('application/x-netcdf', 'binary'),
|
||||
'.cpio': ('application/x-cpio', 'binary'),
|
||||
'.csh': ('application/x-csh', 'binary'),
|
||||
'.css': ('text/css', 'document'),
|
||||
'.csv': ('text/csv', 'document'),
|
||||
'.dll': ('application/octet-stream', 'binary'),
|
||||
'.doc': ('application/msword', 'document'),
|
||||
'.dot': ('application/msword', 'document'),
|
||||
'.dvi': ('application/x-dvi', 'binary'),
|
||||
'.eml': ('message/rfc822', 'document'),
|
||||
'.eps': ('application/postscript', 'document'),
|
||||
'.epub': ('application/epub+zip', 'document'),
|
||||
'.etx': ('text/x-setext', 'document'),
|
||||
'.exe': ('application/octet-stream', 'binary'),
|
||||
'.gif': ('image/gif', 'image'),
|
||||
'.gtar': ('application/x-gtar', 'binary'),
|
||||
'.h': ('text/plain', 'document'),
|
||||
'.hdf': ('application/x-hdf', 'binary'),
|
||||
'.htm': ('text/html', 'document'),
|
||||
'.html': ('text/html', 'document'),
|
||||
'.ico': ('image/vnd.microsoft.icon', 'image'),
|
||||
'.ief': ('image/ief', 'image'),
|
||||
'.iges': ('model/iges', 'model'),
|
||||
'.jpe': ('image/jpeg', 'image'),
|
||||
'.jpeg': ('image/jpeg', 'image'),
|
||||
'.jpg': ('image/jpeg', 'image'),
|
||||
'.js': ('application/javascript', 'document'),
|
||||
'.json': ('application/json', 'document'),
|
||||
'.ksh': ('text/plain', 'document'),
|
||||
'.latex': ('application/x-latex', 'binary'),
|
||||
'.m1v': ('video/mpeg', 'video'),
|
||||
'.m3u': ('application/vnd.apple.mpegurl', 'audio'),
|
||||
'.m3u8': ('application/vnd.apple.mpegurl', 'audio'),
|
||||
'.man': ('application/x-troff-man', 'document'),
|
||||
'.markdown': ('text/markdown', 'document'),
|
||||
'.md': ('text/markdown', 'document'),
|
||||
'.me': ('application/x-troff-me', 'binary'),
|
||||
'.mht': ('message/rfc822', 'document'),
|
||||
'.mhtml': ('message/rfc822', 'document'),
|
||||
'.mif': ('application/x-mif', 'binary'),
|
||||
'.mov': ('video/quicktime', 'video'),
|
||||
'.movie': ('video/x-sgi-movie', 'video'),
|
||||
'.mp2': ('audio/mpeg', 'audio'),
|
||||
'.mp3': ('audio/mpeg', 'audio'),
|
||||
'.mp4': ('video/mp4', 'video'),
|
||||
'.mpa': ('video/mpeg', 'video'),
|
||||
'.mpe': ('video/mpeg', 'video'),
|
||||
'.mpeg': ('video/mpeg', 'video'),
|
||||
'.mpg': ('video/mpeg', 'video'),
|
||||
'.ms': ('application/x-troff-ms', 'binary'),
|
||||
'.nc': ('application/x-netcdf', 'binary'),
|
||||
'.nws': ('message/rfc822', 'document'),
|
||||
'.o': ('application/octet-stream', 'binary'),
|
||||
'.obj': ('application/octet-stream', 'model'),
|
||||
'.oda': ('application/oda', 'binary'),
|
||||
'.p12': ('application/x-pkcs12', 'binary'),
|
||||
'.p7c': ('application/pkcs7-mime', 'binary'),
|
||||
'.pbm': ('image/x-portable-bitmap', 'image'),
|
||||
'.pdf': ('application/pdf', 'document'),
|
||||
'.pfx': ('application/x-pkcs12', 'binary'),
|
||||
'.pgm': ('image/x-portable-graymap', 'image'),
|
||||
'.pl': ('text/plain', 'document'),
|
||||
'.png': ('image/png', 'image'),
|
||||
'.pnm': ('image/x-portable-anymap', 'image'),
|
||||
'.pot': ('application/vnd.ms-powerpoint', 'document'),
|
||||
'.ppa': ('application/vnd.ms-powerpoint', 'document'),
|
||||
'.ppm': ('image/x-portable-pixmap', 'image'),
|
||||
'.pps': ('application/vnd.ms-powerpoint', 'document'),
|
||||
'.ppt': ('application/vnd.ms-powerpoint', 'document'),
|
||||
'.ps': ('application/postscript', 'document'),
|
||||
'.pwz': ('application/vnd.ms-powerpoint', 'document'),
|
||||
'.py': ('text/x-python', 'document'),
|
||||
'.pyc': ('application/x-python-code', 'binary'),
|
||||
'.pyo': ('application/x-python-code', 'binary'),
|
||||
'.qt': ('video/quicktime', 'video'),
|
||||
'.ra': ('audio/x-pn-realaudio', 'audio'),
|
||||
'.ram': ('application/x-pn-realaudio', 'audio'),
|
||||
'.ras': ('image/x-cmu-raster', 'image'),
|
||||
'.rdf': ('application/xml', 'binary'),
|
||||
'.rgb': ('image/x-rgb', 'image'),
|
||||
'.roff': ('application/x-troff', 'binary'),
|
||||
'.rtx': ('text/richtext', 'document'),
|
||||
'.sgm': ('text/x-sgml', 'document'),
|
||||
'.sgml': ('text/x-sgml', 'document'),
|
||||
'.sh': ('application/x-sh', 'document'),
|
||||
'.shar': ('application/x-shar', 'binary'),
|
||||
'.snd': ('audio/basic', 'audio'),
|
||||
'.so': ('application/octet-stream', 'binary'),
|
||||
'.src': ('application/x-wais-source', 'binary'),
|
||||
'.stl': ('model/stl', 'model'),
|
||||
'.sv4cpio': ('application/x-sv4cpio', 'binary'),
|
||||
'.sv4crc': ('application/x-sv4crc', 'binary'),
|
||||
'.svg': ('image/svg+xml', 'image'),
|
||||
'.swf': ('application/x-shockwave-flash', 'binary'),
|
||||
'.t': ('application/x-troff', 'binary'),
|
||||
'.tar': ('application/x-tar', 'binary'),
|
||||
'.tcl': ('application/x-tcl', 'binary'),
|
||||
'.tex': ('application/x-tex', 'binary'),
|
||||
'.texi': ('application/x-texinfo', 'binary'),
|
||||
'.texinfo': ('application/x-texinfo', 'binary'),
|
||||
'.tif': ('image/tiff', 'image'),
|
||||
'.tiff': ('image/tiff', 'image'),
|
||||
'.tr': ('application/x-troff', 'binary'),
|
||||
'.tsv': ('text/tab-separated-values', 'document'),
|
||||
'.txt': ('text/plain', 'document'),
|
||||
'.ustar': ('application/x-ustar', 'binary'),
|
||||
'.vcf': ('text/x-vcard', 'document'),
|
||||
'.wav': ('audio/x-wav', 'audio'),
|
||||
'.webm': ('video/webm', 'video'),
|
||||
'.wiz': ('application/msword', 'document'),
|
||||
'.wsdl': ('application/xml', 'document'),
|
||||
'.xbm': ('image/x-xbitmap', 'image'),
|
||||
'.xlb': ('application/vnd.ms-excel', 'document'),
|
||||
'.xls': ('application/vnd.ms-excel', 'document'),
|
||||
'.xml': ('text/xml', 'document'),
|
||||
'.xpdl': ('application/xml', 'document'),
|
||||
'.xpm': ('image/x-xpixmap', 'image'),
|
||||
'.xsl': ('application/xml', 'document'),
|
||||
'.xwd': ('image/x-xwindowdump', 'image'),
|
||||
'.zip': ('application/zip', 'binary'),
|
||||
|
||||
# These are non-standard types, commonly found in the wild.
|
||||
'.mid': 'audio/midi',
|
||||
'.midi': 'audio/midi',
|
||||
'.pct': 'image/pict',
|
||||
'.pic': 'image/pict',
|
||||
'.pict': 'image/pict',
|
||||
'.rtf': 'application/rtf',
|
||||
'.xul': 'text/xul',
|
||||
'.m4v': 'video/m4v',
|
||||
'.cbr': ('application/vnd.comicbook-rar', 'document'),
|
||||
'.cbz': ('application/vnd.comicbook+zip', 'document'),
|
||||
'.lbry': ('application/x-ext-lbry', 'document'),
|
||||
'.mid': ('audio/midi', 'audio'),
|
||||
'.midi': ('audio/midi', 'audio'),
|
||||
'.mobi': ('application/x-mobipocket-ebook', 'document'),
|
||||
'.pct': ('image/pict', 'image'),
|
||||
'.pic': ('image/pict', 'image'),
|
||||
'.pict': ('image/pict', 'image'),
|
||||
'.prc': ('application/x-mobipocket-ebook', 'document'),
|
||||
'.rtf': ('application/rtf', 'document'),
|
||||
'.xul': ('text/xul', 'document'),
|
||||
'.m4v': ('video/m4v', 'video'),
|
||||
|
||||
# microsoft is special and has its own 'standard'
|
||||
# https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
|
||||
'.wmv': 'video/x-ms-wmv'
|
||||
'.wmv': ('video/x-ms-wmv', 'video')
|
||||
}
|
||||
|
||||
|
||||
def guess_media_type(path):
|
||||
_, ext = os.path.splitext(path)
|
||||
return types_map.get(
|
||||
ext.strip().lower(),
|
||||
'application/octet-stream' if not (ext and ext[1:].strip()) else f'application/x-ext-{ext[1:].strip().lower()}'
|
||||
)
|
||||
extension = ext.strip().lower()
|
||||
if extension[1:]:
|
||||
if extension in types_map:
|
||||
return types_map[extension]
|
||||
return f'application/x-ext-{extension[1:]}', 'binary'
|
||||
return 'application/octet-stream', 'binary'
|
||||
|
||||
|
||||
def guess_stream_type(media_type):
|
||||
for media, stream in types_map.values():
|
||||
if media == media_type:
|
||||
return stream
|
||||
return 'binary'
|
||||
@@ -19,11 +19,33 @@ DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
package='pb',
|
||||
syntax='proto3',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n\x0b\x63laim.proto\x12\x02pb\"M\n\x05\x43laim\x12\x1c\n\x06stream\x18\x01 \x01(\x0b\x32\n.pb.StreamH\x00\x12\x1e\n\x07\x63hannel\x18\x02 \x01(\x0b\x32\x0b.pb.ChannelH\x00\x42\x06\n\x04type\"\xe8\x01\n\x07\x43hannel\x12\x12\n\npublic_key\x18\x01 \x01(\x0c\x12\r\n\x05title\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x15\n\rthumbnail_url\x18\x04 \x01(\t\x12\x0c\n\x04tags\x18\x05 \x03(\t\x12\x1f\n\tlanguages\x18\x06 \x03(\x0b\x32\x0c.pb.Language\x12\x1f\n\tlocations\x18\x07 \x03(\x0b\x32\x0c.pb.Location\x12\x15\n\rcontact_email\x18\x08 \x01(\t\x12\x14\n\x0chomepage_url\x18\t \x01(\t\x12\x11\n\tcover_url\x18\n \x01(\t\"\x8e\x03\n\x06Stream\x12\x0f\n\x07sd_hash\x18\x01 \x01(\x0c\x12\r\n\x05title\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x15\n\rthumbnail_url\x18\x04 \x01(\t\x12\x0c\n\x04tags\x18\x05 \x03(\t\x12\x1f\n\tlanguages\x18\x06 \x03(\x0b\x32\x0c.pb.Language\x12\x1f\n\tlocations\x18\x07 \x03(\x0b\x32\x0c.pb.Location\x12\x0e\n\x06\x61uthor\x18\x08 \x01(\t\x12\x0f\n\x07license\x18\t \x01(\t\x12\x13\n\x0blicense_url\x18\n \x01(\t\x12\x14\n\x0crelease_time\x18\x0b \x01(\x03\x12\x12\n\nmedia_type\x18\r \x01(\t\x12\x16\n\x04\x66ile\x18\x0e \x01(\x0b\x32\x08.pb.File\x12\x14\n\x03\x66\x65\x65\x18\x0f \x01(\x0b\x32\x07.pb.Fee\x12\x1a\n\x05image\x18\x10 \x01(\x0b\x32\t.pb.ImageH\x00\x12\x1a\n\x05video\x18\x11 \x01(\x0b\x32\t.pb.VideoH\x00\x12\x1a\n\x05\x61udio\x18\x12 \x01(\x0b\x32\t.pb.AudioH\x00\x42\x06\n\x04type\"h\n\x03\x46\x65\x65\x12\"\n\x08\x63urrency\x18\x01 \x01(\x0e\x32\x10.pb.Fee.Currency\x12\x0f\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0c\x12\x0e\n\x06\x61mount\x18\x03 \x01(\x04\"\x1c\n\x08\x43urrency\x12\x07\n\x03LBC\x10\x00\x12\x07\n\x03USD\x10\x02\"0\n\x04\x46ile\x12\x0c\n\x04hash\x18\x03 \x01(\x0c\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04size\x18\x02 \x01(\x04\"&\n\x05Image\x12\r\n\x05width\x18\x01 \x01(\r\x12\x0e\n\x06height\x18\x02 \x01(\r\"8\n\x05Video\x12\r\n\x05width\x18\x01 \x01(\r\x12\x0e\n\x06height\x18\x02 \x01(\r\x12\x10\n\x08\x64uration\x18\x03 \x01(\r\"\x19\n\x05\x41udio\x12\x10\n\x08\x64uration\x18\x01 \x01(\r\"\xc7\x1d\n\x08Language\x12\'\n\x08language\x18\x01 \x01(\x0e\x32\x15.pb.Language.Language\x12#\n\x06script\x18\x02 \x01(\x0e\x32\x13.pb.Language.Script\x12$\n\x06region\x18\x03 \x01(\x0e\x32\x14.pb.Location.Country\"\x99\x0c\n\x08Language\x12\x14\n\x10UNKNOWN_LANGUAGE\x10\x00\x12\x06\n\x02\x65n\x10\x01\x12\x06\n\x02\x61\x61\x10\x02\x12\x06\n\x02\x61\x62\x10\x03\x12\x06\n\x02\x61\x65\x10\x04\x12\x06\n\x02\x61\x66\x10\x05\x12\x06\n\x02\x61k\x10\x06\x12\x06\n\x02\x61m\x10\x07\x12\x06\n\x02\x61n\x10\x08\x12\x06\n\x02\x61r\x10\t\x12\x06\n\x02\x61s\x10\n\x12\x06\n\x02\x61v\x10\x0b\x12\x06\n\x02\x61y\x10\x0c\x12\x06\n\x02\x61z\x10\r\x12\x06\n\x02\x62\x61\x10\x0e\x12\x06\n\x02\x62\x65\x10\x0f\x12\x06\n\x02\x62g\x10\x10\x12\x06\n\x02\x62h\x10\x11\x12\x06\n\x02\x62i\x10\x12\x12\x06\n\x02\x62m\x10\x13\x12\x06\n\x02\x62n\x10\x14\x12\x06\n\x02\x62o\x10\x15\x12\x06\n\x02\x62r\x10\x16\x12\x06\n\x02\x62s\x10\x17\x12\x06\n\x02\x63\x61\x10\x18\x12\x06\n\x02\x63\x65\x10\x19\x12\x06\n\x02\x63h\x10\x1a\x12\x06\n\x02\x63o\x10\x1b\x12\x06\n\x02\x63r\x10\x1c\x12\x06\n\x02\x63s\x10\x1d\x12\x06\n\x02\x63u\x10\x1e\x12\x06\n\x02\x63v\x10\x1f\x12\x06\n\x02\x63y\x10 \x12\x06\n\x02\x64\x61\x10!\x12\x06\n\x02\x64\x65\x10\"\x12\x06\n\x02\x64v\x10#\x12\x06\n\x02\x64z\x10$\x12\x06\n\x02\x65\x65\x10%\x12\x06\n\x02\x65l\x10&\x12\x06\n\x02\x65o\x10\'\x12\x06\n\x02\x65s\x10(\x12\x06\n\x02\x65t\x10)\x12\x06\n\x02\x65u\x10*\x12\x06\n\x02\x66\x61\x10+\x12\x06\n\x02\x66\x66\x10,\x12\x06\n\x02\x66i\x10-\x12\x06\n\x02\x66j\x10.\x12\x06\n\x02\x66o\x10/\x12\x06\n\x02\x66r\x10\x30\x12\x06\n\x02\x66y\x10\x31\x12\x06\n\x02ga\x10\x32\x12\x06\n\x02gd\x10\x33\x12\x06\n\x02gl\x10\x34\x12\x06\n\x02gn\x10\x35\x12\x06\n\x02gu\x10\x36\x12\x06\n\x02gv\x10\x37\x12\x06\n\x02ha\x10\x38\x12\x06\n\x02he\x10\x39\x12\x06\n\x02hi\x10:\x12\x06\n\x02ho\x10;\x12\x06\n\x02hr\x10<\x12\x06\n\x02ht\x10=\x12\x06\n\x02hu\x10>\x12\x06\n\x02hy\x10?\x12\x06\n\x02hz\x10@\x12\x06\n\x02ia\x10\x41\x12\x06\n\x02id\x10\x42\x12\x06\n\x02ie\x10\x43\x12\x06\n\x02ig\x10\x44\x12\x06\n\x02ii\x10\x45\x12\x06\n\x02ik\x10\x46\x12\x06\n\x02io\x10G\x12\x06\n\x02is\x10H\x12\x06\n\x02it\x10I\x12\x06\n\x02iu\x10J\x12\x06\n\x02ja\x10K\x12\x06\n\x02jv\x10L\x12\x06\n\x02ka\x10M\x12\x06\n\x02kg\x10N\x12\x06\n\x02ki\x10O\x12\x06\n\x02kj\x10P\x12\x06\n\x02kk\x10Q\x12\x06\n\x02kl\x10R\x12\x06\n\x02km\x10S\x12\x06\n\x02kn\x10T\x12\x06\n\x02ko\x10U\x12\x06\n\x02kr\x10V\x12\x06\n\x02ks\x10W\x12\x06\n\x02ku\x10X\x12\x06\n\x02kv\x10Y\x12\x06\n\x02kw\x10Z\x12\x06\n\x02ky\x10[\x12\x06\n\x02la\x10\\\x12\x06\n\x02lb\x10]\x12\x06\n\x02lg\x10^\x12\x06\n\x02li\x10_\x12\x06\n\x02ln\x10`\x12\x06\n\x02lo\x10\x61\x12\x06\n\x02lt\x10\x62\x12\x06\n\x02lu\x10\x63\x12\x06\n\x02lv\x10\x64\x12\x06\n\x02mg\x10\x65\x12\x06\n\x02mh\x10\x66\x12\x06\n\x02mi\x10g\x12\x06\n\x02mk\x10h\x12\x06\n\x02ml\x10i\x12\x06\n\x02mn\x10j\x12\x06\n\x02mr\Line truncated
|
||||
serialized_pb=_b('\n\x0b\x63laim.proto\x12\x02pb\"\xab\x02\n\x05\x43laim\x12\x1c\n\x06stream\x18\x01 \x01(\x0b\x32\n.pb.StreamH\x00\x12\x1e\n\x07\x63hannel\x18\x02 \x01(\x0b\x32\x0b.pb.ChannelH\x00\x12#\n\ncollection\x18\x03 \x01(\x0b\x32\r.pb.ClaimListH\x00\x12$\n\x06repost\x18\x04 \x01(\x0b\x32\x12.pb.ClaimReferenceH\x00\x12\r\n\x05title\x18\x08 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\t \x01(\t\x12\x1d\n\tthumbnail\x18\n \x01(\x0b\x32\n.pb.Source\x12\x0c\n\x04tags\x18\x0b \x03(\t\x12\x1f\n\tlanguages\x18\x0c \x03(\x0b\x32\x0c.pb.Language\x12\x1f\n\tlocations\x18\r \x03(\x0b\x32\x0c.pb.LocationB\x06\n\x04type\"\x84\x02\n\x06Stream\x12\x1a\n\x06source\x18\x01 \x01(\x0b\x32\n.pb.Source\x12\x0e\n\x06\x61uthor\x18\x02 \x01(\t\x12\x0f\n\x07license\x18\x03 \x01(\t\x12\x13\n\x0blicense_url\x18\x04 \x01(\t\x12\x14\n\x0crelease_time\x18\x05 \x01(\x03\x12\x14\n\x03\x66\x65\x65\x18\x06 \x01(\x0b\x32\x07.pb.Fee\x12\x1a\n\x05image\x18\n \x01(\x0b\x32\t.pb.ImageH\x00\x12\x1a\n\x05video\x18\x0b \x01(\x0b\x32\t.pb.VideoH\x00\x12\x1a\n\x05\x61udio\x18\x0c \x01(\x0b\x32\t.pb.AudioH\x00\x12 \n\x08software\x18\r \x01(\x0b\x32\x0c.pb.SoftwareH\x00\x42\x06\n\x04type\"}\n\x07\x43hannel\x12\x12\n\npublic_key\x18\x01 \x01(\x0c\x12\r\n\x05\x65mail\x18\x02 \x01(\t\x12\x13\n\x0bwebsite_url\x18\x03 \x01(\t\x12\x19\n\x05\x63over\x18\x04 \x01(\x0b\x32\n.pb.Source\x12\x1f\n\x08\x66\x65\x61tured\x18\x05 \x01(\x0b\x32\r.pb.ClaimList\"$\n\x0e\x43laimReference\x12\x12\n\nclaim_hash\x18\x01 \x01(\x0c\"\x90\x01\n\tClaimList\x12)\n\tlist_type\x18\x01 \x01(\x0e\x32\x16.pb.ClaimList.ListType\x12,\n\x10\x63laim_references\x18\x02 \x03(\x0b\x32\x12.pb.ClaimReference\"*\n\x08ListType\x12\x0e\n\nCOLLECTION\x10\x00\x12\x0e\n\nDERIVATION\x10\x02\"d\n\x06Source\x12\x0c\n\x04hash\x18\x01 \x01(\x0c\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0c\n\x04size\x18\x03 \x01(\x04\x12\x12\n\nmedia_type\x18\x04 \x01(\t\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0f\n\x07sd_hash\x18\x06 \x01(\x0c\"\x87\x01\n\x03\x46\x65\x65\x12\"\n\x08\x63urrency\x18\x01 \x01(\x0e\x32\x10.pb.Fee.Currency\x12\x0f\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0c\x12\x0e\n\x06\x61mount\x18\x03 \x01(\x04\";\n\x08\x43urrency\x12\x14\n\x10UNKNOWN_CURRENCY\x10\x00\x12\x07\n\x03LBC\x10\x01\x12\x07\n\x03\x42TC\x10\x02\x12\x07\n\x03USD\x10\x03\"&\n\x05Image\x12\r\n\x05width\x18\x01 \x01(\r\x12\x0e\n\x06height\x18\x02 \x01(\r\"R\n\x05Video\x12\r\n\x05width\x18\x01 \x01(\r\x12\x0e\n\x06height\x18\x02 \x01(\r\x12\x10\n\x08\x64uration\x18\x03 \x01(\r\x12\x18\n\x05\x61udio\x18\x0f \x01(\x0b\x32\t.pb.Audio\"\x19\n\x05\x41udio\x12\x10\n\x08\x64uration\x18\x01 \x01(\r\"l\n\x08Software\x12\n\n\x02os\x18\x01 \x01(\t\"T\n\x02OS\x12\x0e\n\nUNKNOWN_OS\x10\x00\x12\x07\n\x03\x41NY\x10\x01\x12\t\n\x05LINUX\x10\x02\x12\x0b\n\x07WINDOWS\x10\x03\x12\x07\n\x03MAC\x10\x04\x12\x0b\n\x07\x41NDROID\x10\x05\x12\x07\n\x03IOS\x10\x06\"\xc7\x1d\n\x08Language\x12\'\n\x08language\x18\x01 \x01(\x0e\x32\x15.pb.Language.Language\x12#\n\x06script\x18\x02 \x01(\x0e\x32\x13.pb.Language.Script\x12$\n\x06region\x18\x03 \x01(\x0e\x32\x14.pb.Location.Country\"\x99\x0c\n\x08Language\x12\x14\n\x10UNKNOWN_LANGUAGE\x10\x00\x12\x06\n\x02\x65n\x10\x01\x12\x06\n\x02\x61\x61\x10\x02\x12\x06\n\x02\x61\x62\x10\x03\x12\x06\n\x02\x61\x65\x10\x04\x12\x06\n\x02\x61\x66\x10\x05\x12\x06\n\x02\x61k\x10\x06\x12\x06\n\x02\x61m\x10\x07\x12\x06\n\x02\x61n\x10\x08\x12\x06\n\x02\x61r\x10\t\x12\x06\n\x02\x61s\x10\n\x12\x06\n\x02\x61v\x10\x0b\x12\x06\n\x02\x61y\x10\x0c\x12\x06\n\x02\x61z\x10\r\x12\x06\n\x02\x62\x61\x10\x0e\x12\x06\n\x02\x62\x65\x10\x0f\x12\x06\n\x02\x62g\x10\x10\x12\x06\n\x02\x62h\x10\x11\x12\x06\n\x02\x62i\x10\x12\x12\x06\n\x02\x62m\x10\x13\x12\x06\n\x02\x62n\x10\x14\x12\x06\n\x02\x62o\x10\x15\x12\x06\n\x02\x62r\x10\x16\x12\x06\n\x02\x62s\x10\x17\x12\x06\n\x02\x63\x61\x10\x18\x12\x06\n\x02\x63\x65\x10\x19\x12\x06\n\x02\x63h\x10\x1a\x12\x06\n\x02\x63o\x10\x1b\x12\x06\n\x02\x63r\x10\x1c\x12\x06\n\x02\x63s\x10\x1d\x12\x06\n\x02\x63u\x10\x1e\x12\x06\n\x02\x63v\x10\x1f\x12\x06\n\x02\x63y\x10 \x12\x06\n\x02\x64\x61\x10!\x12\x06\n\x02\x64\x65\x10\"\x12\x06\n\x02\x64v\x10#\x12\x06\n\x02\x64z\x10$\x12\x06\n\x02\x65\x65\x10%\x12\x06\n\x02\x65l\x10&\x12\x06\n\x02\x65o\x10\'\x12\x06\n\x02\x65s\x10(\x12\x06\n\x02\x65t\x10)\x12\x06\n\x02\x65u\x10*\x12\x06\n\x02\x66\x61\x10+\x12\x06\n\x02\x66\x66\x10,\x12\x06\n\x02\x66i\x10-\x12\x06\n\x02\x66j\x10.\x12\x06\n\x02\x66o\x10/\x12\x06\n\x02\x66r\x10\x30\x12\x06\n\x02\x66y\x10\x31\x12\x06\n\x02ga\x10\x32\x12\x06\n\x02gd\x10\x33\x12\x06\n\x02gl\x10\x34\x12\x06\n\x02gn\x10\x35\x12\x06\n\x02gu\x10\x36\x12\x06\n\x02gv\x10\x37\x12\x06\n\x02ha\x10\x38\x12\x06\n\x02he\x10\x39\x12\x06\n\x02hi\x10:\x12\x06\n\x02ho\x10;\x12\x06\n\x02hr\x10<\x12\x06\n\x02ht\x10=\x12\x06\n\x02hu\x10>\x12\x06\n\x02hy\x10?\x12\x06\n\x02hz\x10@\x12\x06\n\x02ia\x10\x41\x12\x06\n\x02id\x10\x42\x12\x06\n\x02ie\x10\x43\x12\x06\n\x02ig\x10\x44\x12\x06\n\x02ii\x10\x45\x12\x06\n\x02ik\x10\x46\x12\x06\n\x02io\x10G\x12\x06\n\x02is\x10H\x12\x06\n\x02it\x10I\x12\x0Line truncated
|
||||
)
|
||||
|
||||
|
||||
|
||||
_CLAIMLIST_LISTTYPE = _descriptor.EnumDescriptor(
|
||||
name='ListType',
|
||||
full_name='pb.ClaimList.ListType',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='COLLECTION', index=0, number=0,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='DERIVATION', index=1, number=2,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=852,
|
||||
serialized_end=894,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_CLAIMLIST_LISTTYPE)
|
||||
|
||||
_FEE_CURRENCY = _descriptor.EnumDescriptor(
|
||||
name='Currency',
|
||||
full_name='pb.Fee.Currency',
|
||||
@@ -31,21 +53,71 @@ _FEE_CURRENCY = _descriptor.EnumDescriptor(
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='LBC', index=0, number=0,
|
||||
name='UNKNOWN_CURRENCY', index=0, number=0,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='USD', index=1, number=2,
|
||||
name='LBC', index=1, number=1,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='BTC', index=2, number=2,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='USD', index=3, number=3,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=810,
|
||||
serialized_end=838,
|
||||
serialized_start=1075,
|
||||
serialized_end=1134,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_FEE_CURRENCY)
|
||||
|
||||
_SOFTWARE_OS = _descriptor.EnumDescriptor(
|
||||
name='OS',
|
||||
full_name='pb.Software.OS',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='UNKNOWN_OS', index=0, number=0,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ANY', index=1, number=1,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='LINUX', index=2, number=2,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='WINDOWS', index=3, number=3,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='MAC', index=4, number=4,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='ANDROID', index=5, number=5,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
_descriptor.EnumValueDescriptor(
|
||||
name='IOS', index=6, number=6,
|
||||
serialized_options=None,
|
||||
type=None),
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=1311,
|
||||
serialized_end=1395,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_SOFTWARE_OS)
|
||||
|
||||
_LANGUAGE_LANGUAGE = _descriptor.EnumDescriptor(
|
||||
name='Language',
|
||||
full_name='pb.Language.Language',
|
||||
@@ -795,8 +867,8 @@ _LANGUAGE_LANGUAGE = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=1145,
|
||||
serialized_end=2706,
|
||||
serialized_start=1527,
|
||||
serialized_end=3088,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_LANGUAGE_LANGUAGE)
|
||||
|
||||
@@ -1609,8 +1681,8 @@ _LANGUAGE_SCRIPT = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=2709,
|
||||
serialized_end=4799,
|
||||
serialized_start=3091,
|
||||
serialized_end=5181,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_LANGUAGE_SCRIPT)
|
||||
|
||||
@@ -2623,8 +2695,8 @@ _LOCATION_COUNTRY = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=4934,
|
||||
serialized_end=7078,
|
||||
serialized_start=5316,
|
||||
serialized_end=7460,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_LOCATION_COUNTRY)
|
||||
|
||||
@@ -2650,6 +2722,62 @@ _CLAIM = _descriptor.Descriptor(
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='collection', full_name='pb.Claim.collection', index=2,
|
||||
number=3, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='repost', full_name='pb.Claim.repost', index=3,
|
||||
number=4, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='title', full_name='pb.Claim.title', index=4,
|
||||
number=8, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='description', full_name='pb.Claim.description', index=5,
|
||||
number=9, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='thumbnail', full_name='pb.Claim.thumbnail', index=6,
|
||||
number=10, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='tags', full_name='pb.Claim.tags', index=7,
|
||||
number=11, type=9, cpp_type=9, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='languages', full_name='pb.Claim.languages', index=8,
|
||||
number=12, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='locations', full_name='pb.Claim.locations', index=9,
|
||||
number=13, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
@@ -2665,102 +2793,8 @@ _CLAIM = _descriptor.Descriptor(
|
||||
name='type', full_name='pb.Claim.type',
|
||||
index=0, containing_type=None, fields=[]),
|
||||
],
|
||||
serialized_start=19,
|
||||
serialized_end=96,
|
||||
)
|
||||
|
||||
|
||||
_CHANNEL = _descriptor.Descriptor(
|
||||
name='Channel',
|
||||
full_name='pb.Channel',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='public_key', full_name='pb.Channel.public_key', index=0,
|
||||
number=1, type=12, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='title', full_name='pb.Channel.title', index=1,
|
||||
number=2, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='description', full_name='pb.Channel.description', index=2,
|
||||
number=3, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='thumbnail_url', full_name='pb.Channel.thumbnail_url', index=3,
|
||||
number=4, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='tags', full_name='pb.Channel.tags', index=4,
|
||||
number=5, type=9, cpp_type=9, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='languages', full_name='pb.Channel.languages', index=5,
|
||||
number=6, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='locations', full_name='pb.Channel.locations', index=6,
|
||||
number=7, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='contact_email', full_name='pb.Channel.contact_email', index=7,
|
||||
number=8, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='homepage_url', full_name='pb.Channel.homepage_url', index=8,
|
||||
number=9, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='cover_url', full_name='pb.Channel.cover_url', index=9,
|
||||
number=10, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=99,
|
||||
serialized_end=331,
|
||||
serialized_start=20,
|
||||
serialized_end=319,
|
||||
)
|
||||
|
||||
|
||||
@@ -2772,120 +2806,71 @@ _STREAM = _descriptor.Descriptor(
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='sd_hash', full_name='pb.Stream.sd_hash', index=0,
|
||||
number=1, type=12, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
name='source', full_name='pb.Stream.source', index=0,
|
||||
number=1, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='title', full_name='pb.Stream.title', index=1,
|
||||
name='author', full_name='pb.Stream.author', index=1,
|
||||
number=2, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='description', full_name='pb.Stream.description', index=2,
|
||||
name='license', full_name='pb.Stream.license', index=2,
|
||||
number=3, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='thumbnail_url', full_name='pb.Stream.thumbnail_url', index=3,
|
||||
name='license_url', full_name='pb.Stream.license_url', index=3,
|
||||
number=4, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='tags', full_name='pb.Stream.tags', index=4,
|
||||
number=5, type=9, cpp_type=9, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='languages', full_name='pb.Stream.languages', index=5,
|
||||
number=6, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='locations', full_name='pb.Stream.locations', index=6,
|
||||
number=7, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='author', full_name='pb.Stream.author', index=7,
|
||||
number=8, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='license', full_name='pb.Stream.license', index=8,
|
||||
number=9, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='license_url', full_name='pb.Stream.license_url', index=9,
|
||||
number=10, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='release_time', full_name='pb.Stream.release_time', index=10,
|
||||
number=11, type=3, cpp_type=2, label=1,
|
||||
name='release_time', full_name='pb.Stream.release_time', index=4,
|
||||
number=5, type=3, cpp_type=2, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='media_type', full_name='pb.Stream.media_type', index=11,
|
||||
number=13, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='file', full_name='pb.Stream.file', index=12,
|
||||
number=14, type=11, cpp_type=10, label=1,
|
||||
name='fee', full_name='pb.Stream.fee', index=5,
|
||||
number=6, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='fee', full_name='pb.Stream.fee', index=13,
|
||||
number=15, type=11, cpp_type=10, label=1,
|
||||
name='image', full_name='pb.Stream.image', index=6,
|
||||
number=10, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='image', full_name='pb.Stream.image', index=14,
|
||||
number=16, type=11, cpp_type=10, label=1,
|
||||
name='video', full_name='pb.Stream.video', index=7,
|
||||
number=11, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='video', full_name='pb.Stream.video', index=15,
|
||||
number=17, type=11, cpp_type=10, label=1,
|
||||
name='audio', full_name='pb.Stream.audio', index=8,
|
||||
number=12, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='audio', full_name='pb.Stream.audio', index=16,
|
||||
number=18, type=11, cpp_type=10, label=1,
|
||||
name='software', full_name='pb.Stream.software', index=9,
|
||||
number=13, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
@@ -2905,8 +2890,203 @@ _STREAM = _descriptor.Descriptor(
|
||||
name='type', full_name='pb.Stream.type',
|
||||
index=0, containing_type=None, fields=[]),
|
||||
],
|
||||
serialized_start=334,
|
||||
serialized_end=732,
|
||||
serialized_start=322,
|
||||
serialized_end=582,
|
||||
)
|
||||
|
||||
|
||||
_CHANNEL = _descriptor.Descriptor(
|
||||
name='Channel',
|
||||
full_name='pb.Channel',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='public_key', full_name='pb.Channel.public_key', index=0,
|
||||
number=1, type=12, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='email', full_name='pb.Channel.email', index=1,
|
||||
number=2, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='website_url', full_name='pb.Channel.website_url', index=2,
|
||||
number=3, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='cover', full_name='pb.Channel.cover', index=3,
|
||||
number=4, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='featured', full_name='pb.Channel.featured', index=4,
|
||||
number=5, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=584,
|
||||
serialized_end=709,
|
||||
)
|
||||
|
||||
|
||||
_CLAIMREFERENCE = _descriptor.Descriptor(
|
||||
name='ClaimReference',
|
||||
full_name='pb.ClaimReference',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='claim_hash', full_name='pb.ClaimReference.claim_hash', index=0,
|
||||
number=1, type=12, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=711,
|
||||
serialized_end=747,
|
||||
)
|
||||
|
||||
|
||||
_CLAIMLIST = _descriptor.Descriptor(
|
||||
name='ClaimList',
|
||||
full_name='pb.ClaimList',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='list_type', full_name='pb.ClaimList.list_type', index=0,
|
||||
number=1, type=14, cpp_type=8, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='claim_references', full_name='pb.ClaimList.claim_references', index=1,
|
||||
number=2, type=11, cpp_type=10, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
_CLAIMLIST_LISTTYPE,
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=750,
|
||||
serialized_end=894,
|
||||
)
|
||||
|
||||
|
||||
_SOURCE = _descriptor.Descriptor(
|
||||
name='Source',
|
||||
full_name='pb.Source',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='hash', full_name='pb.Source.hash', index=0,
|
||||
number=1, type=12, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='name', full_name='pb.Source.name', index=1,
|
||||
number=2, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='size', full_name='pb.Source.size', index=2,
|
||||
number=3, type=4, cpp_type=4, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='media_type', full_name='pb.Source.media_type', index=3,
|
||||
number=4, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='url', full_name='pb.Source.url', index=4,
|
||||
number=5, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='sd_hash', full_name='pb.Source.sd_hash', index=5,
|
||||
number=6, type=12, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=896,
|
||||
serialized_end=996,
|
||||
)
|
||||
|
||||
|
||||
@@ -2951,53 +3131,8 @@ _FEE = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=734,
|
||||
serialized_end=838,
|
||||
)
|
||||
|
||||
|
||||
_FILE = _descriptor.Descriptor(
|
||||
name='File',
|
||||
full_name='pb.File',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='hash', full_name='pb.File.hash', index=0,
|
||||
number=3, type=12, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='name', full_name='pb.File.name', index=1,
|
||||
number=1, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='size', full_name='pb.File.size', index=2,
|
||||
number=2, type=4, cpp_type=4, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=840,
|
||||
serialized_end=888,
|
||||
serialized_start=999,
|
||||
serialized_end=1134,
|
||||
)
|
||||
|
||||
|
||||
@@ -3034,8 +3169,8 @@ _IMAGE = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=890,
|
||||
serialized_end=928,
|
||||
serialized_start=1136,
|
||||
serialized_end=1174,
|
||||
)
|
||||
|
||||
|
||||
@@ -3067,6 +3202,13 @@ _VIDEO = _descriptor.Descriptor(
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='audio', full_name='pb.Video.audio', index=3,
|
||||
number=15, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
@@ -3079,8 +3221,8 @@ _VIDEO = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=930,
|
||||
serialized_end=986,
|
||||
serialized_start=1176,
|
||||
serialized_end=1258,
|
||||
)
|
||||
|
||||
|
||||
@@ -3110,8 +3252,40 @@ _AUDIO = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=988,
|
||||
serialized_end=1013,
|
||||
serialized_start=1260,
|
||||
serialized_end=1285,
|
||||
)
|
||||
|
||||
|
||||
_SOFTWARE = _descriptor.Descriptor(
|
||||
name='Software',
|
||||
full_name='pb.Software',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='os', full_name='pb.Software.os', index=0,
|
||||
number=1, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
_SOFTWARE_OS,
|
||||
],
|
||||
serialized_options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=1287,
|
||||
serialized_end=1395,
|
||||
)
|
||||
|
||||
|
||||
@@ -3157,8 +3331,8 @@ _LANGUAGE = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=1016,
|
||||
serialized_end=4799,
|
||||
serialized_start=1398,
|
||||
serialized_end=5181,
|
||||
)
|
||||
|
||||
|
||||
@@ -3199,15 +3373,15 @@ _LOCATION = _descriptor.Descriptor(
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='latitude', full_name='pb.Location.latitude', index=4,
|
||||
number=5, type=1, cpp_type=5, label=1,
|
||||
has_default_value=False, default_value=float(0),
|
||||
number=5, type=17, cpp_type=1, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='longitude', full_name='pb.Location.longitude', index=5,
|
||||
number=6, type=1, cpp_type=5, label=1,
|
||||
has_default_value=False, default_value=float(0),
|
||||
number=6, type=17, cpp_type=1, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
@@ -3224,27 +3398,35 @@ _LOCATION = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=4802,
|
||||
serialized_end=7078,
|
||||
serialized_start=5184,
|
||||
serialized_end=7460,
|
||||
)
|
||||
|
||||
_CLAIM.fields_by_name['stream'].message_type = _STREAM
|
||||
_CLAIM.fields_by_name['channel'].message_type = _CHANNEL
|
||||
_CLAIM.fields_by_name['collection'].message_type = _CLAIMLIST
|
||||
_CLAIM.fields_by_name['repost'].message_type = _CLAIMREFERENCE
|
||||
_CLAIM.fields_by_name['thumbnail'].message_type = _SOURCE
|
||||
_CLAIM.fields_by_name['languages'].message_type = _LANGUAGE
|
||||
_CLAIM.fields_by_name['locations'].message_type = _LOCATION
|
||||
_CLAIM.oneofs_by_name['type'].fields.append(
|
||||
_CLAIM.fields_by_name['stream'])
|
||||
_CLAIM.fields_by_name['stream'].containing_oneof = _CLAIM.oneofs_by_name['type']
|
||||
_CLAIM.oneofs_by_name['type'].fields.append(
|
||||
_CLAIM.fields_by_name['channel'])
|
||||
_CLAIM.fields_by_name['channel'].containing_oneof = _CLAIM.oneofs_by_name['type']
|
||||
_CHANNEL.fields_by_name['languages'].message_type = _LANGUAGE
|
||||
_CHANNEL.fields_by_name['locations'].message_type = _LOCATION
|
||||
_STREAM.fields_by_name['languages'].message_type = _LANGUAGE
|
||||
_STREAM.fields_by_name['locations'].message_type = _LOCATION
|
||||
_STREAM.fields_by_name['file'].message_type = _FILE
|
||||
_CLAIM.oneofs_by_name['type'].fields.append(
|
||||
_CLAIM.fields_by_name['collection'])
|
||||
_CLAIM.fields_by_name['collection'].containing_oneof = _CLAIM.oneofs_by_name['type']
|
||||
_CLAIM.oneofs_by_name['type'].fields.append(
|
||||
_CLAIM.fields_by_name['repost'])
|
||||
_CLAIM.fields_by_name['repost'].containing_oneof = _CLAIM.oneofs_by_name['type']
|
||||
_STREAM.fields_by_name['source'].message_type = _SOURCE
|
||||
_STREAM.fields_by_name['fee'].message_type = _FEE
|
||||
_STREAM.fields_by_name['image'].message_type = _IMAGE
|
||||
_STREAM.fields_by_name['video'].message_type = _VIDEO
|
||||
_STREAM.fields_by_name['audio'].message_type = _AUDIO
|
||||
_STREAM.fields_by_name['software'].message_type = _SOFTWARE
|
||||
_STREAM.oneofs_by_name['type'].fields.append(
|
||||
_STREAM.fields_by_name['image'])
|
||||
_STREAM.fields_by_name['image'].containing_oneof = _STREAM.oneofs_by_name['type']
|
||||
@@ -3254,8 +3436,18 @@ _STREAM.fields_by_name['video'].containing_oneof = _STREAM.oneofs_by_name['type'
|
||||
_STREAM.oneofs_by_name['type'].fields.append(
|
||||
_STREAM.fields_by_name['audio'])
|
||||
_STREAM.fields_by_name['audio'].containing_oneof = _STREAM.oneofs_by_name['type']
|
||||
_STREAM.oneofs_by_name['type'].fields.append(
|
||||
_STREAM.fields_by_name['software'])
|
||||
_STREAM.fields_by_name['software'].containing_oneof = _STREAM.oneofs_by_name['type']
|
||||
_CHANNEL.fields_by_name['cover'].message_type = _SOURCE
|
||||
_CHANNEL.fields_by_name['featured'].message_type = _CLAIMLIST
|
||||
_CLAIMLIST.fields_by_name['list_type'].enum_type = _CLAIMLIST_LISTTYPE
|
||||
_CLAIMLIST.fields_by_name['claim_references'].message_type = _CLAIMREFERENCE
|
||||
_CLAIMLIST_LISTTYPE.containing_type = _CLAIMLIST
|
||||
_FEE.fields_by_name['currency'].enum_type = _FEE_CURRENCY
|
||||
_FEE_CURRENCY.containing_type = _FEE
|
||||
_VIDEO.fields_by_name['audio'].message_type = _AUDIO
|
||||
_SOFTWARE_OS.containing_type = _SOFTWARE
|
||||
_LANGUAGE.fields_by_name['language'].enum_type = _LANGUAGE_LANGUAGE
|
||||
_LANGUAGE.fields_by_name['script'].enum_type = _LANGUAGE_SCRIPT
|
||||
_LANGUAGE.fields_by_name['region'].enum_type = _LOCATION_COUNTRY
|
||||
@@ -3264,13 +3456,16 @@ _LANGUAGE_SCRIPT.containing_type = _LANGUAGE
|
||||
_LOCATION.fields_by_name['country'].enum_type = _LOCATION_COUNTRY
|
||||
_LOCATION_COUNTRY.containing_type = _LOCATION
|
||||
DESCRIPTOR.message_types_by_name['Claim'] = _CLAIM
|
||||
DESCRIPTOR.message_types_by_name['Channel'] = _CHANNEL
|
||||
DESCRIPTOR.message_types_by_name['Stream'] = _STREAM
|
||||
DESCRIPTOR.message_types_by_name['Channel'] = _CHANNEL
|
||||
DESCRIPTOR.message_types_by_name['ClaimReference'] = _CLAIMREFERENCE
|
||||
DESCRIPTOR.message_types_by_name['ClaimList'] = _CLAIMLIST
|
||||
DESCRIPTOR.message_types_by_name['Source'] = _SOURCE
|
||||
DESCRIPTOR.message_types_by_name['Fee'] = _FEE
|
||||
DESCRIPTOR.message_types_by_name['File'] = _FILE
|
||||
DESCRIPTOR.message_types_by_name['Image'] = _IMAGE
|
||||
DESCRIPTOR.message_types_by_name['Video'] = _VIDEO
|
||||
DESCRIPTOR.message_types_by_name['Audio'] = _AUDIO
|
||||
DESCRIPTOR.message_types_by_name['Software'] = _SOFTWARE
|
||||
DESCRIPTOR.message_types_by_name['Language'] = _LANGUAGE
|
||||
DESCRIPTOR.message_types_by_name['Location'] = _LOCATION
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
@@ -3282,13 +3477,6 @@ Claim = _reflection.GeneratedProtocolMessageType('Claim', (_message.Message,), d
|
||||
))
|
||||
_sym_db.RegisterMessage(Claim)
|
||||
|
||||
Channel = _reflection.GeneratedProtocolMessageType('Channel', (_message.Message,), dict(
|
||||
DESCRIPTOR = _CHANNEL,
|
||||
__module__ = 'claim_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.Channel)
|
||||
))
|
||||
_sym_db.RegisterMessage(Channel)
|
||||
|
||||
Stream = _reflection.GeneratedProtocolMessageType('Stream', (_message.Message,), dict(
|
||||
DESCRIPTOR = _STREAM,
|
||||
__module__ = 'claim_pb2'
|
||||
@@ -3296,6 +3484,34 @@ Stream = _reflection.GeneratedProtocolMessageType('Stream', (_message.Message,),
|
||||
))
|
||||
_sym_db.RegisterMessage(Stream)
|
||||
|
||||
Channel = _reflection.GeneratedProtocolMessageType('Channel', (_message.Message,), dict(
|
||||
DESCRIPTOR = _CHANNEL,
|
||||
__module__ = 'claim_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.Channel)
|
||||
))
|
||||
_sym_db.RegisterMessage(Channel)
|
||||
|
||||
ClaimReference = _reflection.GeneratedProtocolMessageType('ClaimReference', (_message.Message,), dict(
|
||||
DESCRIPTOR = _CLAIMREFERENCE,
|
||||
__module__ = 'claim_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.ClaimReference)
|
||||
))
|
||||
_sym_db.RegisterMessage(ClaimReference)
|
||||
|
||||
ClaimList = _reflection.GeneratedProtocolMessageType('ClaimList', (_message.Message,), dict(
|
||||
DESCRIPTOR = _CLAIMLIST,
|
||||
__module__ = 'claim_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.ClaimList)
|
||||
))
|
||||
_sym_db.RegisterMessage(ClaimList)
|
||||
|
||||
Source = _reflection.GeneratedProtocolMessageType('Source', (_message.Message,), dict(
|
||||
DESCRIPTOR = _SOURCE,
|
||||
__module__ = 'claim_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.Source)
|
||||
))
|
||||
_sym_db.RegisterMessage(Source)
|
||||
|
||||
Fee = _reflection.GeneratedProtocolMessageType('Fee', (_message.Message,), dict(
|
||||
DESCRIPTOR = _FEE,
|
||||
__module__ = 'claim_pb2'
|
||||
@@ -3303,13 +3519,6 @@ Fee = _reflection.GeneratedProtocolMessageType('Fee', (_message.Message,), dict(
|
||||
))
|
||||
_sym_db.RegisterMessage(Fee)
|
||||
|
||||
File = _reflection.GeneratedProtocolMessageType('File', (_message.Message,), dict(
|
||||
DESCRIPTOR = _FILE,
|
||||
__module__ = 'claim_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.File)
|
||||
))
|
||||
_sym_db.RegisterMessage(File)
|
||||
|
||||
Image = _reflection.GeneratedProtocolMessageType('Image', (_message.Message,), dict(
|
||||
DESCRIPTOR = _IMAGE,
|
||||
__module__ = 'claim_pb2'
|
||||
@@ -3331,6 +3540,13 @@ Audio = _reflection.GeneratedProtocolMessageType('Audio', (_message.Message,), d
|
||||
))
|
||||
_sym_db.RegisterMessage(Audio)
|
||||
|
||||
Software = _reflection.GeneratedProtocolMessageType('Software', (_message.Message,), dict(
|
||||
DESCRIPTOR = _SOFTWARE,
|
||||
__module__ = 'claim_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.Software)
|
||||
))
|
||||
_sym_db.RegisterMessage(Software)
|
||||
|
||||
Language = _reflection.GeneratedProtocolMessageType('Language', (_message.Message,), dict(
|
||||
DESCRIPTOR = _LANGUAGE,
|
||||
__module__ = 'claim_pb2'
|
||||
|
||||
@@ -1,136 +0,0 @@
|
||||
import os
|
||||
import binascii
|
||||
import logging
|
||||
import typing
|
||||
import asyncio
|
||||
from lbrynet.blob import MAX_BLOB_SIZE
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_info import BlobInfo
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_next_available_file_name(download_directory: str, file_name: str) -> str:
|
||||
base_name, ext = os.path.splitext(os.path.basename(file_name))
|
||||
i = 0
|
||||
while os.path.isfile(os.path.join(download_directory, file_name)):
|
||||
i += 1
|
||||
file_name = "%s_%i%s" % (base_name, i, ext)
|
||||
|
||||
return file_name
|
||||
|
||||
|
||||
async def get_next_available_file_name(loop: asyncio.BaseEventLoop, download_directory: str, file_name: str) -> str:
|
||||
return await loop.run_in_executor(None, _get_next_available_file_name, download_directory, file_name)
|
||||
|
||||
|
||||
class StreamAssembler:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', sd_hash: str,
|
||||
output_file_name: typing.Optional[str] = None):
|
||||
self.output_file_name = output_file_name
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.sd_hash = sd_hash
|
||||
self.sd_blob: 'BlobFile' = None
|
||||
self.descriptor: StreamDescriptor = None
|
||||
self.got_descriptor = asyncio.Event(loop=self.loop)
|
||||
self.wrote_bytes_event = asyncio.Event(loop=self.loop)
|
||||
self.stream_finished_event = asyncio.Event(loop=self.loop)
|
||||
self.output_path = ''
|
||||
self.stream_handle = None
|
||||
self.written_bytes: int = 0
|
||||
|
||||
async def _decrypt_blob(self, blob: 'BlobFile', blob_info: 'BlobInfo', key: str):
|
||||
if not blob or not self.stream_handle or self.stream_handle.closed:
|
||||
return False
|
||||
|
||||
def _decrypt_and_write():
|
||||
offset = blob_info.blob_num * (MAX_BLOB_SIZE - 1)
|
||||
self.stream_handle.seek(offset)
|
||||
_decrypted = blob.decrypt(
|
||||
binascii.unhexlify(key), binascii.unhexlify(blob_info.iv.encode())
|
||||
)
|
||||
self.stream_handle.write(_decrypted)
|
||||
self.stream_handle.flush()
|
||||
self.written_bytes += len(_decrypted)
|
||||
log.debug("decrypted %s", blob.blob_hash[:8])
|
||||
|
||||
await self.loop.run_in_executor(None, _decrypt_and_write)
|
||||
return True
|
||||
|
||||
async def setup(self):
|
||||
pass
|
||||
|
||||
async def after_got_descriptor(self):
|
||||
pass
|
||||
|
||||
async def after_finished(self):
|
||||
pass
|
||||
|
||||
async def assemble_decrypted_stream(self, output_dir: str, output_file_name: typing.Optional[str] = None):
|
||||
if not os.path.isdir(output_dir):
|
||||
raise OSError(f"output directory does not exist: '{output_dir}' '{output_file_name}'")
|
||||
await self.setup()
|
||||
self.sd_blob = await self.get_blob(self.sd_hash)
|
||||
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(self.loop, self.blob_manager.blob_dir,
|
||||
self.sd_blob)
|
||||
await self.after_got_descriptor()
|
||||
self.output_file_name = output_file_name or self.descriptor.suggested_file_name
|
||||
self.output_file_name = await get_next_available_file_name(self.loop, output_dir, self.output_file_name)
|
||||
self.output_path = os.path.join(output_dir, self.output_file_name)
|
||||
if not self.got_descriptor.is_set():
|
||||
self.got_descriptor.set()
|
||||
await self.blob_manager.storage.store_stream(
|
||||
self.sd_blob, self.descriptor
|
||||
)
|
||||
await self.blob_manager.blob_completed(self.sd_blob)
|
||||
written_blobs = None
|
||||
save_tasks = []
|
||||
try:
|
||||
with open(self.output_path, 'wb') as stream_handle:
|
||||
self.stream_handle = stream_handle
|
||||
for i, blob_info in enumerate(self.descriptor.blobs[:-1]):
|
||||
if blob_info.blob_num != i:
|
||||
log.error("sd blob %s is invalid, cannot assemble stream", self.descriptor.sd_hash)
|
||||
return
|
||||
while self.stream_handle and not self.stream_handle.closed:
|
||||
try:
|
||||
blob = await self.get_blob(blob_info.blob_hash, blob_info.length)
|
||||
if blob and blob.length != blob_info.length:
|
||||
log.warning("Found incomplete, deleting: %s", blob_info.blob_hash)
|
||||
await self.blob_manager.delete_blobs([blob_info.blob_hash])
|
||||
continue
|
||||
if await self._decrypt_blob(blob, blob_info, self.descriptor.key):
|
||||
save_tasks.append(asyncio.ensure_future(self.blob_manager.blob_completed(blob)))
|
||||
written_blobs = i
|
||||
if not self.wrote_bytes_event.is_set():
|
||||
self.wrote_bytes_event.set()
|
||||
log.debug("written %i/%i", written_blobs, len(self.descriptor.blobs) - 2)
|
||||
break
|
||||
except FileNotFoundError:
|
||||
log.debug("stream assembler stopped")
|
||||
return
|
||||
except (ValueError, IOError, OSError):
|
||||
log.warning("failed to decrypt blob %s for stream %s", blob_info.blob_hash,
|
||||
self.descriptor.sd_hash)
|
||||
continue
|
||||
finally:
|
||||
if written_blobs == len(self.descriptor.blobs) - 2:
|
||||
log.debug("finished decrypting and assembling stream")
|
||||
if save_tasks:
|
||||
await asyncio.wait(save_tasks)
|
||||
await self.after_finished()
|
||||
self.stream_finished_event.set()
|
||||
else:
|
||||
log.debug("stream decryption and assembly did not finish (%i/%i blobs are done)", written_blobs or 0,
|
||||
len(self.descriptor.blobs) - 2)
|
||||
if self.output_path and os.path.isfile(self.output_path):
|
||||
log.debug("erasing incomplete file assembly: %s", self.output_path)
|
||||
os.unlink(self.output_path)
|
||||
|
||||
async def get_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
|
||||
return self.blob_manager.get_blob(blob_hash, length)
|
||||
@@ -8,7 +8,7 @@ from collections import OrderedDict
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from lbrynet.blob import MAX_BLOB_SIZE
|
||||
from lbrynet.blob.blob_info import BlobInfo
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from lbrynet.blob.blob_file import AbstractBlob, BlobFile
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.error import InvalidStreamDescriptorError
|
||||
|
||||
@@ -47,6 +47,17 @@ def file_reader(file_path: str):
|
||||
|
||||
|
||||
class StreamDescriptor:
|
||||
__slots__ = [
|
||||
'loop',
|
||||
'blob_dir',
|
||||
'stream_name',
|
||||
'key',
|
||||
'suggested_file_name',
|
||||
'blobs',
|
||||
'stream_hash',
|
||||
'sd_hash'
|
||||
]
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, stream_name: str, key: str,
|
||||
suggested_file_name: str, blobs: typing.List[BlobInfo], stream_hash: typing.Optional[str] = None,
|
||||
sd_hash: typing.Optional[str] = None):
|
||||
@@ -108,29 +119,30 @@ class StreamDescriptor:
|
||||
h.update(self.old_sort_json())
|
||||
return h.hexdigest()
|
||||
|
||||
async def make_sd_blob(self, blob_file_obj: typing.Optional[BlobFile] = None,
|
||||
old_sort: typing.Optional[bool] = False):
|
||||
async def make_sd_blob(self, blob_file_obj: typing.Optional[AbstractBlob] = None,
|
||||
old_sort: typing.Optional[bool] = False,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None):
|
||||
sd_hash = self.calculate_sd_hash() if not old_sort else self.calculate_old_sort_sd_hash()
|
||||
if not old_sort:
|
||||
sd_data = self.as_json()
|
||||
else:
|
||||
sd_data = self.old_sort_json()
|
||||
sd_blob = blob_file_obj or BlobFile(self.loop, self.blob_dir, sd_hash, len(sd_data))
|
||||
sd_blob = blob_file_obj or BlobFile(self.loop, sd_hash, len(sd_data), blob_completed_callback, self.blob_dir)
|
||||
if blob_file_obj:
|
||||
blob_file_obj.set_length(len(sd_data))
|
||||
if not sd_blob.get_is_verified():
|
||||
writer = sd_blob.open_for_writing()
|
||||
writer = sd_blob.get_blob_writer()
|
||||
writer.write(sd_data)
|
||||
|
||||
await sd_blob.verified.wait()
|
||||
sd_blob.close()
|
||||
return sd_blob
|
||||
|
||||
@classmethod
|
||||
def _from_stream_descriptor_blob(cls, loop: asyncio.BaseEventLoop, blob_dir: str,
|
||||
blob: BlobFile) -> 'StreamDescriptor':
|
||||
assert os.path.isfile(blob.file_path)
|
||||
with open(blob.file_path, 'rb') as f:
|
||||
json_bytes = f.read()
|
||||
blob: AbstractBlob) -> 'StreamDescriptor':
|
||||
with blob.reader_context() as blob_reader:
|
||||
json_bytes = blob_reader.read()
|
||||
try:
|
||||
decoded = json.loads(json_bytes.decode())
|
||||
except json.JSONDecodeError:
|
||||
@@ -160,8 +172,10 @@ class StreamDescriptor:
|
||||
|
||||
@classmethod
|
||||
async def from_stream_descriptor_blob(cls, loop: asyncio.BaseEventLoop, blob_dir: str,
|
||||
blob: BlobFile) -> 'StreamDescriptor':
|
||||
return await loop.run_in_executor(None, lambda: cls._from_stream_descriptor_blob(loop, blob_dir, blob))
|
||||
blob: AbstractBlob) -> 'StreamDescriptor':
|
||||
if not blob.is_readable():
|
||||
raise InvalidStreamDescriptorError(f"unreadable/missing blob: {blob.blob_hash}")
|
||||
return await loop.run_in_executor(None, cls._from_stream_descriptor_blob, loop, blob_dir, blob)
|
||||
|
||||
@staticmethod
|
||||
def get_blob_hashsum(b: typing.Dict):
|
||||
@@ -194,11 +208,12 @@ class StreamDescriptor:
|
||||
return h.hexdigest()
|
||||
|
||||
@classmethod
|
||||
async def create_stream(cls, loop: asyncio.BaseEventLoop, blob_dir: str,
|
||||
file_path: str, key: typing.Optional[bytes] = None,
|
||||
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None,
|
||||
old_sort: bool = False) -> 'StreamDescriptor':
|
||||
|
||||
async def create_stream(
|
||||
cls, loop: asyncio.BaseEventLoop, blob_dir: str, file_path: str, key: typing.Optional[bytes] = None,
|
||||
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None,
|
||||
old_sort: bool = False,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'],
|
||||
asyncio.Task]] = None) -> 'StreamDescriptor':
|
||||
blobs: typing.List[BlobInfo] = []
|
||||
|
||||
iv_generator = iv_generator or random_iv_generator()
|
||||
@@ -207,7 +222,7 @@ class StreamDescriptor:
|
||||
for blob_bytes in file_reader(file_path):
|
||||
blob_num += 1
|
||||
blob_info = await BlobFile.create_from_unencrypted(
|
||||
loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num
|
||||
loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num, blob_completed_callback
|
||||
)
|
||||
blobs.append(blob_info)
|
||||
blobs.append(
|
||||
@@ -216,7 +231,7 @@ class StreamDescriptor:
|
||||
loop, blob_dir, os.path.basename(file_path), binascii.hexlify(key).decode(), os.path.basename(file_path),
|
||||
blobs
|
||||
)
|
||||
sd_blob = await descriptor.make_sd_blob(old_sort=old_sort)
|
||||
sd_blob = await descriptor.make_sd_blob(old_sort=old_sort, blob_completed_callback=blob_completed_callback)
|
||||
descriptor.sd_hash = sd_blob.blob_hash
|
||||
return descriptor
|
||||
|
||||
@@ -228,7 +243,7 @@ class StreamDescriptor:
|
||||
return self.lower_bound_decrypted_length() + (AES.block_size // 8)
|
||||
|
||||
@classmethod
|
||||
async def recover(cls, blob_dir: str, sd_blob: 'BlobFile', stream_hash: str, stream_name: str,
|
||||
async def recover(cls, blob_dir: str, sd_blob: 'AbstractBlob', stream_hash: str, stream_name: str,
|
||||
suggested_file_name: str, key: str,
|
||||
blobs: typing.List['BlobInfo']) -> typing.Optional['StreamDescriptor']:
|
||||
descriptor = cls(asyncio.get_event_loop(), blob_dir, stream_name, key, suggested_file_name,
|
||||
|
||||
+103
-79
@@ -1,108 +1,132 @@
|
||||
import os
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
import binascii
|
||||
from lbrynet.error import DownloadSDTimeout
|
||||
from lbrynet.utils import resolve_host
|
||||
from lbrynet.stream.assembler import StreamAssembler
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.blob_exchange.downloader import BlobDownloader
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
from lbrynet.blob.blob_file import AbstractBlob
|
||||
from lbrynet.blob.blob_info import BlobInfo
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def drain_into(a: list, b: list):
|
||||
while a:
|
||||
b.append(a.pop())
|
||||
|
||||
|
||||
class StreamDownloader(StreamAssembler):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager', sd_hash: str,
|
||||
output_dir: typing.Optional[str] = None, output_file_name: typing.Optional[str] = None):
|
||||
super().__init__(loop, blob_manager, sd_hash, output_file_name)
|
||||
class StreamDownloader:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobManager', sd_hash: str,
|
||||
descriptor: typing.Optional[StreamDescriptor] = None):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
self.output_dir = output_dir or self.config.download_dir
|
||||
self.output_file_name = output_file_name
|
||||
self.blob_downloader: typing.Optional[BlobDownloader] = None
|
||||
self.search_queue = asyncio.Queue(loop=loop)
|
||||
self.peer_queue = asyncio.Queue(loop=loop)
|
||||
self.accumulate_task: typing.Optional[asyncio.Task] = None
|
||||
self.descriptor: typing.Optional[StreamDescriptor]
|
||||
self.blob_manager = blob_manager
|
||||
self.sd_hash = sd_hash
|
||||
self.search_queue = asyncio.Queue(loop=loop) # blob hashes to feed into the iterative finder
|
||||
self.peer_queue = asyncio.Queue(loop=loop) # new peers to try
|
||||
self.blob_downloader = BlobDownloader(self.loop, self.config, self.blob_manager, self.peer_queue)
|
||||
self.descriptor: typing.Optional[StreamDescriptor] = descriptor
|
||||
self.node: typing.Optional['Node'] = None
|
||||
self.assemble_task: typing.Optional[asyncio.Task] = None
|
||||
self.accumulate_task: typing.Optional[asyncio.Task] = None
|
||||
self.fixed_peers_handle: typing.Optional[asyncio.Handle] = None
|
||||
self.fixed_peers_delay: typing.Optional[float] = None
|
||||
self.added_fixed_peers = False
|
||||
self.time_to_descriptor: typing.Optional[float] = None
|
||||
self.time_to_first_bytes: typing.Optional[float] = None
|
||||
|
||||
async def setup(self): # start the peer accumulator and initialize the downloader
|
||||
if self.blob_downloader:
|
||||
raise Exception("downloader is already set up")
|
||||
if self.node:
|
||||
_, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue)
|
||||
self.blob_downloader = BlobDownloader(self.loop, self.config, self.blob_manager, self.peer_queue)
|
||||
self.search_queue.put_nowait(self.sd_hash)
|
||||
async def add_fixed_peers(self):
|
||||
def _delayed_add_fixed_peers():
|
||||
self.added_fixed_peers = True
|
||||
self.peer_queue.put_nowait([
|
||||
KademliaPeer(self.loop, address=address, tcp_port=port + 1)
|
||||
for address, port in addresses
|
||||
])
|
||||
|
||||
async def after_got_descriptor(self):
|
||||
self.search_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
|
||||
log.info("added head blob to search")
|
||||
|
||||
async def after_finished(self):
|
||||
log.info("downloaded stream %s -> %s", self.sd_hash, self.output_path)
|
||||
await self.blob_manager.storage.change_file_status(self.descriptor.stream_hash, 'finished')
|
||||
self.blob_downloader.close()
|
||||
|
||||
def stop(self):
|
||||
if self.accumulate_task:
|
||||
self.accumulate_task.cancel()
|
||||
self.accumulate_task = None
|
||||
if self.assemble_task:
|
||||
self.assemble_task.cancel()
|
||||
self.assemble_task = None
|
||||
if self.fixed_peers_handle:
|
||||
self.fixed_peers_handle.cancel()
|
||||
self.fixed_peers_handle = None
|
||||
self.blob_downloader = None
|
||||
if self.stream_handle:
|
||||
if not self.stream_handle.closed:
|
||||
self.stream_handle.close()
|
||||
self.stream_handle = None
|
||||
if not self.stream_finished_event.is_set() and self.wrote_bytes_event.is_set() and \
|
||||
self.output_path and os.path.isfile(self.output_path):
|
||||
os.remove(self.output_path)
|
||||
|
||||
async def get_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
|
||||
return await self.blob_downloader.download_blob(blob_hash, length)
|
||||
|
||||
def add_fixed_peers(self):
|
||||
async def _add_fixed_peers():
|
||||
addresses = [
|
||||
(await resolve_host(url, port + 1, proto='tcp'), port)
|
||||
for url, port in self.config.reflector_servers
|
||||
]
|
||||
|
||||
def _delayed_add_fixed_peers():
|
||||
self.added_fixed_peers = True
|
||||
self.peer_queue.put_nowait([
|
||||
KademliaPeer(self.loop, address=address, tcp_port=port + 1)
|
||||
for address, port in addresses
|
||||
])
|
||||
|
||||
self.fixed_peers_handle = self.loop.call_later(self.fixed_peers_delay, _delayed_add_fixed_peers)
|
||||
if not self.config.reflector_servers:
|
||||
return
|
||||
addresses = [
|
||||
(await resolve_host(url, port + 1, proto='tcp'), port)
|
||||
for url, port in self.config.reflector_servers
|
||||
]
|
||||
if 'dht' in self.config.components_to_skip or not self.node or not \
|
||||
len(self.node.protocol.routing_table.get_peers()):
|
||||
self.fixed_peers_delay = 0.0
|
||||
else:
|
||||
self.fixed_peers_delay = self.config.fixed_peer_delay
|
||||
self.loop.create_task(_add_fixed_peers())
|
||||
|
||||
def download(self, node: typing.Optional['Node'] = None):
|
||||
self.node = node
|
||||
self.assemble_task = self.loop.create_task(self.assemble_decrypted_stream(self.config.download_dir))
|
||||
self.add_fixed_peers()
|
||||
self.fixed_peers_handle = self.loop.call_later(self.fixed_peers_delay, _delayed_add_fixed_peers)
|
||||
|
||||
async def load_descriptor(self, connection_id: int = 0):
|
||||
# download or get the sd blob
|
||||
sd_blob = self.blob_manager.get_blob(self.sd_hash)
|
||||
if not sd_blob.get_is_verified():
|
||||
try:
|
||||
now = self.loop.time()
|
||||
sd_blob = await asyncio.wait_for(
|
||||
self.blob_downloader.download_blob(self.sd_hash, connection_id),
|
||||
self.config.blob_download_timeout, loop=self.loop
|
||||
)
|
||||
log.info("downloaded sd blob %s", self.sd_hash)
|
||||
self.time_to_descriptor = self.loop.time() - now
|
||||
except asyncio.TimeoutError:
|
||||
raise DownloadSDTimeout(self.sd_hash)
|
||||
|
||||
# parse the descriptor
|
||||
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
|
||||
self.loop, self.blob_manager.blob_dir, sd_blob
|
||||
)
|
||||
log.info("loaded stream manifest %s", self.sd_hash)
|
||||
|
||||
async def start(self, node: typing.Optional['Node'] = None, connection_id: int = 0):
|
||||
# set up peer accumulation
|
||||
if node:
|
||||
self.node = node
|
||||
_, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue)
|
||||
await self.add_fixed_peers()
|
||||
# start searching for peers for the sd hash
|
||||
self.search_queue.put_nowait(self.sd_hash)
|
||||
log.info("searching for peers for stream %s", self.sd_hash)
|
||||
|
||||
if not self.descriptor:
|
||||
await self.load_descriptor(connection_id)
|
||||
|
||||
# add the head blob to the peer search
|
||||
self.search_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
|
||||
log.info("added head blob to peer search for stream %s", self.sd_hash)
|
||||
|
||||
if not await self.blob_manager.storage.stream_exists(self.sd_hash):
|
||||
await self.blob_manager.storage.store_stream(
|
||||
self.blob_manager.get_blob(self.sd_hash, length=self.descriptor.length), self.descriptor
|
||||
)
|
||||
|
||||
async def download_stream_blob(self, blob_info: 'BlobInfo', connection_id: int = 0) -> 'AbstractBlob':
|
||||
if not filter(lambda blob: blob.blob_hash == blob_info.blob_hash, self.descriptor.blobs[:-1]):
|
||||
raise ValueError(f"blob {blob_info.blob_hash} is not part of stream with sd hash {self.sd_hash}")
|
||||
blob = await self.blob_downloader.download_blob(blob_info.blob_hash, blob_info.length, connection_id)
|
||||
return blob
|
||||
|
||||
def decrypt_blob(self, blob_info: 'BlobInfo', blob: 'AbstractBlob') -> bytes:
|
||||
return blob.decrypt(
|
||||
binascii.unhexlify(self.descriptor.key.encode()), binascii.unhexlify(blob_info.iv.encode())
|
||||
)
|
||||
|
||||
async def read_blob(self, blob_info: 'BlobInfo', connection_id: int = 0) -> bytes:
|
||||
start = None
|
||||
if self.time_to_first_bytes is None:
|
||||
start = self.loop.time()
|
||||
blob = await self.download_stream_blob(blob_info, connection_id)
|
||||
decrypted = self.decrypt_blob(blob_info, blob)
|
||||
if start:
|
||||
self.time_to_first_bytes = self.loop.time() - start
|
||||
return decrypted
|
||||
|
||||
def stop(self):
|
||||
if self.accumulate_task:
|
||||
self.accumulate_task.cancel()
|
||||
self.accumulate_task = None
|
||||
if self.fixed_peers_handle:
|
||||
self.fixed_peers_handle.cancel()
|
||||
self.fixed_peers_handle = None
|
||||
self.blob_downloader.close()
|
||||
@@ -3,56 +3,124 @@ import asyncio
|
||||
import typing
|
||||
import logging
|
||||
import binascii
|
||||
from aiohttp.web import Request, StreamResponse
|
||||
from lbrynet.utils import generate_id
|
||||
from lbrynet.error import DownloadSDTimeout
|
||||
from lbrynet.schema.mime_types import guess_media_type
|
||||
from lbrynet.stream.downloader import StreamDownloader
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.stream.reflector.client import StreamReflectorClient
|
||||
from lbrynet.extras.daemon.storage import StoredStreamClaim
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.schema.claim import Claim
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
from lbrynet.blob.blob_info import BlobInfo
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.extras.daemon.analytics import AnalyticsManager
|
||||
from lbrynet.wallet.transaction import Transaction
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_next_available_file_name(download_directory: str, file_name: str) -> str:
|
||||
base_name, ext = os.path.splitext(os.path.basename(file_name))
|
||||
i = 0
|
||||
while os.path.isfile(os.path.join(download_directory, file_name)):
|
||||
i += 1
|
||||
file_name = "%s_%i%s" % (base_name, i, ext)
|
||||
|
||||
return file_name
|
||||
|
||||
|
||||
async def get_next_available_file_name(loop: asyncio.BaseEventLoop, download_directory: str, file_name: str) -> str:
|
||||
return await loop.run_in_executor(None, _get_next_available_file_name, download_directory, file_name)
|
||||
|
||||
|
||||
class ManagedStream:
|
||||
STATUS_RUNNING = "running"
|
||||
STATUS_STOPPED = "stopped"
|
||||
STATUS_FINISHED = "finished"
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', rowid: int,
|
||||
descriptor: 'StreamDescriptor', download_directory: str, file_name: typing.Optional[str],
|
||||
downloader: typing.Optional[StreamDownloader] = None,
|
||||
__slots__ = [
|
||||
'loop',
|
||||
'config',
|
||||
'blob_manager',
|
||||
'sd_hash',
|
||||
'download_directory',
|
||||
'_file_name',
|
||||
'_status',
|
||||
'stream_claim_info',
|
||||
'download_id',
|
||||
'rowid',
|
||||
'written_bytes',
|
||||
'content_fee',
|
||||
'downloader',
|
||||
'analytics_manager',
|
||||
'fully_reflected',
|
||||
'file_output_task',
|
||||
'delayed_stop_task',
|
||||
'streaming_responses',
|
||||
'streaming',
|
||||
'_running',
|
||||
'saving',
|
||||
'finished_writing',
|
||||
'started_writing',
|
||||
|
||||
]
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
sd_hash: str, download_directory: typing.Optional[str] = None, file_name: typing.Optional[str] = None,
|
||||
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional[StoredStreamClaim] = None,
|
||||
download_id: typing.Optional[str] = None):
|
||||
download_id: typing.Optional[str] = None, rowid: typing.Optional[int] = None,
|
||||
descriptor: typing.Optional[StreamDescriptor] = None,
|
||||
content_fee: typing.Optional['Transaction'] = None,
|
||||
analytics_manager: typing.Optional['AnalyticsManager'] = None):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
self.blob_manager = blob_manager
|
||||
self.rowid = rowid
|
||||
self.sd_hash = sd_hash
|
||||
self.download_directory = download_directory
|
||||
self._file_name = file_name
|
||||
self.descriptor = descriptor
|
||||
self.downloader = downloader
|
||||
self.stream_hash = descriptor.stream_hash
|
||||
self.stream_claim_info = claim
|
||||
self._status = status
|
||||
self.stream_claim_info = claim
|
||||
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
|
||||
self.rowid = rowid
|
||||
self.written_bytes = 0
|
||||
self.content_fee = content_fee
|
||||
self.downloader = StreamDownloader(self.loop, self.config, self.blob_manager, sd_hash, descriptor)
|
||||
self.analytics_manager = analytics_manager
|
||||
|
||||
self.fully_reflected = asyncio.Event(loop=self.loop)
|
||||
self.tx = None
|
||||
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
|
||||
self.file_output_task: typing.Optional[asyncio.Task] = None
|
||||
self.delayed_stop_task: typing.Optional[asyncio.Task] = None
|
||||
self.streaming_responses: typing.List[typing.Tuple[Request, StreamResponse]] = []
|
||||
self.streaming = asyncio.Event(loop=self.loop)
|
||||
self._running = asyncio.Event(loop=self.loop)
|
||||
self.saving = asyncio.Event(loop=self.loop)
|
||||
self.finished_writing = asyncio.Event(loop=self.loop)
|
||||
self.started_writing = asyncio.Event(loop=self.loop)
|
||||
|
||||
@property
|
||||
def descriptor(self) -> StreamDescriptor:
|
||||
return self.downloader.descriptor
|
||||
|
||||
@property
|
||||
def stream_hash(self) -> str:
|
||||
return self.descriptor.stream_hash
|
||||
|
||||
@property
|
||||
def file_name(self) -> typing.Optional[str]:
|
||||
return self.downloader.output_file_name if self.downloader else self._file_name
|
||||
return self._file_name or (self.descriptor.suggested_file_name if self.descriptor else None)
|
||||
|
||||
@property
|
||||
def status(self) -> str:
|
||||
return self._status
|
||||
|
||||
def update_status(self, status: str):
|
||||
async def update_status(self, status: str):
|
||||
assert status in [self.STATUS_RUNNING, self.STATUS_STOPPED, self.STATUS_FINISHED]
|
||||
self._status = status
|
||||
await self.blob_manager.storage.change_file_status(self.stream_hash, status)
|
||||
|
||||
@property
|
||||
def finished(self) -> bool:
|
||||
@@ -95,57 +163,67 @@ class ManagedStream:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim_name
|
||||
|
||||
@property
|
||||
def metadata(self) ->typing.Optional[typing.Dict]:
|
||||
def metadata(self) -> typing.Optional[typing.Dict]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict()
|
||||
|
||||
@property
|
||||
def metadata_protobuf(self) -> bytes:
|
||||
if self.stream_claim_info:
|
||||
return binascii.hexlify(self.stream_claim_info.claim.to_bytes())
|
||||
|
||||
@property
|
||||
def blobs_completed(self) -> int:
|
||||
return sum([1 if self.blob_manager.get_blob(b.blob_hash).get_is_verified() else 0
|
||||
return sum([1 if self.blob_manager.is_blob_verified(b.blob_hash) else 0
|
||||
for b in self.descriptor.blobs[:-1]])
|
||||
|
||||
@property
|
||||
def blobs_in_stream(self) -> int:
|
||||
return len(self.descriptor.blobs) - 1
|
||||
|
||||
@property
|
||||
def sd_hash(self):
|
||||
return self.descriptor.sd_hash
|
||||
|
||||
@property
|
||||
def blobs_remaining(self) -> int:
|
||||
return self.blobs_in_stream - self.blobs_completed
|
||||
|
||||
@property
|
||||
def full_path(self) -> typing.Optional[str]:
|
||||
return os.path.join(self.download_directory, os.path.basename(self.file_name)) if self.file_name else None
|
||||
return os.path.join(self.download_directory, os.path.basename(self.file_name)) \
|
||||
if self.file_name and self.download_directory else None
|
||||
|
||||
@property
|
||||
def output_file_exists(self):
|
||||
return os.path.isfile(self.full_path) if self.full_path else False
|
||||
|
||||
def as_dict(self) -> typing.Dict:
|
||||
full_path = self.full_path if self.output_file_exists else None
|
||||
mime_type = guess_media_type(os.path.basename(self.descriptor.suggested_file_name))
|
||||
@property
|
||||
def mime_type(self):
|
||||
return guess_media_type(os.path.basename(self.descriptor.suggested_file_name))[0]
|
||||
|
||||
if self.downloader and self.downloader.written_bytes:
|
||||
written_bytes = self.downloader.written_bytes
|
||||
elif full_path:
|
||||
written_bytes = os.stat(full_path).st_size
|
||||
def as_dict(self) -> typing.Dict:
|
||||
full_path = self.full_path
|
||||
file_name = self.file_name
|
||||
download_directory = self.download_directory
|
||||
if self.full_path and self.output_file_exists:
|
||||
if self.written_bytes:
|
||||
written_bytes = self.written_bytes
|
||||
else:
|
||||
written_bytes = os.stat(self.full_path).st_size
|
||||
else:
|
||||
full_path = None
|
||||
file_name = None
|
||||
download_directory = None
|
||||
written_bytes = None
|
||||
return {
|
||||
'completed': self.finished,
|
||||
'file_name': self.file_name,
|
||||
'download_directory': self.download_directory,
|
||||
'completed': (self.output_file_exists and self.status in ('stopped', 'finished')) or all(
|
||||
self.blob_manager.is_blob_verified(b.blob_hash) for b in self.descriptor.blobs[:-1]),
|
||||
'file_name': file_name,
|
||||
'download_directory': download_directory,
|
||||
'points_paid': 0.0,
|
||||
'tx': self.tx,
|
||||
'stopped': not self.running,
|
||||
'stream_hash': self.stream_hash,
|
||||
'stream_name': self.descriptor.stream_name,
|
||||
'suggested_file_name': self.descriptor.suggested_file_name,
|
||||
'sd_hash': self.descriptor.sd_hash,
|
||||
'download_path': full_path,
|
||||
'mime_type': mime_type,
|
||||
'mime_type': self.mime_type,
|
||||
'key': self.descriptor.key,
|
||||
'total_bytes_lower_bound': self.descriptor.lower_bound_decrypted_length(),
|
||||
'total_bytes': self.descriptor.upper_bound_decrypted_length(),
|
||||
@@ -159,38 +237,180 @@ class ManagedStream:
|
||||
'nout': self.nout,
|
||||
'outpoint': self.outpoint,
|
||||
'metadata': self.metadata,
|
||||
'protobuf': self.metadata_protobuf,
|
||||
'channel_claim_id': self.channel_claim_id,
|
||||
'channel_name': self.channel_name,
|
||||
'claim_name': self.claim_name
|
||||
'claim_name': self.claim_name,
|
||||
'content_fee': self.content_fee
|
||||
}
|
||||
|
||||
@classmethod
|
||||
async def create(cls, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager',
|
||||
async def create(cls, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
file_path: str, key: typing.Optional[bytes] = None,
|
||||
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedStream':
|
||||
descriptor = await StreamDescriptor.create_stream(
|
||||
loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator
|
||||
loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
|
||||
blob_completed_callback=blob_manager.blob_completed
|
||||
)
|
||||
sd_blob = blob_manager.get_blob(descriptor.sd_hash)
|
||||
await blob_manager.storage.store_stream(
|
||||
blob_manager.get_blob(descriptor.sd_hash), descriptor
|
||||
)
|
||||
await blob_manager.blob_completed(sd_blob)
|
||||
for blob in descriptor.blobs[:-1]:
|
||||
await blob_manager.blob_completed(blob_manager.get_blob(blob.blob_hash, blob.length))
|
||||
row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path),
|
||||
os.path.dirname(file_path), 0)
|
||||
return cls(loop, blob_manager, row_id, descriptor, os.path.dirname(file_path), os.path.basename(file_path),
|
||||
status=cls.STATUS_FINISHED)
|
||||
return cls(loop, config, blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
|
||||
os.path.basename(file_path), status=cls.STATUS_FINISHED, rowid=row_id, descriptor=descriptor)
|
||||
|
||||
def start_download(self, node: typing.Optional['Node']):
|
||||
self.downloader.download(node)
|
||||
self.update_status(self.STATUS_RUNNING)
|
||||
async def start(self, node: typing.Optional['Node'] = None, timeout: typing.Optional[float] = None,
|
||||
save_now: bool = False):
|
||||
timeout = timeout or self.config.download_timeout
|
||||
if self._running.is_set():
|
||||
return
|
||||
log.info("start downloader for stream (sd hash: %s)", self.sd_hash)
|
||||
self._running.set()
|
||||
try:
|
||||
await asyncio.wait_for(self.downloader.start(node), timeout, loop=self.loop)
|
||||
except asyncio.TimeoutError:
|
||||
self._running.clear()
|
||||
raise DownloadSDTimeout(self.sd_hash)
|
||||
|
||||
def stop_download(self):
|
||||
if self.downloader:
|
||||
self.downloader.stop()
|
||||
self.downloader = None
|
||||
if self.delayed_stop_task and not self.delayed_stop_task.done():
|
||||
self.delayed_stop_task.cancel()
|
||||
self.delayed_stop_task = self.loop.create_task(self._delayed_stop())
|
||||
if not await self.blob_manager.storage.file_exists(self.sd_hash):
|
||||
if save_now:
|
||||
file_name, download_dir = self._file_name, self.download_directory
|
||||
else:
|
||||
file_name, download_dir = None, None
|
||||
self.rowid = await self.blob_manager.storage.save_downloaded_file(
|
||||
self.stream_hash, file_name, download_dir, 0.0
|
||||
)
|
||||
if self.status != self.STATUS_RUNNING:
|
||||
await self.update_status(self.STATUS_RUNNING)
|
||||
|
||||
async def stop(self, finished: bool = False):
|
||||
"""
|
||||
Stop any running save/stream tasks as well as the downloader and update the status in the database
|
||||
"""
|
||||
|
||||
self.stop_tasks()
|
||||
if (finished and self.status != self.STATUS_FINISHED) or self.status == self.STATUS_RUNNING:
|
||||
await self.update_status(self.STATUS_FINISHED if finished else self.STATUS_STOPPED)
|
||||
|
||||
async def _aiter_read_stream(self, start_blob_num: typing.Optional[int] = 0, connection_id: int = 0)\
|
||||
-> typing.AsyncIterator[typing.Tuple['BlobInfo', bytes]]:
|
||||
if start_blob_num >= len(self.descriptor.blobs[:-1]):
|
||||
raise IndexError(start_blob_num)
|
||||
for i, blob_info in enumerate(self.descriptor.blobs[start_blob_num:-1]):
|
||||
assert i + start_blob_num == blob_info.blob_num
|
||||
decrypted = await self.downloader.read_blob(blob_info, connection_id)
|
||||
yield (blob_info, decrypted)
|
||||
|
||||
async def stream_file(self, request: Request, node: typing.Optional['Node'] = None) -> StreamResponse:
|
||||
log.info("stream file to browser for lbry://%s#%s (sd hash %s...)", self.claim_name, self.claim_id,
|
||||
self.sd_hash[:6])
|
||||
await self.start(node)
|
||||
headers, size, skip_blobs = self._prepare_range_response_headers(request.headers.get('range', 'bytes=0-'))
|
||||
response = StreamResponse(
|
||||
status=206,
|
||||
headers=headers
|
||||
)
|
||||
await response.prepare(request)
|
||||
self.streaming_responses.append((request, response))
|
||||
self.streaming.set()
|
||||
try:
|
||||
wrote = 0
|
||||
async for blob_info, decrypted in self._aiter_read_stream(skip_blobs, connection_id=2):
|
||||
if (blob_info.blob_num == len(self.descriptor.blobs) - 2) or (len(decrypted) + wrote >= size):
|
||||
decrypted += (b'\x00' * (size - len(decrypted) - wrote - (skip_blobs * 2097151)))
|
||||
await response.write_eof(decrypted)
|
||||
else:
|
||||
await response.write(decrypted)
|
||||
wrote += len(decrypted)
|
||||
log.info("sent browser %sblob %i/%i", "(final) " if response._eof_sent else "",
|
||||
blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
|
||||
if response._eof_sent:
|
||||
break
|
||||
return response
|
||||
finally:
|
||||
response.force_close()
|
||||
if (request, response) in self.streaming_responses:
|
||||
self.streaming_responses.remove((request, response))
|
||||
if not self.streaming_responses:
|
||||
self.streaming.clear()
|
||||
|
||||
@staticmethod
|
||||
def _write_decrypted_blob(handle: typing.IO, data: bytes):
|
||||
handle.write(data)
|
||||
handle.flush()
|
||||
|
||||
async def _save_file(self, output_path: str):
|
||||
log.info("save file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id, self.sd_hash[:6],
|
||||
output_path)
|
||||
self.saving.set()
|
||||
self.finished_writing.clear()
|
||||
self.started_writing.clear()
|
||||
try:
|
||||
with open(output_path, 'wb') as file_write_handle:
|
||||
async for blob_info, decrypted in self._aiter_read_stream(connection_id=1):
|
||||
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
|
||||
await self.loop.run_in_executor(None, self._write_decrypted_blob, file_write_handle, decrypted)
|
||||
self.written_bytes += len(decrypted)
|
||||
if not self.started_writing.is_set():
|
||||
self.started_writing.set()
|
||||
await self.update_status(ManagedStream.STATUS_FINISHED)
|
||||
if self.analytics_manager:
|
||||
self.loop.create_task(self.analytics_manager.send_download_finished(
|
||||
self.download_id, self.claim_name, self.sd_hash
|
||||
))
|
||||
self.finished_writing.set()
|
||||
log.info("finished saving file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id,
|
||||
self.sd_hash[:6], self.full_path)
|
||||
await self.blob_manager.storage.set_saved_file(self.stream_hash)
|
||||
except Exception as err:
|
||||
if os.path.isfile(output_path):
|
||||
log.warning("removing incomplete download %s for %s", output_path, self.sd_hash)
|
||||
os.remove(output_path)
|
||||
if not isinstance(err, asyncio.CancelledError):
|
||||
log.exception("unexpected error encountered writing file for stream %s", self.sd_hash)
|
||||
raise err
|
||||
finally:
|
||||
self.saving.clear()
|
||||
|
||||
async def save_file(self, file_name: typing.Optional[str] = None, download_directory: typing.Optional[str] = None,
|
||||
node: typing.Optional['Node'] = None):
|
||||
await self.start(node)
|
||||
if self.file_output_task and not self.file_output_task.done(): # cancel an already running save task
|
||||
self.file_output_task.cancel()
|
||||
self.download_directory = download_directory or self.download_directory or self.config.download_dir
|
||||
if not self.download_directory:
|
||||
raise ValueError("no directory to download to")
|
||||
if not (file_name or self._file_name or self.descriptor.suggested_file_name):
|
||||
raise ValueError("no file name to download to")
|
||||
if not os.path.isdir(self.download_directory):
|
||||
log.warning("download directory '%s' does not exist, attempting to make it", self.download_directory)
|
||||
os.mkdir(self.download_directory)
|
||||
self._file_name = await get_next_available_file_name(
|
||||
self.loop, self.download_directory,
|
||||
file_name or self.descriptor.suggested_file_name
|
||||
)
|
||||
await self.blob_manager.storage.change_file_download_dir_and_file_name(
|
||||
self.stream_hash, self.download_directory, self.file_name
|
||||
)
|
||||
await self.update_status(ManagedStream.STATUS_RUNNING)
|
||||
self.written_bytes = 0
|
||||
self.file_output_task = self.loop.create_task(self._save_file(self.full_path))
|
||||
await self.started_writing.wait()
|
||||
|
||||
def stop_tasks(self):
|
||||
if self.file_output_task and not self.file_output_task.done():
|
||||
self.file_output_task.cancel()
|
||||
self.file_output_task = None
|
||||
while self.streaming_responses:
|
||||
req, response = self.streaming_responses.pop()
|
||||
response.force_close()
|
||||
req.transport.close()
|
||||
self.downloader.stop()
|
||||
self._running.clear()
|
||||
|
||||
async def upload_to_reflector(self, host: str, port: int) -> typing.List[str]:
|
||||
sent = []
|
||||
@@ -206,7 +426,9 @@ class ManagedStream:
|
||||
self.fully_reflected.set()
|
||||
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
|
||||
return []
|
||||
we_have = [blob_hash for blob_hash in needed if blob_hash in self.blob_manager.completed_blob_hashes]
|
||||
we_have = [
|
||||
blob_hash for blob_hash in needed if blob_hash in self.blob_manager.completed_blob_hashes
|
||||
]
|
||||
for blob_hash in we_have:
|
||||
await protocol.send_blob(blob_hash)
|
||||
sent.append(blob_hash)
|
||||
@@ -229,3 +451,54 @@ class ManagedStream:
|
||||
binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'],
|
||||
claim_info['claim_sequence'], claim_info.get('channel_name')
|
||||
)
|
||||
|
||||
async def update_content_claim(self, claim_info: typing.Optional[typing.Dict] = None):
|
||||
if not claim_info:
|
||||
claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash)
|
||||
self.set_claim(claim_info, claim_info['value'])
|
||||
|
||||
async def _delayed_stop(self):
|
||||
stalled_count = 0
|
||||
while self._running.is_set():
|
||||
if self.saving.is_set() or self.streaming.is_set():
|
||||
stalled_count = 0
|
||||
else:
|
||||
stalled_count += 1
|
||||
if stalled_count > 1:
|
||||
log.info("stopping inactive download for lbry://%s#%s (%s...)", self.claim_name, self.claim_id,
|
||||
self.sd_hash[:6])
|
||||
await self.stop()
|
||||
return
|
||||
await asyncio.sleep(1, loop=self.loop)
|
||||
|
||||
def _prepare_range_response_headers(self, get_range: str) -> typing.Tuple[typing.Dict[str, str], int, int]:
|
||||
if '=' in get_range:
|
||||
get_range = get_range.split('=')[1]
|
||||
start, end = get_range.split('-')
|
||||
size = 0
|
||||
|
||||
for blob in self.descriptor.blobs[:-1]:
|
||||
size += blob.length - 1
|
||||
if self.stream_claim_info and self.stream_claim_info.claim.stream.source.size:
|
||||
size_from_claim = int(self.stream_claim_info.claim.stream.source.size)
|
||||
if not size_from_claim <= size <= size_from_claim + 16:
|
||||
raise ValueError("claim contains implausible stream size")
|
||||
log.debug("using stream size from claim")
|
||||
size = size_from_claim
|
||||
elif self.stream_claim_info:
|
||||
log.debug("estimating stream size")
|
||||
|
||||
start = int(start)
|
||||
end = int(end) if end else size - 1
|
||||
skip_blobs = start // 2097150
|
||||
skip = skip_blobs * 2097151
|
||||
start = skip
|
||||
final_size = end - start + 1
|
||||
|
||||
headers = {
|
||||
'Accept-Ranges': 'bytes',
|
||||
'Content-Range': f'bytes {start}-{end}/{size}',
|
||||
'Content-Length': str(final_size),
|
||||
'Content-Type': self.mime_type
|
||||
}
|
||||
return headers, size, skip_blobs
|
||||
@@ -4,7 +4,7 @@ import logging
|
||||
import typing
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
|
||||
REFLECTOR_V1 = 0
|
||||
@@ -14,7 +14,7 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StreamReflectorClient(asyncio.Protocol):
|
||||
def __init__(self, blob_manager: 'BlobFileManager', descriptor: 'StreamDescriptor'):
|
||||
def __init__(self, blob_manager: 'BlobManager', descriptor: 'StreamDescriptor'):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.transport: asyncio.StreamWriter = None
|
||||
self.blob_manager = blob_manager
|
||||
@@ -64,7 +64,7 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
|
||||
async def send_descriptor(self) -> typing.Tuple[bool, typing.List[str]]: # returns a list of needed blob hashes
|
||||
sd_blob = self.blob_manager.get_blob(self.descriptor.sd_hash)
|
||||
assert sd_blob.get_is_verified(), "need to have a sd blob to send at this point"
|
||||
assert self.blob_manager.is_blob_verified(self.descriptor.sd_hash), "need to have sd blob to send at this point"
|
||||
response = await self.send_request({
|
||||
'sd_blob_hash': sd_blob.blob_hash,
|
||||
'sd_blob_size': sd_blob.length
|
||||
@@ -80,7 +80,7 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
sent_sd = True
|
||||
if not needed:
|
||||
for blob in self.descriptor.blobs[:-1]:
|
||||
if self.blob_manager.get_blob(blob.blob_hash, blob.length).get_is_verified():
|
||||
if self.blob_manager.is_blob_verified(blob.blob_hash, blob.length):
|
||||
needed.append(blob.blob_hash)
|
||||
log.info("Sent reflector descriptor %s", sd_blob.blob_hash[:8])
|
||||
self.reflected_blobs.append(sd_blob.blob_hash)
|
||||
@@ -91,8 +91,8 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
return sent_sd, needed
|
||||
|
||||
async def send_blob(self, blob_hash: str):
|
||||
assert self.blob_manager.is_blob_verified(blob_hash), "need to have a blob to send at this point"
|
||||
blob = self.blob_manager.get_blob(blob_hash)
|
||||
assert blob.get_is_verified(), "need to have a blob to send at this point"
|
||||
response = await self.send_request({
|
||||
'blob_hash': blob.blob_hash,
|
||||
'blob_size': blob.length
|
||||
|
||||
@@ -7,7 +7,7 @@ from lbrynet.stream.descriptor import StreamDescriptor
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
from lbrynet.blob.writer import HashBlobWriter
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ReflectorServerProtocol(asyncio.Protocol):
|
||||
def __init__(self, blob_manager: 'BlobFileManager'):
|
||||
def __init__(self, blob_manager: 'BlobManager'):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
@@ -63,11 +63,11 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
return
|
||||
self.sd_blob = self.blob_manager.get_blob(request['sd_blob_hash'], request['sd_blob_size'])
|
||||
if not self.sd_blob.get_is_verified():
|
||||
self.writer = self.sd_blob.open_for_writing()
|
||||
self.writer = self.sd_blob.get_blob_writer(self.transport.get_extra_info('peername'))
|
||||
self.incoming.set()
|
||||
self.send_response({"send_sd_blob": True})
|
||||
try:
|
||||
await asyncio.wait_for(self.sd_blob.finished_writing.wait(), 30, loop=self.loop)
|
||||
await asyncio.wait_for(self.sd_blob.verified.wait(), 30, loop=self.loop)
|
||||
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
|
||||
self.loop, self.blob_manager.blob_dir, self.sd_blob
|
||||
)
|
||||
@@ -102,11 +102,11 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
return
|
||||
blob = self.blob_manager.get_blob(request['blob_hash'], request['blob_size'])
|
||||
if not blob.get_is_verified():
|
||||
self.writer = blob.open_for_writing()
|
||||
self.writer = blob.get_blob_writer(self.transport.get_extra_info('peername'))
|
||||
self.incoming.set()
|
||||
self.send_response({"send_blob": True})
|
||||
try:
|
||||
await asyncio.wait_for(blob.finished_writing.wait(), 30, loop=self.loop)
|
||||
await asyncio.wait_for(blob.verified.wait(), 30, loop=self.loop)
|
||||
self.send_response({"received_blob": True})
|
||||
except asyncio.TimeoutError:
|
||||
self.send_response({"received_blob": False})
|
||||
@@ -121,7 +121,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
|
||||
|
||||
class ReflectorServer:
|
||||
def __init__(self, blob_manager: 'BlobFileManager'):
|
||||
def __init__(self, blob_manager: 'BlobManager'):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
|
||||
+226
-303
@@ -5,22 +5,23 @@ import binascii
|
||||
import logging
|
||||
import random
|
||||
from decimal import Decimal
|
||||
from lbrynet.error import ResolveError, InvalidStreamDescriptorError, KeyFeeAboveMaxAllowed, InsufficientFundsError, \
|
||||
DownloadDataTimeout, DownloadSDTimeout
|
||||
from lbrynet.utils import generate_id
|
||||
from aiohttp.web import Request
|
||||
from lbrynet.error import ResolveError, InvalidStreamDescriptorError, KeyFeeAboveMaxAllowed, InsufficientFundsError
|
||||
from lbrynet.error import ResolveTimeout, DownloadDataTimeout
|
||||
from lbrynet.utils import cache_concurrent
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.stream.downloader import StreamDownloader
|
||||
from lbrynet.stream.managed_stream import ManagedStream
|
||||
from lbrynet.schema.claim import Claim
|
||||
from lbrynet.schema.uri import parse_lbry_uri
|
||||
from lbrynet.extras.daemon.storage import lbc_to_dewies
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.extras.daemon.analytics import AnalyticsManager
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage, StoredStreamClaim
|
||||
from lbrynet.wallet import LbryWalletManager
|
||||
from lbrynet.wallet.transaction import Transaction
|
||||
from lbrynet.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -54,8 +55,14 @@ comparison_operators = {
|
||||
}
|
||||
|
||||
|
||||
def path_or_none(p) -> typing.Optional[str]:
|
||||
if not p:
|
||||
return
|
||||
return binascii.unhexlify(p).decode()
|
||||
|
||||
|
||||
class StreamManager:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager',
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
wallet: 'LbryWalletManager', storage: 'SQLiteStorage', node: typing.Optional['Node'],
|
||||
analytics_manager: typing.Optional['AnalyticsManager'] = None):
|
||||
self.loop = loop
|
||||
@@ -65,79 +72,23 @@ class StreamManager:
|
||||
self.storage = storage
|
||||
self.node = node
|
||||
self.analytics_manager = analytics_manager
|
||||
self.streams: typing.Set[ManagedStream] = set()
|
||||
self.starting_streams: typing.Dict[str, asyncio.Future] = {}
|
||||
self.resume_downloading_task: asyncio.Task = None
|
||||
self.re_reflect_task: asyncio.Task = None
|
||||
self.streams: typing.Dict[str, ManagedStream] = {}
|
||||
self.resume_saving_task: typing.Optional[asyncio.Task] = None
|
||||
self.re_reflect_task: typing.Optional[asyncio.Task] = None
|
||||
self.update_stream_finished_futs: typing.List[asyncio.Future] = []
|
||||
self.running_reflector_uploads: typing.List[asyncio.Task] = []
|
||||
self.started = asyncio.Event(loop=self.loop)
|
||||
|
||||
async def _update_content_claim(self, stream: ManagedStream):
|
||||
claim_info = await self.storage.get_content_claim(stream.stream_hash)
|
||||
stream.set_claim(claim_info, claim_info['value'])
|
||||
|
||||
async def start_stream(self, stream: ManagedStream) -> bool:
|
||||
"""
|
||||
Resume or rebuild a partial or completed stream
|
||||
"""
|
||||
if not stream.running and not stream.output_file_exists:
|
||||
if stream.downloader:
|
||||
stream.downloader.stop()
|
||||
stream.downloader = None
|
||||
|
||||
# the directory is gone, can happen when the folder that contains a published file is deleted
|
||||
# reset the download directory to the default and update the file name
|
||||
if not os.path.isdir(stream.download_directory):
|
||||
stream.download_directory = self.config.download_dir
|
||||
|
||||
stream.downloader = self.make_downloader(
|
||||
stream.sd_hash, stream.download_directory, stream.descriptor.suggested_file_name
|
||||
)
|
||||
if stream.status != ManagedStream.STATUS_FINISHED:
|
||||
await self.storage.change_file_status(stream.stream_hash, 'running')
|
||||
stream.update_status('running')
|
||||
stream.start_download(self.node)
|
||||
try:
|
||||
await asyncio.wait_for(self.loop.create_task(stream.downloader.wrote_bytes_event.wait()),
|
||||
self.config.download_timeout)
|
||||
except asyncio.TimeoutError:
|
||||
await self.stop_stream(stream)
|
||||
if stream in self.streams:
|
||||
self.streams.remove(stream)
|
||||
return False
|
||||
file_name = os.path.basename(stream.downloader.output_path)
|
||||
output_dir = os.path.dirname(stream.downloader.output_path)
|
||||
await self.storage.change_file_download_dir_and_file_name(
|
||||
stream.stream_hash, output_dir, file_name
|
||||
)
|
||||
stream._file_name = file_name
|
||||
stream.download_directory = output_dir
|
||||
self.wait_for_stream_finished(stream)
|
||||
return True
|
||||
return True
|
||||
|
||||
async def stop_stream(self, stream: ManagedStream):
|
||||
stream.stop_download()
|
||||
if not stream.finished and stream.output_file_exists:
|
||||
try:
|
||||
os.remove(stream.full_path)
|
||||
except OSError as err:
|
||||
log.warning("Failed to delete partial download %s from downloads directory: %s", stream.full_path,
|
||||
str(err))
|
||||
if stream.running:
|
||||
stream.update_status(ManagedStream.STATUS_STOPPED)
|
||||
await self.storage.change_file_status(stream.stream_hash, ManagedStream.STATUS_STOPPED)
|
||||
|
||||
def make_downloader(self, sd_hash: str, download_directory: str, file_name: str):
|
||||
return StreamDownloader(
|
||||
self.loop, self.config, self.blob_manager, sd_hash, download_directory, file_name
|
||||
)
|
||||
self.streams.setdefault(stream.sd_hash, stream).set_claim(claim_info, claim_info['value'])
|
||||
|
||||
async def recover_streams(self, file_infos: typing.List[typing.Dict]):
|
||||
to_restore = []
|
||||
|
||||
async def recover_stream(sd_hash: str, stream_hash: str, stream_name: str,
|
||||
suggested_file_name: str, key: str) -> typing.Optional[StreamDescriptor]:
|
||||
suggested_file_name: str, key: str,
|
||||
content_fee: typing.Optional['Transaction']) -> typing.Optional[StreamDescriptor]:
|
||||
sd_blob = self.blob_manager.get_blob(sd_hash)
|
||||
blobs = await self.storage.get_blobs_for_stream(stream_hash)
|
||||
descriptor = await StreamDescriptor.recover(
|
||||
@@ -145,116 +96,126 @@ class StreamManager:
|
||||
)
|
||||
if not descriptor:
|
||||
return
|
||||
to_restore.append((descriptor, sd_blob))
|
||||
to_restore.append((descriptor, sd_blob, content_fee))
|
||||
|
||||
await asyncio.gather(*[
|
||||
recover_stream(
|
||||
file_info['sd_hash'], file_info['stream_hash'], binascii.unhexlify(file_info['stream_name']).decode(),
|
||||
binascii.unhexlify(file_info['suggested_file_name']).decode(), file_info['key']
|
||||
binascii.unhexlify(file_info['suggested_file_name']).decode(), file_info['key'],
|
||||
file_info['content_fee']
|
||||
) for file_info in file_infos
|
||||
])
|
||||
|
||||
if to_restore:
|
||||
await self.storage.recover_streams(to_restore, self.config.download_dir)
|
||||
log.info("Recovered %i/%i attempted streams", len(to_restore), len(file_infos))
|
||||
|
||||
async def add_stream(self, rowid: int, sd_hash: str, file_name: str, download_directory: str, status: str,
|
||||
claim: typing.Optional['StoredStreamClaim']):
|
||||
sd_blob = self.blob_manager.get_blob(sd_hash)
|
||||
if not sd_blob.get_is_verified():
|
||||
return
|
||||
# if self.blob_manager._save_blobs:
|
||||
# log.info("Recovered %i/%i attempted streams", len(to_restore), len(file_infos))
|
||||
|
||||
async def add_stream(self, rowid: int, sd_hash: str, file_name: typing.Optional[str],
|
||||
download_directory: typing.Optional[str], status: str,
|
||||
claim: typing.Optional['StoredStreamClaim'], content_fee: typing.Optional['Transaction']):
|
||||
try:
|
||||
descriptor = await self.blob_manager.get_stream_descriptor(sd_blob.blob_hash)
|
||||
descriptor = await self.blob_manager.get_stream_descriptor(sd_hash)
|
||||
except InvalidStreamDescriptorError as err:
|
||||
log.warning("Failed to start stream for sd %s - %s", sd_hash, str(err))
|
||||
return
|
||||
if status == ManagedStream.STATUS_RUNNING:
|
||||
downloader = self.make_downloader(descriptor.sd_hash, download_directory, file_name)
|
||||
else:
|
||||
downloader = None
|
||||
stream = ManagedStream(
|
||||
self.loop, self.blob_manager, rowid, descriptor, download_directory, file_name, downloader, status, claim
|
||||
self.loop, self.config, self.blob_manager, descriptor.sd_hash, download_directory, file_name, status,
|
||||
claim, content_fee=content_fee, rowid=rowid, descriptor=descriptor,
|
||||
analytics_manager=self.analytics_manager
|
||||
)
|
||||
self.streams.add(stream)
|
||||
self.streams[sd_hash] = stream
|
||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||
|
||||
async def load_streams_from_database(self):
|
||||
async def load_and_resume_streams_from_database(self):
|
||||
to_recover = []
|
||||
for file_info in await self.storage.get_all_lbry_files():
|
||||
if not self.blob_manager.get_blob(file_info['sd_hash']).get_is_verified():
|
||||
to_recover.append(file_info)
|
||||
to_start = []
|
||||
|
||||
await self.storage.update_manually_removed_files_since_last_run()
|
||||
|
||||
for file_info in await self.storage.get_all_lbry_files():
|
||||
# if the sd blob is not verified, try to reconstruct it from the database
|
||||
# this could either be because the blob files were deleted manually or save_blobs was not true when
|
||||
# the stream was downloaded
|
||||
if not self.blob_manager.is_blob_verified(file_info['sd_hash']):
|
||||
to_recover.append(file_info)
|
||||
to_start.append(file_info)
|
||||
if to_recover:
|
||||
log.info("Attempting to recover %i streams", len(to_recover))
|
||||
await self.recover_streams(to_recover)
|
||||
|
||||
to_start = []
|
||||
for file_info in await self.storage.get_all_lbry_files():
|
||||
if self.blob_manager.get_blob(file_info['sd_hash']).get_is_verified():
|
||||
to_start.append(file_info)
|
||||
log.info("Initializing %i files", len(to_start))
|
||||
|
||||
await asyncio.gather(*[
|
||||
self.add_stream(
|
||||
file_info['rowid'], file_info['sd_hash'], binascii.unhexlify(file_info['file_name']).decode(),
|
||||
binascii.unhexlify(file_info['download_directory']).decode(), file_info['status'],
|
||||
file_info['claim']
|
||||
) for file_info in to_start
|
||||
])
|
||||
to_resume_saving = []
|
||||
add_stream_tasks = []
|
||||
for file_info in to_start:
|
||||
file_name = path_or_none(file_info['file_name'])
|
||||
download_directory = path_or_none(file_info['download_directory'])
|
||||
if file_name and download_directory and not file_info['saved_file'] and file_info['status'] == 'running':
|
||||
to_resume_saving.append((file_name, download_directory, file_info['sd_hash']))
|
||||
add_stream_tasks.append(self.loop.create_task(self.add_stream(
|
||||
file_info['rowid'], file_info['sd_hash'], file_name,
|
||||
download_directory, file_info['status'],
|
||||
file_info['claim'], file_info['content_fee']
|
||||
)))
|
||||
if add_stream_tasks:
|
||||
await asyncio.gather(*add_stream_tasks, loop=self.loop)
|
||||
log.info("Started stream manager with %i files", len(self.streams))
|
||||
|
||||
async def resume(self):
|
||||
if self.node:
|
||||
await self.node.joined.wait()
|
||||
else:
|
||||
if not self.node:
|
||||
log.warning("no DHT node given, resuming downloads trusting that we can contact reflector")
|
||||
t = [
|
||||
(stream.start_download(self.node), self.wait_for_stream_finished(stream))
|
||||
for stream in self.streams if stream.status == ManagedStream.STATUS_RUNNING
|
||||
]
|
||||
if t:
|
||||
log.info("resuming %i downloads", len(t))
|
||||
if to_resume_saving:
|
||||
self.resume_saving_task = self.loop.create_task(self.resume(to_resume_saving))
|
||||
|
||||
async def resume(self, to_resume_saving):
|
||||
log.info("Resuming saving %i files", len(to_resume_saving))
|
||||
await asyncio.gather(
|
||||
*(self.streams[sd_hash].save_file(file_name, download_directory, node=self.node)
|
||||
for (file_name, download_directory, sd_hash) in to_resume_saving),
|
||||
loop=self.loop
|
||||
)
|
||||
|
||||
async def reflect_streams(self):
|
||||
while True:
|
||||
if self.config.reflect_streams and self.config.reflector_servers:
|
||||
sd_hashes = await self.storage.get_streams_to_re_reflect()
|
||||
streams = list(filter(lambda s: s.sd_hash in sd_hashes, self.streams))
|
||||
sd_hashes = [sd for sd in sd_hashes if sd in self.streams]
|
||||
batch = []
|
||||
while streams:
|
||||
stream = streams.pop()
|
||||
if not stream.fully_reflected.is_set():
|
||||
host, port = random.choice(self.config.reflector_servers)
|
||||
batch.append(stream.upload_to_reflector(host, port))
|
||||
while sd_hashes:
|
||||
stream = self.streams[sd_hashes.pop()]
|
||||
if self.blob_manager.is_blob_verified(stream.sd_hash) and stream.blobs_completed:
|
||||
if not stream.fully_reflected.is_set():
|
||||
host, port = random.choice(self.config.reflector_servers)
|
||||
batch.append(stream.upload_to_reflector(host, port))
|
||||
if len(batch) >= self.config.concurrent_reflector_uploads:
|
||||
await asyncio.gather(*batch)
|
||||
await asyncio.gather(*batch, loop=self.loop)
|
||||
batch = []
|
||||
if batch:
|
||||
await asyncio.gather(*batch)
|
||||
await asyncio.gather(*batch, loop=self.loop)
|
||||
await asyncio.sleep(300, loop=self.loop)
|
||||
|
||||
async def start(self):
|
||||
await self.load_streams_from_database()
|
||||
self.resume_downloading_task = self.loop.create_task(self.resume())
|
||||
await self.load_and_resume_streams_from_database()
|
||||
self.re_reflect_task = self.loop.create_task(self.reflect_streams())
|
||||
self.started.set()
|
||||
|
||||
def stop(self):
|
||||
if self.resume_downloading_task and not self.resume_downloading_task.done():
|
||||
self.resume_downloading_task.cancel()
|
||||
if self.resume_saving_task and not self.resume_saving_task.done():
|
||||
self.resume_saving_task.cancel()
|
||||
if self.re_reflect_task and not self.re_reflect_task.done():
|
||||
self.re_reflect_task.cancel()
|
||||
while self.streams:
|
||||
stream = self.streams.pop()
|
||||
stream.stop_download()
|
||||
_, stream = self.streams.popitem()
|
||||
stream.stop_tasks()
|
||||
while self.update_stream_finished_futs:
|
||||
self.update_stream_finished_futs.pop().cancel()
|
||||
while self.running_reflector_uploads:
|
||||
self.running_reflector_uploads.pop().cancel()
|
||||
self.started.clear()
|
||||
log.info("finished stopping the stream manager")
|
||||
|
||||
async def create_stream(self, file_path: str, key: typing.Optional[bytes] = None,
|
||||
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
|
||||
stream = await ManagedStream.create(self.loop, self.blob_manager, file_path, key, iv_generator)
|
||||
self.streams.add(stream)
|
||||
stream = await ManagedStream.create(self.loop, self.config, self.blob_manager, file_path, key, iv_generator)
|
||||
self.streams[stream.sd_hash] = stream
|
||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||
if self.config.reflect_streams and self.config.reflector_servers:
|
||||
host, port = random.choice(self.config.reflector_servers)
|
||||
@@ -267,9 +228,9 @@ class StreamManager:
|
||||
return stream
|
||||
|
||||
async def delete_stream(self, stream: ManagedStream, delete_file: typing.Optional[bool] = False):
|
||||
await self.stop_stream(stream)
|
||||
if stream in self.streams:
|
||||
self.streams.remove(stream)
|
||||
stream.stop_tasks()
|
||||
if stream.sd_hash in self.streams:
|
||||
del self.streams[stream.sd_hash]
|
||||
blob_hashes = [stream.sd_hash] + [b.blob_hash for b in stream.descriptor.blobs[:-1]]
|
||||
await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False)
|
||||
await self.storage.delete_stream(stream.descriptor)
|
||||
@@ -277,7 +238,7 @@ class StreamManager:
|
||||
os.remove(stream.full_path)
|
||||
|
||||
def get_stream_by_stream_hash(self, stream_hash: str) -> typing.Optional[ManagedStream]:
|
||||
streams = tuple(filter(lambda stream: stream.stream_hash == stream_hash, self.streams))
|
||||
streams = tuple(filter(lambda stream: stream.stream_hash == stream_hash, self.streams.values()))
|
||||
if streams:
|
||||
return streams[0]
|
||||
|
||||
@@ -304,66 +265,33 @@ class StreamManager:
|
||||
if search_by:
|
||||
comparison = comparison or 'eq'
|
||||
streams = []
|
||||
for stream in self.streams:
|
||||
for stream in self.streams.values():
|
||||
for search, val in search_by.items():
|
||||
if comparison_operators[comparison](getattr(stream, search), val):
|
||||
streams.append(stream)
|
||||
break
|
||||
else:
|
||||
streams = list(self.streams)
|
||||
streams = list(self.streams.values())
|
||||
if sort_by:
|
||||
streams.sort(key=lambda s: getattr(s, sort_by))
|
||||
if reverse:
|
||||
streams.reverse()
|
||||
return streams
|
||||
|
||||
def wait_for_stream_finished(self, stream: ManagedStream):
|
||||
async def _wait_for_stream_finished():
|
||||
if stream.downloader and stream.running:
|
||||
await stream.downloader.stream_finished_event.wait()
|
||||
stream.update_status(ManagedStream.STATUS_FINISHED)
|
||||
if self.analytics_manager:
|
||||
self.loop.create_task(self.analytics_manager.send_download_finished(
|
||||
stream.download_id, stream.claim_name, stream.sd_hash
|
||||
))
|
||||
|
||||
task = self.loop.create_task(_wait_for_stream_finished())
|
||||
self.update_stream_finished_futs.append(task)
|
||||
task.add_done_callback(
|
||||
lambda _: None if task not in self.update_stream_finished_futs else
|
||||
self.update_stream_finished_futs.remove(task)
|
||||
)
|
||||
|
||||
async def _store_stream(self, downloader: StreamDownloader) -> int:
|
||||
file_name = os.path.basename(downloader.output_path)
|
||||
download_directory = os.path.dirname(downloader.output_path)
|
||||
if not await self.storage.stream_exists(downloader.sd_hash):
|
||||
await self.storage.store_stream(downloader.sd_blob, downloader.descriptor)
|
||||
if not await self.storage.file_exists(downloader.sd_hash):
|
||||
return await self.storage.save_downloaded_file(
|
||||
downloader.descriptor.stream_hash, file_name, download_directory,
|
||||
0.0
|
||||
)
|
||||
else:
|
||||
return await self.storage.rowid_for_stream(downloader.descriptor.stream_hash)
|
||||
|
||||
async def _check_update_or_replace(self, outpoint: str, claim_id: str, claim: Claim) -> typing.Tuple[
|
||||
typing.Optional[ManagedStream], typing.Optional[ManagedStream]]:
|
||||
existing = self.get_filtered_streams(outpoint=outpoint)
|
||||
if existing:
|
||||
await self.start_stream(existing[0])
|
||||
return existing[0], None
|
||||
existing = self.get_filtered_streams(sd_hash=claim.stream.sd_hash)
|
||||
existing = self.get_filtered_streams(sd_hash=claim.stream.source.sd_hash)
|
||||
if existing and existing[0].claim_id != claim_id:
|
||||
raise ResolveError(f"stream for {existing[0].claim_id} collides with existing "
|
||||
f"download {claim_id}")
|
||||
raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {claim_id}")
|
||||
if existing:
|
||||
log.info("claim contains a metadata only update to a stream we have")
|
||||
await self.storage.save_content_claim(
|
||||
existing[0].stream_hash, outpoint
|
||||
)
|
||||
await self._update_content_claim(existing[0])
|
||||
await self.start_stream(existing[0])
|
||||
return existing[0], None
|
||||
else:
|
||||
existing_for_claim_id = self.get_filtered_streams(claim_id=claim_id)
|
||||
@@ -372,142 +300,137 @@ class StreamManager:
|
||||
return None, existing_for_claim_id[0]
|
||||
return None, None
|
||||
|
||||
async def start_downloader(self, got_descriptor_time: asyncio.Future, downloader: StreamDownloader,
|
||||
download_id: str, outpoint: str, claim: Claim, resolved: typing.Dict,
|
||||
file_name: typing.Optional[str] = None) -> ManagedStream:
|
||||
start_time = self.loop.time()
|
||||
downloader.download(self.node)
|
||||
await downloader.got_descriptor.wait()
|
||||
got_descriptor_time.set_result(self.loop.time() - start_time)
|
||||
rowid = await self._store_stream(downloader)
|
||||
await self.storage.save_content_claim(
|
||||
downloader.descriptor.stream_hash, outpoint
|
||||
)
|
||||
stream = ManagedStream(self.loop, self.blob_manager, rowid, downloader.descriptor, self.config.download_dir,
|
||||
file_name, downloader, ManagedStream.STATUS_RUNNING, download_id=download_id)
|
||||
stream.set_claim(resolved, claim)
|
||||
await stream.downloader.wrote_bytes_event.wait()
|
||||
self.streams.add(stream)
|
||||
return stream
|
||||
|
||||
async def _download_stream_from_uri(self, uri, timeout: float, exchange_rate_manager: 'ExchangeRateManager',
|
||||
file_name: typing.Optional[str] = None) -> ManagedStream:
|
||||
start_time = self.loop.time()
|
||||
parsed_uri = parse_lbry_uri(uri)
|
||||
if parsed_uri.is_channel:
|
||||
raise ResolveError("cannot download a channel claim, specify a /path")
|
||||
|
||||
# resolve the claim
|
||||
resolved = (await self.wallet.ledger.resolve(0, 10, uri)).get(uri, {})
|
||||
resolved = resolved if 'value' in resolved else resolved.get('claim')
|
||||
|
||||
if not resolved:
|
||||
raise ResolveError(f"Failed to resolve stream at '{uri}'")
|
||||
if 'error' in resolved:
|
||||
raise ResolveError(f"error resolving stream: {resolved['error']}")
|
||||
|
||||
claim = Claim.from_bytes(binascii.unhexlify(resolved['hex']))
|
||||
outpoint = f"{resolved['txid']}:{resolved['nout']}"
|
||||
resolved_time = self.loop.time() - start_time
|
||||
|
||||
# resume or update an existing stream, if the stream changed download it and delete the old one after
|
||||
updated_stream, to_replace = await self._check_update_or_replace(outpoint, resolved['claim_id'], claim)
|
||||
if updated_stream:
|
||||
return updated_stream
|
||||
|
||||
# check that the fee is payable
|
||||
fee_amount, fee_address = None, None
|
||||
if claim.stream.has_fee:
|
||||
fee_amount = round(exchange_rate_manager.convert_currency(
|
||||
claim.stream.fee.currency, "LBC", claim.stream.fee.amount
|
||||
), 5)
|
||||
max_fee_amount = round(exchange_rate_manager.convert_currency(
|
||||
self.config.max_key_fee['currency'], "LBC", Decimal(self.config.max_key_fee['amount'])
|
||||
), 5)
|
||||
if fee_amount > max_fee_amount:
|
||||
msg = f"fee of {fee_amount} exceeds max configured to allow of {max_fee_amount}"
|
||||
log.warning(msg)
|
||||
raise KeyFeeAboveMaxAllowed(msg)
|
||||
balance = await self.wallet.default_account.get_balance()
|
||||
if lbc_to_dewies(str(fee_amount)) > balance:
|
||||
msg = f"fee of {fee_amount} exceeds max available balance"
|
||||
log.warning(msg)
|
||||
raise InsufficientFundsError(msg)
|
||||
fee_address = claim.stream.fee.address
|
||||
|
||||
# download the stream
|
||||
download_id = binascii.hexlify(generate_id()).decode()
|
||||
downloader = StreamDownloader(self.loop, self.config, self.blob_manager, claim.stream.sd_hash,
|
||||
self.config.download_dir, file_name)
|
||||
|
||||
stream = None
|
||||
descriptor_time_fut = self.loop.create_future()
|
||||
start_download_time = self.loop.time()
|
||||
time_to_descriptor = None
|
||||
time_to_first_bytes = None
|
||||
error = None
|
||||
try:
|
||||
stream = await asyncio.wait_for(
|
||||
asyncio.ensure_future(
|
||||
self.start_downloader(descriptor_time_fut, downloader, download_id, outpoint, claim, resolved,
|
||||
file_name)
|
||||
), timeout
|
||||
)
|
||||
time_to_descriptor = await descriptor_time_fut
|
||||
time_to_first_bytes = self.loop.time() - start_download_time - time_to_descriptor
|
||||
self.wait_for_stream_finished(stream)
|
||||
if fee_address and fee_amount and not to_replace:
|
||||
stream.tx = await self.wallet.send_amount_to_address(
|
||||
lbc_to_dewies(str(fee_amount)), fee_address.encode('latin1'))
|
||||
elif to_replace: # delete old stream now that the replacement has started downloading
|
||||
await self.delete_stream(to_replace)
|
||||
except asyncio.TimeoutError:
|
||||
if descriptor_time_fut.done():
|
||||
time_to_descriptor = descriptor_time_fut.result()
|
||||
error = DownloadDataTimeout(downloader.sd_hash)
|
||||
self.blob_manager.delete_blob(downloader.sd_hash)
|
||||
await self.storage.delete_stream(downloader.descriptor)
|
||||
else:
|
||||
descriptor_time_fut.cancel()
|
||||
error = DownloadSDTimeout(downloader.sd_hash)
|
||||
if stream:
|
||||
await self.stop_stream(stream)
|
||||
else:
|
||||
downloader.stop()
|
||||
if error:
|
||||
log.warning(error)
|
||||
if self.analytics_manager:
|
||||
self.loop.create_task(
|
||||
self.analytics_manager.send_time_to_first_bytes(
|
||||
resolved_time, self.loop.time() - start_time, download_id, parse_lbry_uri(uri).name, outpoint,
|
||||
None if not stream else len(stream.downloader.blob_downloader.active_connections),
|
||||
None if not stream else len(stream.downloader.blob_downloader.scores),
|
||||
False if not downloader else downloader.added_fixed_peers,
|
||||
self.config.fixed_peer_delay if not downloader else downloader.fixed_peers_delay,
|
||||
claim.stream.sd_hash, time_to_descriptor,
|
||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
|
||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
|
||||
time_to_first_bytes, None if not error else error.__class__.__name__
|
||||
)
|
||||
)
|
||||
if error:
|
||||
raise error
|
||||
return stream
|
||||
|
||||
@cache_concurrent
|
||||
async def download_stream_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
|
||||
timeout: typing.Optional[float] = None,
|
||||
file_name: typing.Optional[str] = None,
|
||||
timeout: typing.Optional[float] = None) -> ManagedStream:
|
||||
download_directory: typing.Optional[str] = None,
|
||||
save_file: typing.Optional[bool] = None,
|
||||
resolve_timeout: float = 3.0) -> ManagedStream:
|
||||
timeout = timeout or self.config.download_timeout
|
||||
if uri in self.starting_streams:
|
||||
return await self.starting_streams[uri]
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
self.starting_streams[uri] = fut
|
||||
start_time = self.loop.time()
|
||||
resolved_time = None
|
||||
stream = None
|
||||
error = None
|
||||
outpoint = None
|
||||
if save_file is None:
|
||||
save_file = self.config.save_files
|
||||
if file_name and not save_file:
|
||||
save_file = True
|
||||
if save_file:
|
||||
download_directory = download_directory or self.config.download_dir
|
||||
else:
|
||||
download_directory = None
|
||||
|
||||
try:
|
||||
stream = await self._download_stream_from_uri(uri, timeout, exchange_rate_manager, file_name)
|
||||
fut.set_result(stream)
|
||||
except Exception as err:
|
||||
fut.set_exception(err)
|
||||
try:
|
||||
return await fut
|
||||
# resolve the claim
|
||||
parsed_uri = parse_lbry_uri(uri)
|
||||
if parsed_uri.is_channel:
|
||||
raise ResolveError("cannot download a channel claim, specify a /path")
|
||||
try:
|
||||
resolved_result = await asyncio.wait_for(self.wallet.ledger.resolve(0, 1, uri), resolve_timeout)
|
||||
except asyncio.TimeoutError:
|
||||
raise ResolveTimeout(uri)
|
||||
await self.storage.save_claims_for_resolve([
|
||||
value for value in resolved_result.values() if 'error' not in value
|
||||
])
|
||||
resolved = resolved_result.get(uri, {})
|
||||
resolved = resolved if 'value' in resolved else resolved.get('claim')
|
||||
if not resolved:
|
||||
raise ResolveError(f"Failed to resolve stream at '{uri}'")
|
||||
if 'error' in resolved:
|
||||
raise ResolveError(f"error resolving stream: {resolved['error']}")
|
||||
|
||||
claim = Claim.from_bytes(binascii.unhexlify(resolved['protobuf']))
|
||||
outpoint = f"{resolved['txid']}:{resolved['nout']}"
|
||||
resolved_time = self.loop.time() - start_time
|
||||
|
||||
# resume or update an existing stream, if the stream changed download it and delete the old one after
|
||||
updated_stream, to_replace = await self._check_update_or_replace(outpoint, resolved['claim_id'], claim)
|
||||
if updated_stream:
|
||||
log.info("already have stream for %s", uri)
|
||||
if save_file and updated_stream.output_file_exists:
|
||||
save_file = False
|
||||
await updated_stream.start(node=self.node, timeout=timeout, save_now=save_file)
|
||||
if not updated_stream.output_file_exists and (save_file or file_name or download_directory):
|
||||
await updated_stream.save_file(
|
||||
file_name=file_name, download_directory=download_directory, node=self.node
|
||||
)
|
||||
return updated_stream
|
||||
|
||||
content_fee = None
|
||||
fee_amount, fee_address = None, None
|
||||
|
||||
# check that the fee is payable
|
||||
if not to_replace and claim.stream.has_fee:
|
||||
fee_amount = round(exchange_rate_manager.convert_currency(
|
||||
claim.stream.fee.currency, "LBC", claim.stream.fee.amount
|
||||
), 5)
|
||||
max_fee_amount = round(exchange_rate_manager.convert_currency(
|
||||
self.config.max_key_fee['currency'], "LBC", Decimal(self.config.max_key_fee['amount'])
|
||||
), 5)
|
||||
if fee_amount > max_fee_amount:
|
||||
msg = f"fee of {fee_amount} exceeds max configured to allow of {max_fee_amount}"
|
||||
log.warning(msg)
|
||||
raise KeyFeeAboveMaxAllowed(msg)
|
||||
balance = await self.wallet.default_account.get_balance()
|
||||
if lbc_to_dewies(str(fee_amount)) > balance:
|
||||
msg = f"fee of {fee_amount} exceeds max available balance"
|
||||
log.warning(msg)
|
||||
raise InsufficientFundsError(msg)
|
||||
fee_address = claim.stream.fee.address
|
||||
|
||||
stream = ManagedStream(
|
||||
self.loop, self.config, self.blob_manager, claim.stream.source.sd_hash, download_directory,
|
||||
file_name, ManagedStream.STATUS_RUNNING, content_fee=content_fee,
|
||||
analytics_manager=self.analytics_manager
|
||||
)
|
||||
log.info("starting download for %s", uri)
|
||||
|
||||
before_download = self.loop.time()
|
||||
await stream.start(self.node, timeout)
|
||||
stream.set_claim(resolved, claim)
|
||||
if to_replace: # delete old stream now that the replacement has started downloading
|
||||
await self.delete_stream(to_replace)
|
||||
elif fee_address:
|
||||
stream.content_fee = await self.wallet.send_amount_to_address(
|
||||
lbc_to_dewies(str(fee_amount)), fee_address.encode('latin1')
|
||||
)
|
||||
log.info("paid fee of %s for %s", fee_amount, uri)
|
||||
await self.storage.save_content_fee(stream.stream_hash, stream.content_fee)
|
||||
|
||||
self.streams[stream.sd_hash] = stream
|
||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||
await self.storage.save_content_claim(stream.stream_hash, outpoint)
|
||||
if save_file:
|
||||
await asyncio.wait_for(stream.save_file(node=self.node), timeout - (self.loop.time() - before_download),
|
||||
loop=self.loop)
|
||||
return stream
|
||||
except asyncio.TimeoutError:
|
||||
error = DownloadDataTimeout(stream.sd_hash)
|
||||
raise error
|
||||
except Exception as err: # forgive data timeout, dont delete stream
|
||||
error = err
|
||||
raise
|
||||
finally:
|
||||
del self.starting_streams[uri]
|
||||
if self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or
|
||||
stream.downloader.time_to_first_bytes))):
|
||||
self.loop.create_task(
|
||||
self.analytics_manager.send_time_to_first_bytes(
|
||||
resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id,
|
||||
uri, outpoint,
|
||||
None if not stream else len(stream.downloader.blob_downloader.active_connections),
|
||||
None if not stream else len(stream.downloader.blob_downloader.scores),
|
||||
False if not stream else stream.downloader.added_fixed_peers,
|
||||
self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay,
|
||||
None if not stream else stream.sd_hash,
|
||||
None if not stream else stream.downloader.time_to_descriptor,
|
||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
|
||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
|
||||
None if not stream else stream.downloader.time_to_first_bytes,
|
||||
None if not error else error.__class__.__name__
|
||||
)
|
||||
)
|
||||
|
||||
async def stream_partial_content(self, request: Request, sd_hash: str):
|
||||
return await self.streams[sd_hash].stream_file(request, self.node)
|
||||
@@ -7,7 +7,6 @@ from binascii import unhexlify
|
||||
from torba.testcase import IntegrationTestCase
|
||||
|
||||
import lbrynet.wallet
|
||||
from lbrynet.schema.claim import Claim
|
||||
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.Daemon import Daemon, jsonrpc_dumps_pretty
|
||||
@@ -19,7 +18,7 @@ from lbrynet.extras.daemon.Components import (
|
||||
)
|
||||
from lbrynet.extras.daemon.ComponentManager import ComponentManager
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
from lbrynet.stream.reflector.server import ReflectorServer
|
||||
from lbrynet.blob_exchange.server import BlobServer
|
||||
|
||||
@@ -108,9 +107,11 @@ class CommandTestCase(IntegrationTestCase):
|
||||
|
||||
server_tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, server_tmp_dir)
|
||||
self.server_storage = SQLiteStorage(Config(), ':memory:')
|
||||
self.server_config = Config()
|
||||
self.server_storage = SQLiteStorage(self.server_config, ':memory:')
|
||||
await self.server_storage.open()
|
||||
self.server_blob_manager = BlobFileManager(self.loop, server_tmp_dir, self.server_storage)
|
||||
|
||||
self.server_blob_manager = BlobManager(self.loop, server_tmp_dir, self.server_storage, self.server_config)
|
||||
self.server = BlobServer(self.loop, self.server_blob_manager, 'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP')
|
||||
self.server.start_server(5567, '127.0.0.1')
|
||||
await self.server.started_listening.wait()
|
||||
@@ -123,7 +124,7 @@ class CommandTestCase(IntegrationTestCase):
|
||||
async def asyncTearDown(self):
|
||||
await super().asyncTearDown()
|
||||
self.wallet_component._running = False
|
||||
await self.daemon.stop()
|
||||
await self.daemon.stop(shutdown_runner=False)
|
||||
|
||||
async def confirm_tx(self, txid):
|
||||
""" Wait for tx to be in mempool, then generate a block, wait for tx to be in a block. """
|
||||
@@ -150,33 +151,64 @@ class CommandTestCase(IntegrationTestCase):
|
||||
await self.blockchain.generate(blocks)
|
||||
await self.ledger.on_header.where(self.blockchain.is_expected_block)
|
||||
|
||||
async def blockchain_claim_name(self, name: str, value: str, amount: str, confirm=True):
|
||||
txid = await self.blockchain._cli_cmnd('claimname', name, value, amount)
|
||||
if confirm:
|
||||
await self.generate(1)
|
||||
return txid
|
||||
|
||||
async def blockchain_update_name(self, txid: str, value: str, amount: str, confirm=True):
|
||||
txid = await self.blockchain._cli_cmnd('updateclaim', txid, value, amount)
|
||||
if confirm:
|
||||
await self.generate(1)
|
||||
return txid
|
||||
|
||||
async def out(self, awaitable):
|
||||
""" Converts Daemon API call results (dictionary)
|
||||
to JSON and then back to a dictionary. """
|
||||
""" Serializes lbrynet API results to JSON then loads and returns it as dictionary. """
|
||||
return json.loads(jsonrpc_dumps_pretty(await awaitable, ledger=self.ledger))['result']
|
||||
|
||||
def sout(self, value):
|
||||
""" Synchronous version of `out` method. """
|
||||
return json.loads(jsonrpc_dumps_pretty(value, ledger=self.ledger))['result']
|
||||
|
||||
async def stream_create(self, name='hovercraft', bid='1.0', data=b'hi!', confirm=True, **kwargs):
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(data)
|
||||
file.flush()
|
||||
claim = await self.out(
|
||||
self.daemon.jsonrpc_stream_create(name, bid, file_path=file.name, **kwargs)
|
||||
)
|
||||
self.assertEqual(claim['outputs'][0]['name'], name)
|
||||
if confirm:
|
||||
await self.on_transaction_dict(claim)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(claim)
|
||||
return claim
|
||||
file = tempfile.NamedTemporaryFile()
|
||||
|
||||
def cleanup():
|
||||
try:
|
||||
file.close()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
self.addCleanup(cleanup)
|
||||
file.write(data)
|
||||
file.flush()
|
||||
claim = await self.out(
|
||||
self.daemon.jsonrpc_stream_create(name, bid, file_path=file.name, **kwargs)
|
||||
)
|
||||
self.assertEqual(claim['outputs'][0]['name'], name)
|
||||
if confirm:
|
||||
await self.on_transaction_dict(claim)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(claim)
|
||||
return claim
|
||||
|
||||
async def stream_update(self, claim_id, data=None, confirm=True, **kwargs):
|
||||
if data:
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(data)
|
||||
file.flush()
|
||||
claim = await self.out(
|
||||
self.daemon.jsonrpc_stream_update(claim_id, file_path=file.name, **kwargs)
|
||||
)
|
||||
file = tempfile.NamedTemporaryFile()
|
||||
file.write(data)
|
||||
file.flush()
|
||||
|
||||
def cleanup():
|
||||
try:
|
||||
file.close()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
self.addCleanup(cleanup)
|
||||
claim = await self.out(
|
||||
self.daemon.jsonrpc_stream_update(claim_id, file_path=file.name, **kwargs)
|
||||
)
|
||||
else:
|
||||
claim = await self.out(self.daemon.jsonrpc_stream_update(claim_id, **kwargs))
|
||||
self.assertIsNotNone(claim['outputs'][0]['name'])
|
||||
@@ -186,6 +218,16 @@ class CommandTestCase(IntegrationTestCase):
|
||||
await self.on_transaction_dict(claim)
|
||||
return claim
|
||||
|
||||
async def stream_abandon(self, *args, confirm=True, **kwargs):
|
||||
if 'blocking' not in kwargs:
|
||||
kwargs['blocking'] = False
|
||||
tx = await self.out(self.daemon.jsonrpc_stream_abandon(*args, **kwargs))
|
||||
if confirm:
|
||||
await self.on_transaction_dict(tx)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(tx)
|
||||
return tx
|
||||
|
||||
async def publish(self, name, *args, confirm=True, **kwargs):
|
||||
claim = await self.out(self.daemon.jsonrpc_publish(name, *args, **kwargs))
|
||||
self.assertEqual(claim['outputs'][0]['name'], name)
|
||||
@@ -213,10 +255,10 @@ class CommandTestCase(IntegrationTestCase):
|
||||
await self.on_transaction_dict(channel)
|
||||
return channel
|
||||
|
||||
async def claim_abandon(self, *args, confirm=True, **kwargs):
|
||||
async def channel_abandon(self, *args, confirm=True, **kwargs):
|
||||
if 'blocking' not in kwargs:
|
||||
kwargs['blocking'] = False
|
||||
tx = await self.out(self.daemon.jsonrpc_claim_abandon(*args, **kwargs))
|
||||
tx = await self.out(self.daemon.jsonrpc_channel_abandon(*args, **kwargs))
|
||||
if confirm:
|
||||
await self.on_transaction_dict(tx)
|
||||
await self.generate(1)
|
||||
+40
-1
@@ -14,6 +14,7 @@ import pkg_resources
|
||||
import contextlib
|
||||
import certifi
|
||||
import aiohttp
|
||||
import functools
|
||||
from lbrynet.schema.claim import Claim
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
@@ -116,7 +117,7 @@ def get_sd_hash(stream_info):
|
||||
if not stream_info:
|
||||
return None
|
||||
if isinstance(stream_info, Claim):
|
||||
return stream_info.stream.sd_hash
|
||||
return stream_info.stream.source.sd_hash
|
||||
result = stream_info.get('claim', {}).\
|
||||
get('value', {}).\
|
||||
get('stream', {}).\
|
||||
@@ -146,6 +147,44 @@ def drain_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
|
||||
cancel_task(tasks.pop())
|
||||
|
||||
|
||||
def async_timed_cache(duration: int):
|
||||
def wrapper(fn):
|
||||
cache: typing.Dict[typing.Tuple,
|
||||
typing.Tuple[typing.Any, float]] = {}
|
||||
|
||||
@functools.wraps(fn)
|
||||
async def _inner(*args, **kwargs) -> typing.Any:
|
||||
loop = asyncio.get_running_loop()
|
||||
now = loop.time()
|
||||
key = tuple([args, tuple([tuple([k, kwargs[k]]) for k in kwargs])])
|
||||
if key in cache and (now - cache[key][1] < duration):
|
||||
return cache[key][0]
|
||||
to_cache = await fn(*args, **kwargs)
|
||||
cache[key] = to_cache, now
|
||||
return to_cache
|
||||
return _inner
|
||||
return wrapper
|
||||
|
||||
|
||||
def cache_concurrent(async_fn):
|
||||
"""
|
||||
When the decorated function has concurrent calls made to it with the same arguments, only run it once
|
||||
"""
|
||||
cache: typing.Dict = {}
|
||||
|
||||
@functools.wraps(async_fn)
|
||||
async def wrapper(*args, **kwargs):
|
||||
key = tuple([args, tuple([tuple([k, kwargs[k]]) for k in kwargs])])
|
||||
cache[key] = cache.get(key) or asyncio.create_task(async_fn(*args, **kwargs))
|
||||
try:
|
||||
return await cache[key]
|
||||
finally:
|
||||
cache.pop(key, None)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
@async_timed_cache(300)
|
||||
async def resolve_host(url: str, port: int, proto: str) -> str:
|
||||
if proto not in ['udp', 'tcp']:
|
||||
raise Exception("invalid protocol")
|
||||
|
||||
@@ -4,7 +4,7 @@ import binascii
|
||||
from hashlib import sha256
|
||||
from string import hexdigits
|
||||
|
||||
from torba.client.baseaccount import BaseAccount
|
||||
from torba.client.baseaccount import BaseAccount, HierarchicalDeterministic
|
||||
from torba.client.basetransaction import TXORef
|
||||
|
||||
|
||||
@@ -167,10 +167,11 @@ class Account(BaseAccount):
|
||||
))
|
||||
|
||||
async def save_max_gap(self):
|
||||
gap = await self.get_max_gap()
|
||||
self.receiving.gap = max(20, gap['max_receiving_gap'] + 1)
|
||||
self.change.gap = max(6, gap['max_change_gap'] + 1)
|
||||
self.wallet.save()
|
||||
if issubclass(self.address_generator, HierarchicalDeterministic):
|
||||
gap = await self.get_max_gap()
|
||||
self.receiving.gap = max(20, gap['max_receiving_gap'] + 1)
|
||||
self.change.gap = max(6, gap['max_change_gap'] + 1)
|
||||
self.wallet.save()
|
||||
|
||||
def get_balance(self, confirmations=0, include_claims=False, **constraints):
|
||||
if not include_claims:
|
||||
|
||||
@@ -3,10 +3,9 @@ import logging
|
||||
from binascii import unhexlify
|
||||
|
||||
from torba.client.baseledger import BaseLedger
|
||||
from lbrynet.schema.uri import parse_lbry_uri, URIParseError
|
||||
from lbrynet.wallet.dewies import dewies_to_lbc
|
||||
from lbrynet.wallet.resolve import Resolver
|
||||
from lbrynet.wallet.account import Account, validate_claim_id
|
||||
from lbrynet.wallet.account import Account
|
||||
from lbrynet.wallet.network import Network
|
||||
from lbrynet.wallet.database import WalletDatabase
|
||||
from lbrynet.wallet.transaction import Transaction
|
||||
@@ -48,39 +47,20 @@ class MainNetLedger(BaseLedger):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.fee_per_name_char = self.config.get('fee_per_name_char', self.default_fee_per_name_char)
|
||||
self.resolver = Resolver(self)
|
||||
|
||||
@property
|
||||
def resolver(self):
|
||||
return Resolver(self.headers.claim_trie_root, self.headers.height, self.transaction_class,
|
||||
hash160_to_address=self.hash160_to_address, network=self.network, ledger=self)
|
||||
|
||||
async def resolve(self, page, page_size, *uris):
|
||||
for uri in uris:
|
||||
try:
|
||||
parsed_uri = parse_lbry_uri(uri)
|
||||
if parsed_uri.claim_id:
|
||||
validate_claim_id(parsed_uri.claim_id)
|
||||
except URIParseError as err:
|
||||
return {'error': err.args[0]}
|
||||
except Exception as e:
|
||||
return {'error': str(e)}
|
||||
try:
|
||||
resolver = self.resolver
|
||||
resolutions = await self.network.get_values_for_uris(self.headers.hash().decode(), *uris)
|
||||
return await resolver._handle_resolutions(resolutions, uris, page, page_size)
|
||||
except Exception as e:
|
||||
log.exception(e)
|
||||
return {'error': str(e)}
|
||||
def resolve(self, page, page_size, *uris):
|
||||
return self.resolver.resolve(page, page_size, *uris)
|
||||
|
||||
async def get_claim_by_claim_id(self, claim_id):
|
||||
result = (await self.network.get_claims_by_ids(claim_id)).pop(claim_id, {})
|
||||
return await self.resolver.get_certificate_and_validate_result(result)
|
||||
return await self.resolver.parse_and_validate_claim_result(result)
|
||||
|
||||
async def get_claim_by_outpoint(self, txid, nout):
|
||||
claims = (await self.network.get_claims_in_tx(txid)) or []
|
||||
for claim in claims:
|
||||
if claim['nout'] == nout:
|
||||
return await self.resolver.get_certificate_and_validate_result(claim)
|
||||
return await self.resolver.parse_and_validate_claim_result(claim)
|
||||
return 'claim not found'
|
||||
|
||||
async def start(self):
|
||||
|
||||
+11
-17
@@ -281,16 +281,17 @@ class LbryWalletManager(BaseWalletManager):
|
||||
'is_tip': not is_my_inputs,
|
||||
'nout': txo.position
|
||||
})
|
||||
for txo in tx.other_support_outputs:
|
||||
item['support_info'].append({
|
||||
'address': txo.get_address(account.ledger),
|
||||
'balance_delta': dewies_to_lbc(-txo.amount),
|
||||
'amount': dewies_to_lbc(txo.amount),
|
||||
'claim_id': txo.claim_id,
|
||||
'claim_name': txo.claim_name,
|
||||
'is_tip': is_my_inputs,
|
||||
'nout': txo.position
|
||||
})
|
||||
if is_my_inputs:
|
||||
for txo in tx.other_support_outputs:
|
||||
item['support_info'].append({
|
||||
'address': txo.get_address(account.ledger),
|
||||
'balance_delta': dewies_to_lbc(-txo.amount),
|
||||
'amount': dewies_to_lbc(txo.amount),
|
||||
'claim_id': txo.claim_id,
|
||||
'claim_name': txo.claim_name,
|
||||
'is_tip': is_my_inputs,
|
||||
'nout': txo.position
|
||||
})
|
||||
for txo in tx.my_abandon_outputs:
|
||||
item['abandon_info'].append({
|
||||
'address': txo.get_address(account.ledger),
|
||||
@@ -307,13 +308,6 @@ class LbryWalletManager(BaseWalletManager):
|
||||
for wallet in self.wallets:
|
||||
wallet.save()
|
||||
|
||||
def get_block(self, block_hash=None, height=None):
|
||||
if height is None:
|
||||
height = self.ledger.headers.height
|
||||
if block_hash is None:
|
||||
block_hash = self.ledger.headers.hash(height).decode()
|
||||
return self.ledger.network.get_block(block_hash)
|
||||
|
||||
def get_claim_by_claim_id(self, claim_id):
|
||||
return self.ledger.get_claim_by_claim_id(claim_id)
|
||||
|
||||
|
||||
@@ -3,9 +3,6 @@ from torba.client.basenetwork import BaseNetwork
|
||||
|
||||
class Network(BaseNetwork):
|
||||
|
||||
def get_block(self, block_hash):
|
||||
return self.rpc('blockchain.block.get_block', block_hash)
|
||||
|
||||
def get_server_height(self):
|
||||
return self.rpc('blockchain.block.get_server_height')
|
||||
|
||||
|
||||
+142
-217
@@ -1,36 +1,71 @@
|
||||
import logging
|
||||
|
||||
import asyncio
|
||||
from functools import lru_cache
|
||||
|
||||
from cryptography.exceptions import InvalidSignature
|
||||
from binascii import unhexlify, hexlify
|
||||
|
||||
from lbrynet.wallet.account import validate_claim_id
|
||||
from lbrynet.wallet.dewies import dewies_to_lbc
|
||||
from lbrynet.error import UnknownNameError, UnknownClaimID, UnknownURI, UnknownOutpoint
|
||||
from lbrynet.schema.claim import Claim
|
||||
from google.protobuf.message import DecodeError
|
||||
from lbrynet.schema.uri import parse_lbry_uri
|
||||
from lbrynet.schema.uri import parse_lbry_uri, URIParseError
|
||||
from lbrynet.wallet.claim_proofs import verify_proof, InvalidProofError
|
||||
from lbrynet.wallet.transaction import Transaction
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Resolver:
|
||||
|
||||
def __init__(self, claim_trie_root, height, transaction_class, hash160_to_address, network, ledger):
|
||||
self.claim_trie_root = claim_trie_root
|
||||
self.height = height
|
||||
self.transaction_class = transaction_class
|
||||
self.hash160_to_address = hash160_to_address
|
||||
self.network = network
|
||||
def __init__(self, ledger):
|
||||
self.transaction_class = ledger.transaction_class
|
||||
self.network = ledger.network
|
||||
self.ledger = ledger
|
||||
|
||||
async def _handle_resolutions(self, resolutions, requested_uris, page, page_size):
|
||||
async def resolve(self, page, page_size, *uris):
|
||||
uris = set(uris)
|
||||
try:
|
||||
for uri in uris:
|
||||
parsed_uri = parse_lbry_uri(uri)
|
||||
if parsed_uri.claim_id:
|
||||
validate_claim_id(parsed_uri.claim_id)
|
||||
claim_trie_root = self.ledger.headers.claim_trie_root
|
||||
resolutions = await self.network.get_values_for_uris(self.ledger.headers.hash().decode(), *uris)
|
||||
if len(uris) > 1:
|
||||
return await self._batch_handle(resolutions, uris, page, page_size, claim_trie_root)
|
||||
return await self._handle_resolutions(resolutions, uris, page, page_size, claim_trie_root)
|
||||
except URIParseError as err:
|
||||
return {'error': err.args[0]}
|
||||
except Exception as e:
|
||||
log.exception(e)
|
||||
return {'error': str(e)}
|
||||
|
||||
async def _batch_handle(self, resolutions, uris, page, page_size, claim_trie_root):
|
||||
futs = []
|
||||
for uri in uris:
|
||||
futs.append(
|
||||
asyncio.ensure_future(self._handle_resolutions(resolutions, [uri], page, page_size, claim_trie_root))
|
||||
)
|
||||
results = await asyncio.gather(*futs)
|
||||
return dict(list(map(lambda result: list(result.items())[0], results)))
|
||||
|
||||
@lru_cache(256)
|
||||
def _fetch_tx(self, txid):
|
||||
async def __fetch_parse(txid):
|
||||
return self.transaction_class(unhexlify(await self.network.get_transaction(txid)))
|
||||
return asyncio.ensure_future(__fetch_parse(txid))
|
||||
|
||||
async def _handle_resolutions(self, resolutions, requested_uris, page, page_size, claim_trie_root):
|
||||
results = {}
|
||||
for uri in requested_uris:
|
||||
resolution = (resolutions or {}).get(uri, {})
|
||||
if resolution:
|
||||
try:
|
||||
results[uri] = _handle_claim_result(
|
||||
await self._handle_resolve_uri_response(uri, resolution, page, page_size)
|
||||
await self._handle_resolve_uri_response(uri, resolution, claim_trie_root, page, page_size),
|
||||
uri
|
||||
)
|
||||
except (UnknownNameError, UnknownClaimID, UnknownURI) as err:
|
||||
log.exception(err)
|
||||
@@ -39,50 +74,24 @@ class Resolver:
|
||||
results[uri] = {'error': "URI lbry://{} cannot be resolved".format(uri.replace("lbry://", ""))}
|
||||
return results
|
||||
|
||||
async def _handle_resolve_uri_response(self, uri, resolution, page=0, page_size=10, raw=False):
|
||||
async def _handle_resolve_uri_response(self, uri, resolution, claim_trie_root, page=0, page_size=10):
|
||||
result = {}
|
||||
claim_trie_root = self.claim_trie_root
|
||||
parsed_uri = parse_lbry_uri(uri)
|
||||
certificate = None
|
||||
certificate_response = None
|
||||
# parse an included certificate
|
||||
if 'certificate' in resolution:
|
||||
certificate_response = resolution['certificate']['result']
|
||||
certificate_resolution_type = resolution['certificate']['resolution_type']
|
||||
if certificate_resolution_type == "winning" and certificate_response:
|
||||
if 'height' in certificate_response:
|
||||
height = certificate_response['height']
|
||||
depth = self.height - height
|
||||
certificate_result = _verify_proof(parsed_uri.name,
|
||||
certificate_response = _verify_proof(parsed_uri.name,
|
||||
claim_trie_root,
|
||||
certificate_response,
|
||||
height, depth,
|
||||
transaction_class=self.transaction_class,
|
||||
hash160_to_address=self.hash160_to_address)
|
||||
result['certificate'] = await self.parse_and_validate_claim_result(certificate_result,
|
||||
raw=raw)
|
||||
elif certificate_resolution_type == "claim_id":
|
||||
result['certificate'] = await self.parse_and_validate_claim_result(certificate_response,
|
||||
raw=raw)
|
||||
elif certificate_resolution_type == "sequence":
|
||||
result['certificate'] = await self.parse_and_validate_claim_result(certificate_response,
|
||||
raw=raw)
|
||||
else:
|
||||
log.error("unknown response type: %s", certificate_resolution_type)
|
||||
|
||||
if 'certificate' in result:
|
||||
certificate = result['certificate']
|
||||
if 'unverified_claims_in_channel' in resolution:
|
||||
max_results = len(resolution['unverified_claims_in_channel'])
|
||||
result['claims_in_channel'] = max_results
|
||||
else:
|
||||
result['claims_in_channel'] = 0
|
||||
else:
|
||||
result['error'] = "claim not found"
|
||||
result['success'] = False
|
||||
result['uri'] = str(parsed_uri)
|
||||
|
||||
else:
|
||||
certificate = None
|
||||
ledger=self.ledger)
|
||||
elif certificate_resolution_type not in ['winning', 'claim_id', 'sequence']:
|
||||
raise Exception(f"unknown response type: {certificate_resolution_type}")
|
||||
result['certificate'] = await self.parse_and_validate_claim_result(certificate_response)
|
||||
result['claims_in_channel'] = len(resolution.get('unverified_claims_in_channel', []))
|
||||
|
||||
# if this was a resolution for a name, parse the result
|
||||
if 'claim' in resolution:
|
||||
@@ -90,27 +99,14 @@ class Resolver:
|
||||
claim_resolution_type = resolution['claim']['resolution_type']
|
||||
if claim_resolution_type == "winning" and claim_response:
|
||||
if 'height' in claim_response:
|
||||
height = claim_response['height']
|
||||
depth = self.height - height
|
||||
claim_result = _verify_proof(parsed_uri.name,
|
||||
claim_trie_root,
|
||||
claim_response,
|
||||
height, depth,
|
||||
transaction_class=self.transaction_class,
|
||||
hash160_to_address=self.hash160_to_address)
|
||||
result['claim'] = await self.parse_and_validate_claim_result(claim_result,
|
||||
certificate,
|
||||
raw)
|
||||
elif claim_resolution_type == "claim_id":
|
||||
result['claim'] = await self.parse_and_validate_claim_result(claim_response,
|
||||
certificate,
|
||||
raw)
|
||||
elif claim_resolution_type == "sequence":
|
||||
result['claim'] = await self.parse_and_validate_claim_result(claim_response,
|
||||
certificate,
|
||||
raw)
|
||||
else:
|
||||
log.error("unknown response type: %s", claim_resolution_type)
|
||||
claim_response = _verify_proof(parsed_uri.name,
|
||||
claim_trie_root,
|
||||
claim_response,
|
||||
ledger=self.ledger)
|
||||
elif claim_resolution_type not in ["sequence", "winning", "claim_id"]:
|
||||
raise Exception(f"unknown response type: {claim_resolution_type}")
|
||||
result['claim'] = await self.parse_and_validate_claim_result(claim_response,
|
||||
certificate_response)
|
||||
|
||||
# if this was a resolution for a name in a channel make sure there is only one valid
|
||||
# match
|
||||
@@ -152,58 +148,51 @@ class Resolver:
|
||||
return {'error': 'claim not found', 'success': False, 'uri': str(parsed_uri)}
|
||||
return result
|
||||
|
||||
async def get_certificate_and_validate_result(self, claim_result):
|
||||
async def parse_and_validate_claim_result(self, claim_result, certificate=None):
|
||||
if not claim_result or 'value' not in claim_result:
|
||||
return claim_result
|
||||
certificate = None
|
||||
certificate_id = Claim.from_bytes(unhexlify(claim_result['value'])).signing_channel_id
|
||||
if certificate_id:
|
||||
certificate = await self.network.get_claims_by_ids(certificate_id)
|
||||
certificate = certificate.pop(certificate_id) if certificate else None
|
||||
return await self.parse_and_validate_claim_result(claim_result, certificate=certificate)
|
||||
claim_result = _decode_claim_result(claim_result)
|
||||
if claim_result.get('height'):
|
||||
claim_result['timestamp'] = self.ledger.headers[claim_result['height']]['timestamp']
|
||||
if claim_result.get('depth'):
|
||||
claim_result['confirmations'] = claim_result.pop('depth')
|
||||
|
||||
async def parse_and_validate_claim_result(self, claim_result, certificate=None, raw=False):
|
||||
if not claim_result or 'value' not in claim_result:
|
||||
return claim_result
|
||||
|
||||
claim_result['decoded_claim'] = False
|
||||
decoded = None
|
||||
|
||||
if not raw:
|
||||
claim_value = claim_result['value']
|
||||
try:
|
||||
decoded = claim_result['value'] = Claim.from_bytes(unhexlify(claim_value))
|
||||
claim_result['decoded_claim'] = True
|
||||
except DecodeError:
|
||||
pass
|
||||
|
||||
if decoded:
|
||||
if claim_result['value']:
|
||||
claim_result['has_signature'] = False
|
||||
if decoded.is_signed:
|
||||
claim_tx = await self.network.get_transaction(claim_result['txid'])
|
||||
if claim_result['value'].is_signed:
|
||||
claim_result['has_signature'] = True
|
||||
claim_tx = await self._fetch_tx(claim_result['txid'])
|
||||
if certificate is None:
|
||||
log.info("fetching certificate to check claim signature")
|
||||
certificate = await self.network.get_claims_by_ids(decoded.signing_channel_id)
|
||||
channel_id = claim_result['value'].signing_channel_id
|
||||
certificate = (await self.network.get_claims_by_ids(channel_id)).get(channel_id)
|
||||
if not certificate:
|
||||
log.warning('Certificate %s not found', decoded.signing_channel_id)
|
||||
cert_tx = await self.network.get_transaction(certificate['txid']) if certificate else None
|
||||
claim_result['has_signature'] = True
|
||||
claim_result['signature_is_valid'] = False
|
||||
validated, channel_name = validate_claim_signature_and_get_channel_name(
|
||||
log.warning('Certificate %s not found', channel_id)
|
||||
claim_result['channel_name'] = certificate['name'] if certificate else None
|
||||
cert_tx = await self._fetch_tx(certificate['txid']) if certificate else None
|
||||
claim_result['signature_is_valid'] = validate_claim_signature_and_get_channel_name(
|
||||
claim_result, certificate, self.ledger, claim_tx=claim_tx, cert_tx=cert_tx
|
||||
)
|
||||
claim_result['channel_name'] = channel_name
|
||||
if validated:
|
||||
claim_result['signature_is_valid'] = True
|
||||
|
||||
if 'height' in claim_result and claim_result['height'] is None:
|
||||
claim_result['height'] = -1
|
||||
# fixme: workaround while json encoder isnt used here
|
||||
if cert_tx:
|
||||
channel_txo = cert_tx.outputs[certificate['nout']]
|
||||
claim_result['signing_channel'] = {
|
||||
'name': channel_txo.claim_name,
|
||||
'claim_id': channel_txo.claim_id,
|
||||
'value': channel_txo.claim
|
||||
}
|
||||
claim_result['is_channel_signature_valid'] = claim_result['signature_is_valid']
|
||||
|
||||
if 'amount' in claim_result:
|
||||
claim_result = format_amount_value(claim_result)
|
||||
claim_result['amount'] = dewies_to_lbc(claim_result['amount'])
|
||||
claim_result['effective_amount'] = dewies_to_lbc(claim_result['effective_amount'])
|
||||
claim_result['supports'] = [
|
||||
{'txid': txid, 'nout': nout, 'amount': dewies_to_lbc(amount)}
|
||||
for (txid, nout, amount) in claim_result['supports']
|
||||
]
|
||||
|
||||
claim_result['permanent_url'] = _get_permanent_url(
|
||||
claim_result, decoded.signing_channel_id if decoded else None)
|
||||
claim_result['height'] = claim_result.get('height', -1) or -1
|
||||
claim_result['permanent_url'] = f"lbry://{claim_result['name']}#{claim_result['claim_id']}"
|
||||
|
||||
return claim_result
|
||||
|
||||
@@ -284,87 +273,51 @@ class Resolver:
|
||||
start_position = (page - 1) * page_size
|
||||
queries, names, claim_positions = self.prepare_claim_queries(start_position, page_size,
|
||||
channel_claim_infos)
|
||||
page_generator = await self.iter_channel_claims_pages(queries, claim_positions, names,
|
||||
certificate, page_size=page_size)
|
||||
upper_bound = len(claim_positions)
|
||||
if not page:
|
||||
return None, upper_bound
|
||||
if start_position > upper_bound:
|
||||
raise IndexError("claim %i greater than max %i" % (start_position, upper_bound))
|
||||
page_generator = await self.iter_channel_claims_pages(queries, claim_positions, names,
|
||||
certificate, page_size=page_size)
|
||||
return page_generator, upper_bound
|
||||
|
||||
|
||||
# Format amount to be decimal encoded string
|
||||
# Format value to be hex encoded string
|
||||
# TODO: refactor. Came from lbryum, there could be another part of torba doing it
|
||||
def format_amount_value(obj):
|
||||
if isinstance(obj, dict):
|
||||
for k, v in obj.items():
|
||||
if k in ('amount', 'effective_amount'):
|
||||
if not isinstance(obj[k], float):
|
||||
obj[k] = dewies_to_lbc(obj[k])
|
||||
elif k == 'supports' and isinstance(v, list):
|
||||
obj[k] = [{'txid': txid, 'nout': nout, 'amount': dewies_to_lbc(amount)}
|
||||
for (txid, nout, amount) in v]
|
||||
elif isinstance(v, (list, dict)):
|
||||
obj[k] = format_amount_value(v)
|
||||
elif isinstance(obj, list):
|
||||
obj = [format_amount_value(o) for o in obj]
|
||||
return obj
|
||||
|
||||
|
||||
def _get_permanent_url(claim_result, certificate_id):
|
||||
if certificate_id:
|
||||
return f"{claim_result['channel_name']}#{certificate_id}/{claim_result['name']}"
|
||||
else:
|
||||
return f"{claim_result['name']}#{claim_result['claim_id']}"
|
||||
|
||||
|
||||
def _verify_proof(name, claim_trie_root, result, height, depth, transaction_class, hash160_to_address):
|
||||
def _verify_proof(name, claim_trie_root, result, ledger):
|
||||
"""
|
||||
Verify proof for name claim
|
||||
"""
|
||||
support_amount = sum([amt for (stxid, snout, amt) in result['supports']])
|
||||
|
||||
def _build_response(name, value, claim_id, txid, n, amount, effective_amount,
|
||||
claim_sequence, claim_address, supports):
|
||||
def _build_response(name, tx, nOut):
|
||||
output = tx.outputs[nOut]
|
||||
r = {
|
||||
'name': name,
|
||||
'value': hexlify(value),
|
||||
'claim_id': claim_id,
|
||||
'txid': txid,
|
||||
'nout': n,
|
||||
'amount': amount,
|
||||
'effective_amount': effective_amount,
|
||||
'height': height,
|
||||
'depth': depth,
|
||||
'claim_sequence': claim_sequence,
|
||||
'address': claim_address,
|
||||
'supports': supports
|
||||
'value': hexlify(output.script.values['claim']),
|
||||
'claim_id': output.claim_id,
|
||||
'txid': tx.id,
|
||||
'nout': nOut,
|
||||
'amount': output.amount,
|
||||
'effective_amount': output.amount + support_amount,
|
||||
'height': result['height'],
|
||||
'confirmations': result['depth'],
|
||||
'claim_sequence': result['claim_sequence'],
|
||||
'address': output.get_address(ledger),
|
||||
'valid_at_height': result['valid_at_height'],
|
||||
'timestamp': ledger.headers[result['height']]['timestamp'],
|
||||
'supports': result['supports']
|
||||
}
|
||||
return r
|
||||
|
||||
def _parse_proof_result(name, result):
|
||||
support_amount = sum([amt for (stxid, snout, amt) in result['supports']])
|
||||
supports = result['supports']
|
||||
if 'txhash' in result['proof'] and 'nOut' in result['proof']:
|
||||
if 'transaction' in result:
|
||||
tx = transaction_class(raw=unhexlify(result['transaction']))
|
||||
tx = ledger.transaction_class(raw=unhexlify(result['transaction']))
|
||||
nOut = result['proof']['nOut']
|
||||
if result['proof']['txhash'] == tx.id:
|
||||
if 0 <= nOut < len(tx.outputs):
|
||||
claim_output = tx.outputs[nOut]
|
||||
effective_amount = claim_output.amount + support_amount
|
||||
claim_address = hash160_to_address(claim_output.script.values['pubkey_hash'])
|
||||
claim_id = result['claim_id']
|
||||
claim_sequence = result['claim_sequence']
|
||||
claim_script = claim_output.script
|
||||
decoded_name = claim_script.values['claim_name'].decode()
|
||||
decoded_value = claim_script.values['claim']
|
||||
if decoded_name == name:
|
||||
return _build_response(name, decoded_value, claim_id,
|
||||
tx.id, nOut, claim_output.amount,
|
||||
effective_amount, claim_sequence,
|
||||
claim_address, supports)
|
||||
if tx.outputs[nOut].claim_name == name:
|
||||
return _build_response(name, tx, nOut)
|
||||
return {'error': 'name in proof did not match requested name'}
|
||||
outputs = len(tx['outputs'])
|
||||
return {'error': 'invalid nOut: %d (let(outputs): %d' % (nOut, outputs)}
|
||||
@@ -375,12 +328,8 @@ def _verify_proof(name, claim_trie_root, result, height, depth, transaction_clas
|
||||
return {'error': 'name is not claimed'}
|
||||
|
||||
if 'proof' in result:
|
||||
proof_name = name
|
||||
if 'name' in result:
|
||||
proof_name = result['name']
|
||||
name = result['name']
|
||||
if 'normalized_name' in result:
|
||||
proof_name = result['normalized_name']
|
||||
name = result.get('name', name)
|
||||
proof_name = result.get('normalized_name', name)
|
||||
try:
|
||||
verify_proof(result['proof'], claim_trie_root, proof_name)
|
||||
except InvalidProofError:
|
||||
@@ -392,48 +341,46 @@ def _verify_proof(name, claim_trie_root, result, height, depth, transaction_clas
|
||||
|
||||
def validate_claim_signature_and_get_channel_name(claim_result, certificate_claim, ledger,
|
||||
claim_tx=None, cert_tx=None):
|
||||
valid_signature = False
|
||||
if cert_tx and certificate_claim and claim_tx and claim_result:
|
||||
tx = Transaction(unhexlify(claim_tx))
|
||||
cert_tx = Transaction(unhexlify(cert_tx))
|
||||
try:
|
||||
is_signed = tx.outputs[claim_result['nout']].is_signed_by(
|
||||
valid_signature = claim_tx.outputs[claim_result['nout']].is_signed_by(
|
||||
cert_tx.outputs[certificate_claim['nout']], ledger
|
||||
)
|
||||
except InvalidSignature:
|
||||
return False, None
|
||||
return is_signed, certificate_claim['name']
|
||||
return False, None
|
||||
pass
|
||||
if not valid_signature:
|
||||
log.warning("lbry://%s#%s has an invalid signature",
|
||||
claim_result['name'], claim_result['claim_id'])
|
||||
return valid_signature
|
||||
|
||||
|
||||
# TODO: The following came from code handling lbryum results. Now that it's all in one place a refactor should unify it.
|
||||
def _decode_claim_result(claim):
|
||||
if 'has_signature' in claim and claim['has_signature']:
|
||||
if not claim['signature_is_valid']:
|
||||
log.warning("lbry://%s#%s has an invalid signature",
|
||||
claim['name'], claim['claim_id'])
|
||||
if 'decoded_claim' in claim:
|
||||
return claim
|
||||
if 'value' not in claim:
|
||||
log.warning('Got an invalid claim while parsing, please report: %s', claim)
|
||||
claim['hex'] = None
|
||||
claim['protobuf'] = None
|
||||
claim['value'] = None
|
||||
backend_message = ' SDK message: ' + claim['error'] if 'error' in claim else ''
|
||||
backend_message = ' SDK message: ' + claim.get('error', '')
|
||||
claim['error'] = "Failed to parse: missing value." + backend_message
|
||||
return claim
|
||||
try:
|
||||
if not isinstance(claim['value'], Claim):
|
||||
claim['value'] = Claim.from_bytes(claim['value'])
|
||||
claim['hex'] = hexlify(claim['value'].to_bytes())
|
||||
claim['value'] = Claim.from_bytes(unhexlify(claim['value']))
|
||||
claim['protobuf'] = hexlify(claim['value'].to_bytes())
|
||||
claim['decoded_claim'] = True
|
||||
except DecodeError:
|
||||
claim['hex'] = claim['value']
|
||||
claim['decoded_claim'] = False
|
||||
claim['protobuf'] = claim['value']
|
||||
claim['value'] = None
|
||||
claim['error'] = "Failed to decode value"
|
||||
return claim
|
||||
|
||||
|
||||
def _handle_claim_result(results):
|
||||
def _handle_claim_result(results, uri):
|
||||
if not results:
|
||||
#TODO: cannot determine what name we searched for here
|
||||
# we should fix lbryum commands that return None
|
||||
raise UnknownNameError("")
|
||||
raise UnknownURI(uri)
|
||||
|
||||
if 'error' in results:
|
||||
if results['error'] in ['name is not claimed', 'claim not found']:
|
||||
@@ -446,26 +393,8 @@ def _handle_claim_result(results):
|
||||
if 'outpoint' in results:
|
||||
raise UnknownOutpoint(results['outpoint'])
|
||||
raise Exception(results['error'])
|
||||
|
||||
# case where return value is {'certificate':{'txid', 'value',...},...}
|
||||
if 'certificate' in results:
|
||||
results['certificate'] = _decode_claim_result(results['certificate'])
|
||||
|
||||
# case where return value is {'claim':{'txid','value',...},...}
|
||||
if 'claim' in results:
|
||||
results['claim'] = _decode_claim_result(results['claim'])
|
||||
|
||||
# case where return value is {'txid','value',...}
|
||||
# returned by queries that are not name resolve related
|
||||
# (getclaimbyoutpoint, getclaimbyid, getclaimsfromtx)
|
||||
elif 'value' in results:
|
||||
results = _decode_claim_result(results)
|
||||
|
||||
# case where there is no 'certificate', 'value', or 'claim' key
|
||||
elif 'certificate' not in results:
|
||||
msg = f'result in unexpected format:{results}'
|
||||
assert False, msg
|
||||
|
||||
if not {'value', 'claim', 'certificate'}.intersection(results.keys()):
|
||||
raise Exception(f'result in unexpected format:{results}')
|
||||
return results
|
||||
|
||||
|
||||
@@ -474,13 +403,9 @@ def pick_winner_from_channel_path_collision(claims_in_channel):
|
||||
# claim triggers another issue where 2 claims cant be saved for the same file. This code picks the oldest, so it
|
||||
# stays the same. Using effective amount would change the resolved claims for a channel path on takeovers,
|
||||
# potentially triggering that.
|
||||
winner = None
|
||||
winner = {}
|
||||
for claim in claims_in_channel:
|
||||
if not claim['signature_is_valid']:
|
||||
continue
|
||||
if winner is None:
|
||||
winner = claim
|
||||
elif claim['height'] < winner['height'] or \
|
||||
if not winner or claim['height'] < winner['height'] or \
|
||||
(claim['height'] == winner['height'] and claim['nout'] < winner['nout']):
|
||||
winner = claim
|
||||
return winner
|
||||
winner = claim if claim['signature_is_valid'] else winner
|
||||
return winner or None
|
||||
@@ -41,7 +41,6 @@ class LBRYElectrumX(ElectrumX):
|
||||
'blockchain.claimtrie.getvaluesforuris': self.claimtrie_getvalueforuris,
|
||||
'blockchain.claimtrie.getclaimssignedbyid': self.claimtrie_getclaimssignedbyid,
|
||||
'blockchain.block.get_server_height': self.get_server_height,
|
||||
'blockchain.block.get_block': self.get_block,
|
||||
}
|
||||
# fixme: methods we use but shouldnt be using anymore. To be removed when torba goes out
|
||||
handlers.update({
|
||||
@@ -104,9 +103,6 @@ class LBRYElectrumX(ElectrumX):
|
||||
verbose = False
|
||||
return await self.daemon_request('getrawtransaction', tx_hash, verbose)
|
||||
|
||||
async def get_block(self, block_hash):
|
||||
return await self.daemon.deserialised_block(block_hash)
|
||||
|
||||
async def get_server_height(self):
|
||||
return self.bp.height
|
||||
|
||||
@@ -241,7 +237,7 @@ class LBRYElectrumX(ElectrumX):
|
||||
"txid": claim['txid'],
|
||||
"nout": claim['n'],
|
||||
"amount": amount,
|
||||
"depth": self.db.db_height - height,
|
||||
"depth": self.db.db_height - height + 1,
|
||||
"height": height,
|
||||
"value": hexlify(claim['value'].encode('ISO-8859-1')).decode(),
|
||||
"address": address, # from index
|
||||
|
||||
@@ -80,13 +80,7 @@ class Output(BaseOutput):
|
||||
@property
|
||||
def permanent_url(self) -> str:
|
||||
if self.script.is_claim_involved:
|
||||
if self.channel is not None:
|
||||
return "{}#{}/{}".format(
|
||||
self.channel.claim_name,
|
||||
self.channel.claim_id,
|
||||
self.claim_name
|
||||
)
|
||||
return f"{self.claim_name}#{self.claim_id}"
|
||||
return f"lbry://{self.claim_name}#{self.claim_id}"
|
||||
raise ValueError('No claim associated.')
|
||||
|
||||
@property
|
||||
@@ -98,11 +92,11 @@ class Output(BaseOutput):
|
||||
pieces = [
|
||||
Base58.decode(self.get_address(ledger)),
|
||||
self.claim.unsigned_payload,
|
||||
self.claim.signing_channel_hash
|
||||
self.claim.signing_channel_hash[::-1]
|
||||
]
|
||||
else:
|
||||
pieces = [
|
||||
self.tx_ref.tx.inputs[0].txo_ref.id.encode(),
|
||||
self.tx_ref.tx.inputs[0].txo_ref.hash,
|
||||
self.claim.signing_channel_hash,
|
||||
self.claim.to_message_bytes()
|
||||
]
|
||||
@@ -120,7 +114,7 @@ class Output(BaseOutput):
|
||||
self.channel = channel
|
||||
self.claim.signing_channel_hash = channel.claim_hash
|
||||
digest = sha256(b''.join([
|
||||
first_input_id or self.tx_ref.tx.inputs[0].txo_ref.id.encode(),
|
||||
first_input_id or self.tx_ref.tx.inputs[0].txo_ref.hash,
|
||||
self.claim.signing_channel_hash,
|
||||
self.claim.to_message_bytes()
|
||||
]))
|
||||
@@ -128,6 +122,10 @@ class Output(BaseOutput):
|
||||
self.claim.signature = private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
|
||||
self.script.generate()
|
||||
|
||||
def clear_signature(self):
|
||||
self.channel = None
|
||||
self.claim.clear_signature()
|
||||
|
||||
def generate_channel_private_key(self):
|
||||
private_key = ecdsa.SigningKey.generate(curve=ecdsa.SECP256k1, hashfunc=hashlib.sha256)
|
||||
self.private_key = private_key.to_pem().decode()
|
||||
@@ -194,15 +192,17 @@ class Transaction(BaseTransaction):
|
||||
|
||||
@classmethod
|
||||
def claim_update(
|
||||
cls, previous_claim: Output, amount: int, holding_address: str,
|
||||
cls, previous_claim: Output, claim: Claim, amount: int, holding_address: str,
|
||||
funding_accounts: List[Account], change_account: Account, signing_channel: Output = None):
|
||||
ledger = cls.ensure_all_have_same_ledger(funding_accounts, change_account)
|
||||
updated_claim = Output.pay_update_claim_pubkey_hash(
|
||||
amount, previous_claim.claim_name, previous_claim.claim_id,
|
||||
previous_claim.claim, ledger.address_to_hash160(holding_address)
|
||||
claim, ledger.address_to_hash160(holding_address)
|
||||
)
|
||||
if signing_channel is not None:
|
||||
updated_claim.sign(signing_channel, b'placeholder txid:nout')
|
||||
else:
|
||||
updated_claim.clear_signature()
|
||||
return cls.create(
|
||||
[Input.spend(previous_claim)], [updated_claim], funding_accounts, change_account, sign=False
|
||||
)
|
||||
|
||||
@@ -25,6 +25,9 @@ RUN python3.7 -m pip install --upgrade pip setuptools
|
||||
# get torba
|
||||
RUN python3.7 -m pip install --user git+https://github.com/lbryio/torba.git#egg=torba
|
||||
|
||||
# get uvloop
|
||||
RUN python3.7 -m pip install --user uvloop
|
||||
|
||||
# copy lbrynet
|
||||
RUN mkdir projects/
|
||||
COPY . projects/lbry
|
||||
@@ -46,4 +49,5 @@ ENV DB_DIRECTORY=$db_dir
|
||||
ENV BANDWIDTH_LIMIT=1000000000000000000000000000000000000000000
|
||||
ENV MAX_SESSIONS=1000000000
|
||||
ENV MAX_SEND=1000000000000000000
|
||||
ENV EVENT_LOOP_POLICY=uvloop
|
||||
ENTRYPOINT ["/home/lbry/.local/bin/torba-server"]
|
||||
@@ -5,7 +5,7 @@ import socket
|
||||
import ipaddress
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
|
||||
import logging
|
||||
|
||||
@@ -32,7 +32,7 @@ async def main(blob_hash: str, url: str):
|
||||
host = host_info[0][4][0]
|
||||
|
||||
storage = SQLiteStorage(conf, os.path.join(conf.data_dir, "lbrynet.sqlite"))
|
||||
blob_manager = BlobFileManager(loop, os.path.join(conf.data_dir, "blobfiles"), storage)
|
||||
blob_manager = BlobManager(loop, os.path.join(conf.data_dir, "blobfiles"), storage)
|
||||
await storage.open()
|
||||
await blob_manager.setup()
|
||||
|
||||
|
||||
@@ -2,7 +2,380 @@ import os
|
||||
import re
|
||||
import json
|
||||
import inspect
|
||||
from lbrynet.extras.daemon.Daemon import Daemon
|
||||
import tempfile
|
||||
import asyncio
|
||||
import time
|
||||
from docopt import docopt
|
||||
from binascii import unhexlify
|
||||
from textwrap import indent
|
||||
from lbrynet.extras.cli import set_kwargs, get_argument_parser
|
||||
from lbrynet.extras.daemon.Daemon import (
|
||||
Daemon, jsonrpc_dumps_pretty, encode_pagination_doc
|
||||
)
|
||||
from lbrynet.extras.daemon.json_response_encoder import (
|
||||
encode_tx_doc, encode_txo_doc, encode_account_doc, encode_file_doc
|
||||
)
|
||||
from lbrynet.testcase import CommandTestCase
|
||||
|
||||
|
||||
RETURN_DOCS = {
|
||||
'Account': encode_account_doc(),
|
||||
'File': encode_file_doc(),
|
||||
'Transaction': encode_tx_doc(),
|
||||
'Output': encode_txo_doc(),
|
||||
'Address': 'an address in base58'
|
||||
}
|
||||
|
||||
|
||||
class ExampleRecorder:
|
||||
def __init__(self, test):
|
||||
self.test = test
|
||||
self.examples = {}
|
||||
|
||||
async def __call__(self, title, *command):
|
||||
parser = get_argument_parser()
|
||||
args, command_args = parser.parse_known_args(command)
|
||||
|
||||
api_method_name = args.api_method_name
|
||||
parsed = docopt(args.doc, command_args)
|
||||
kwargs = set_kwargs(parsed)
|
||||
for k, v in kwargs.items():
|
||||
if v and isinstance(v, str) and (v[0], v[-1]) == ('"', '"'):
|
||||
kwargs[k] = v[1:-1]
|
||||
params = json.dumps({"method": api_method_name, "params": kwargs})
|
||||
|
||||
method = getattr(self.test.daemon, f'jsonrpc_{api_method_name}')
|
||||
result = method(**kwargs)
|
||||
if asyncio.iscoroutine(result):
|
||||
result = await result
|
||||
output = jsonrpc_dumps_pretty(result, ledger=self.test.daemon.ledger)
|
||||
self.examples.setdefault(api_method_name, []).append({
|
||||
'title': title,
|
||||
'curl': f"curl -d'{params}' http://localhost:5279/",
|
||||
'lbrynet': 'lbrynet ' + ' '.join(command),
|
||||
'python': f'requests.post("http://localhost:5279", json={params}).json()',
|
||||
'output': output.strip()
|
||||
})
|
||||
return json.loads(output)['result']
|
||||
|
||||
|
||||
class Examples(CommandTestCase):
|
||||
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
self.recorder = ExampleRecorder(self)
|
||||
|
||||
async def play(self):
|
||||
r = self.recorder
|
||||
|
||||
# general sdk
|
||||
|
||||
await r(
|
||||
'Get status',
|
||||
'status'
|
||||
)
|
||||
await r(
|
||||
'Get version',
|
||||
'version'
|
||||
)
|
||||
await r(
|
||||
'Get settings',
|
||||
'settings', 'get'
|
||||
)
|
||||
|
||||
# accounts
|
||||
|
||||
await r(
|
||||
'List your accounts',
|
||||
'account', 'list'
|
||||
)
|
||||
|
||||
account = await r(
|
||||
'Create an account',
|
||||
'account', 'create', '"generated account"'
|
||||
)
|
||||
|
||||
await r(
|
||||
'Remove an account',
|
||||
'account', 'remove', account['id']
|
||||
)
|
||||
|
||||
await r(
|
||||
'Add an account from seed',
|
||||
'account', 'add', '"new account"', f"--seed=\"{account['seed']}\""
|
||||
)
|
||||
|
||||
await r(
|
||||
'Modify maximum number of times a change address can be reused',
|
||||
'account', 'set', account['id'], '--change_max_uses=10'
|
||||
)
|
||||
|
||||
# addresses
|
||||
|
||||
await r(
|
||||
'List addresses in default account',
|
||||
'address', 'list'
|
||||
)
|
||||
|
||||
an_address = await r(
|
||||
'Get an unused address',
|
||||
'address', 'unused'
|
||||
)
|
||||
|
||||
address_list_by_id = await r(
|
||||
'List addresses in specified account',
|
||||
'address', 'list', f"--account_id=\"{account['id']}\""
|
||||
)
|
||||
|
||||
await r(
|
||||
'Check if address is mine',
|
||||
'address', 'is_mine', an_address
|
||||
)
|
||||
|
||||
# sends/funds
|
||||
|
||||
transfer = await r(
|
||||
'Transfer 2 LBC from default account to specific account',
|
||||
'account', 'fund', f"--to_account=\"{account['id']}\"", "--amount=2.0", "--broadcast"
|
||||
)
|
||||
|
||||
await self.on_transaction_dict(transfer)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(transfer)
|
||||
|
||||
await r(
|
||||
'Get default account balance',
|
||||
'account', 'balance'
|
||||
)
|
||||
|
||||
txlist = await r(
|
||||
'List your transactions',
|
||||
'transaction', 'list'
|
||||
)
|
||||
|
||||
await r(
|
||||
'Get balance for specific account by id',
|
||||
'account', 'balance', f"\"{account['id']}\""
|
||||
)
|
||||
|
||||
spread_transaction = await r(
|
||||
'Spread LBC between multiple addresses',
|
||||
'account', 'fund', f"--to_account=\"{account['id']}\"", f"--from_account=\"{account['id']}\"", '--amount=1.5', '--outputs=2', '--broadcast'
|
||||
)
|
||||
|
||||
await self.on_transaction_dict(spread_transaction)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(spread_transaction)
|
||||
|
||||
await r(
|
||||
'Transfer all LBC to a specified account',
|
||||
'account', 'fund', f"--from_account=\"{account['id']}\"", "--everything", "--broadcast"
|
||||
)
|
||||
|
||||
# channels
|
||||
|
||||
channel = await r(
|
||||
'Create a channel claim without metadata',
|
||||
'channel', 'create', '@channel', '1.0'
|
||||
)
|
||||
channel_id = channel['outputs'][0]['claim_id']
|
||||
await self.on_transaction_dict(channel)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(channel)
|
||||
|
||||
await r(
|
||||
'List your channel claims',
|
||||
'channel', 'list'
|
||||
)
|
||||
|
||||
await r(
|
||||
'Paginate your channel claims',
|
||||
'channel', 'list', '--page=1', '--page_size=20'
|
||||
)
|
||||
|
||||
channel = await r(
|
||||
'Update a channel claim',
|
||||
'channel', 'update', channel['outputs'][0]['claim_id'], '--title="New Channel"'
|
||||
)
|
||||
|
||||
await self.on_transaction_dict(channel)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(channel)
|
||||
|
||||
big_channel = await r(
|
||||
'Create a channel claim with all metadata',
|
||||
'channel', 'create', '@bigchannel', '1.0',
|
||||
'--title="Big Channel"', '--description="A channel with lots of videos."',
|
||||
'--email="creator@smallmedia.com"', '--tags=music', '--tags=art',
|
||||
'--languages=pt-BR', '--languages=uk', '--locations=BR', '--locations=UA::Kiyv',
|
||||
'--website_url="http://smallmedia.com"', '--thumbnail_url="http://smallmedia.com/logo.jpg"',
|
||||
'--cover_url="http://smallmedia.com/logo.jpg"'
|
||||
)
|
||||
await self.on_transaction_dict(big_channel)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(big_channel)
|
||||
await self.daemon.jsonrpc_channel_abandon(big_channel['outputs'][0]['claim_id'])
|
||||
await self.generate(1)
|
||||
|
||||
# stream claims
|
||||
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'hello world')
|
||||
file.flush()
|
||||
stream = await r(
|
||||
'Create a stream claim without metadata',
|
||||
'stream', 'create', 'astream', '1.0', file.name
|
||||
)
|
||||
await self.on_transaction_dict(stream)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(stream)
|
||||
stream_id = stream['outputs'][0]['claim_id']
|
||||
stream_name = stream['outputs'][0]['name']
|
||||
stream = await r(
|
||||
'Update a stream claim to add channel',
|
||||
'stream', 'update', stream_id,
|
||||
f'--channel_id="{channel_id}"'
|
||||
)
|
||||
await self.on_transaction_dict(stream)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(stream)
|
||||
|
||||
await r(
|
||||
'List all your claims',
|
||||
'claim', 'list'
|
||||
)
|
||||
|
||||
await r(
|
||||
'Paginate your claims',
|
||||
'claim', 'list', '--page=1', '--page_size=20'
|
||||
)
|
||||
|
||||
await r(
|
||||
'List all your stream claims',
|
||||
'stream', 'list'
|
||||
)
|
||||
|
||||
await r(
|
||||
'Paginate your stream claims',
|
||||
'stream', 'list', '--page=1', '--page_size=20'
|
||||
)
|
||||
|
||||
await r(
|
||||
'Search for all claims in channel',
|
||||
'claim', 'search', f'--channel_id="{channel_id}"'
|
||||
)
|
||||
|
||||
await r(
|
||||
'Search for claims matching a name',
|
||||
'claim', 'search', f'--name="{stream_name}"'
|
||||
)
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix='.png') as file:
|
||||
file.write(unhexlify(
|
||||
b'89504e470d0a1a0a0000000d49484452000000050000000708020000004fc'
|
||||
b'510b9000000097048597300000b1300000b1301009a9c1800000015494441'
|
||||
b'5408d763fcffff3f031260624005d4e603004c45030b5286e9ea000000004'
|
||||
b'9454e44ae426082'
|
||||
))
|
||||
file.flush()
|
||||
big_stream = await r(
|
||||
'Create an image stream claim with all metadata and fee',
|
||||
'stream', 'create', 'blank-image', '1.0', file.name,
|
||||
'--tags=blank', '--tags=art', '--languages=en', '--locations=US:NH:Manchester',
|
||||
'--fee_currency=LBC', '--fee_amount=0.3',
|
||||
'--title="Blank Image"', '--description="A blank PNG that is 5x7."', '--author=Picaso',
|
||||
'--license="Public Domain"', '--license_url=http://public-domain.org',
|
||||
'--thumbnail_url="http://smallmedia.com/thumbnail.jpg"', f'--release_time={int(time.time())}',
|
||||
f'--channel_id="{channel_id}"'
|
||||
)
|
||||
await self.on_transaction_dict(big_stream)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(big_stream)
|
||||
await self.daemon.jsonrpc_channel_abandon(big_stream['outputs'][0]['claim_id'])
|
||||
await self.generate(1)
|
||||
|
||||
# files
|
||||
|
||||
file_list_result = await r(
|
||||
'List local files',
|
||||
'file', 'list'
|
||||
)
|
||||
file_uri = f"{file_list_result[0]['claim_name']}#{file_list_result[0]['claim_id']}"
|
||||
await r(
|
||||
'Resolve a claim',
|
||||
'resolve', file_uri
|
||||
)
|
||||
|
||||
await r(
|
||||
'List files matching a parameter',
|
||||
'file', 'list', f"--claim_id=\"{file_list_result[0]['claim_id']}\""
|
||||
)
|
||||
|
||||
await r(
|
||||
'Delete a file',
|
||||
'file', 'delete', f"--claim_id=\"{file_list_result[0]['claim_id']}\""
|
||||
)
|
||||
|
||||
await r(
|
||||
'Get a file',
|
||||
'get', file_uri
|
||||
)
|
||||
|
||||
await r(
|
||||
'Save a file to the downloads directory',
|
||||
'file', 'save', f"--sd_hash=\"{file_list_result[0]['sd_hash']}\""
|
||||
)
|
||||
|
||||
# blobs
|
||||
|
||||
bloblist = await r(
|
||||
'List your local blobs',
|
||||
'blob', 'list'
|
||||
)
|
||||
|
||||
await r(
|
||||
'Delete a blob',
|
||||
'blob', 'delete', f"{bloblist[0]}"
|
||||
)
|
||||
|
||||
# abandon all the things
|
||||
|
||||
abandon_stream = await r(
|
||||
'Abandon a stream claim',
|
||||
'stream', 'abandon', stream_id
|
||||
)
|
||||
await self.on_transaction_dict(abandon_stream)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(abandon_stream)
|
||||
|
||||
abandon_channel = await r(
|
||||
'Abandon a channel claim',
|
||||
'channel', 'abandon', channel_id
|
||||
)
|
||||
await self.on_transaction_dict(abandon_channel)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(abandon_channel)
|
||||
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
file.write(b'hello world')
|
||||
file.flush()
|
||||
stream = await r(
|
||||
'Publish a file',
|
||||
'publish', 'a-new-stream', '--bid=1.0', f'--file_path={file.name}'
|
||||
)
|
||||
await self.on_transaction_dict(stream)
|
||||
await self.generate(1)
|
||||
await self.on_transaction_dict(stream)
|
||||
|
||||
|
||||
def get_examples():
|
||||
player = Examples('play')
|
||||
result = player.run()
|
||||
if result.errors:
|
||||
for error in result.errors:
|
||||
print(error[1])
|
||||
raise Exception('See above for errors while running the examples.')
|
||||
return player.recorder.examples
|
||||
|
||||
|
||||
SECTIONS = re.compile("(.*?)Usage:(.*?)Options:(.*?)Returns:(.*)", re.DOTALL)
|
||||
@@ -11,7 +384,27 @@ ARGUMENT_NAME = re.compile("--([^=]+)")
|
||||
ARGUMENT_TYPE = re.compile("\s*\((.*?)\)(.*)")
|
||||
|
||||
|
||||
def get_api(obj):
|
||||
def get_return_def(returns):
|
||||
result = returns.strip()
|
||||
if (result[0], result[-1]) == ('{', '}'):
|
||||
obj_type = result[1:-1]
|
||||
if '[' in obj_type:
|
||||
sub_type = obj_type[obj_type.index('[')+1:-1]
|
||||
obj_type = obj_type[:obj_type.index('[')]
|
||||
if obj_type == 'Paginated':
|
||||
obj_def = encode_pagination_doc(RETURN_DOCS[sub_type])
|
||||
elif obj_type == 'List':
|
||||
obj_def = [RETURN_DOCS[sub_type]]
|
||||
else:
|
||||
raise NameError(f'Unknown return type: {obj_type}')
|
||||
else:
|
||||
obj_def = RETURN_DOCS[obj_type]
|
||||
return indent(json.dumps(obj_def, indent=4), ' '*12)
|
||||
return result
|
||||
|
||||
|
||||
def get_api(name, examples):
|
||||
obj = Daemon.callable_methods[name]
|
||||
docstr = inspect.getdoc(obj).strip()
|
||||
|
||||
try:
|
||||
@@ -45,17 +438,35 @@ def get_api(obj):
|
||||
arg['description'] = ' '.join(arg['description'])
|
||||
|
||||
return {
|
||||
'name': obj.__name__[len('jsonrpc_'):],
|
||||
'name': name,
|
||||
'description': description.strip(),
|
||||
'arguments': arguments,
|
||||
'returns': returns.strip()
|
||||
'returns': get_return_def(returns),
|
||||
'examples': examples
|
||||
}
|
||||
|
||||
|
||||
def write_api(f):
|
||||
apis = []
|
||||
for method_name in sorted(Daemon.callable_methods.keys()):
|
||||
apis.append(get_api(Daemon.callable_methods[method_name]))
|
||||
examples = get_examples()
|
||||
api_definitions = Daemon.get_api_definitions()
|
||||
apis = {
|
||||
'main': {
|
||||
'doc': 'Ungrouped commands.',
|
||||
'commands': []
|
||||
}
|
||||
}
|
||||
for group_name, group_doc in api_definitions['groups'].items():
|
||||
apis[group_name] = {
|
||||
'doc': group_doc,
|
||||
'commands': []
|
||||
}
|
||||
for method_name, command in api_definitions['commands'].items():
|
||||
if 'replaced_by' in command:
|
||||
continue
|
||||
apis[command['group'] or 'main']['commands'].append(get_api(
|
||||
method_name,
|
||||
examples.get(method_name, [])
|
||||
))
|
||||
json.dump(apis, f, indent=4)
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import sys
|
||||
import os
|
||||
import asyncio
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
from lbrynet.blob_exchange.server import BlobServer
|
||||
from lbrynet.schema.address import decode_address
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
@@ -17,7 +17,7 @@ async def main(address: str):
|
||||
|
||||
storage = SQLiteStorage(os.path.expanduser("~/.lbrynet/lbrynet.sqlite"))
|
||||
await storage.open()
|
||||
blob_manager = BlobFileManager(loop, os.path.expanduser("~/.lbrynet/blobfiles"), storage)
|
||||
blob_manager = BlobManager(loop, os.path.expanduser("~/.lbrynet/blobfiles"), storage)
|
||||
await blob_manager.setup()
|
||||
|
||||
server = await loop.create_server(
|
||||
|
||||
@@ -11,7 +11,7 @@ from lbrynet.blob.blob_file import MAX_BLOB_SIZE
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.schema.uri import parse_lbry_uri
|
||||
from lbrynet.extras.daemon.client import daemon_rpc
|
||||
from lbrynet.extras import system_info, cli
|
||||
from lbrynet.extras import system_info
|
||||
|
||||
|
||||
def extract_uris(response):
|
||||
@@ -58,7 +58,7 @@ def variance(times):
|
||||
return round(sum(((i - mean) ** 2.0 for i in times)) / (len(times) - 1), 3)
|
||||
|
||||
|
||||
async def wait_for_done(conf, uri):
|
||||
async def wait_for_done(conf, uri, timeout):
|
||||
name = uri.split("#")[0]
|
||||
last_complete = 0
|
||||
hang_count = 0
|
||||
@@ -73,11 +73,11 @@ async def wait_for_done(conf, uri):
|
||||
else:
|
||||
hang_count += 1
|
||||
await asyncio.sleep(1.0)
|
||||
if hang_count > 10:
|
||||
if hang_count > timeout:
|
||||
return False, file['blobs_completed'], file['blobs_in_stream']
|
||||
|
||||
|
||||
async def main(uris=None, allow_fees=False):
|
||||
async def main(uris=None, cmd_args=None):
|
||||
if not uris:
|
||||
uris = await get_frontpage_uris()
|
||||
conf = Config()
|
||||
@@ -90,15 +90,16 @@ async def main(uris=None, allow_fees=False):
|
||||
print("**********************************************")
|
||||
|
||||
resolvable = []
|
||||
for name in uris:
|
||||
resolved = await daemon_rpc(conf, 'resolve', name)
|
||||
async def __resolve(name):
|
||||
resolved = await daemon_rpc(conf, 'resolve', urls=[name])
|
||||
if 'error' not in resolved.get(name, {}):
|
||||
if ("fee" not in resolved[name]['claim']['value']['stream']['metadata']) or allow_fees:
|
||||
if ("fee" not in resolved[name]['claim']['value']) or cmd_args.allow_fees:
|
||||
resolvable.append(name)
|
||||
else:
|
||||
print(f"{name} has a fee, skipping it")
|
||||
else:
|
||||
print(f"failed to resolve {name}: {resolved[name]['error']}")
|
||||
await asyncio.gather(*(__resolve(name) for name in uris))
|
||||
print(f"attempting to download {len(resolvable)}/{len(uris)} frontpage streams")
|
||||
|
||||
first_byte_times = []
|
||||
@@ -113,24 +114,29 @@ async def main(uris=None, allow_fees=False):
|
||||
for i, uri in enumerate(resolvable):
|
||||
start = time.time()
|
||||
try:
|
||||
await daemon_rpc(conf, 'get', uri)
|
||||
await daemon_rpc(conf, 'get', uri=uri, save_file=True)
|
||||
first_byte = time.time()
|
||||
first_byte_times.append(first_byte - start)
|
||||
print(f"{i + 1}/{len(resolvable)} - {first_byte - start} {uri}")
|
||||
downloaded, amount_downloaded, blobs_in_stream = await wait_for_done(conf, uri)
|
||||
if downloaded:
|
||||
download_successes.append(uri)
|
||||
else:
|
||||
download_failures.append(uri)
|
||||
mbs = round((blobs_in_stream * (MAX_BLOB_SIZE - 1)) / (time.time() - start) / 1000000, 2)
|
||||
download_speeds.append(mbs)
|
||||
print(f"downloaded {amount_downloaded}/{blobs_in_stream} blobs for {uri} at "
|
||||
f"{mbs}mb/s")
|
||||
except:
|
||||
print(f"{i + 1}/{len(uris)} - failed to start {uri}")
|
||||
if not cmd_args.head_blob_only:
|
||||
downloaded, amount_downloaded, blobs_in_stream = await wait_for_done(
|
||||
conf, uri, cmd_args.stall_download_timeout
|
||||
)
|
||||
if downloaded:
|
||||
download_successes.append(uri)
|
||||
else:
|
||||
download_failures.append(uri)
|
||||
mbs = round((blobs_in_stream * (MAX_BLOB_SIZE - 1)) / (time.time() - start) / 1000000, 2)
|
||||
download_speeds.append(mbs)
|
||||
print(f"downloaded {amount_downloaded}/{blobs_in_stream} blobs for {uri} at "
|
||||
f"{mbs}mb/s")
|
||||
except Exception as e:
|
||||
print(f"{i + 1}/{len(uris)} - failed to start {uri}: {e}")
|
||||
failed_to_start.append(uri)
|
||||
return
|
||||
# await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=parse_lbry_uri(uri).name)
|
||||
if cmd_args.exit_on_error:
|
||||
return
|
||||
if cmd_args.delete_after_download or cmd_args.head_blob_only:
|
||||
await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=parse_lbry_uri(uri).name)
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
print("**********************************************")
|
||||
@@ -139,12 +145,13 @@ async def main(uris=None, allow_fees=False):
|
||||
f"Best first byte time: {round(min(first_byte_times), 2)}\n" \
|
||||
f"95% confidence time-to-first-byte: {confidence(first_byte_times, 1.984)}s\n" \
|
||||
f"99% confidence time-to-first-byte: {confidence(first_byte_times, 2.626)}s\n" \
|
||||
f"Variance: {variance(first_byte_times)}\n" \
|
||||
f"Downloaded {len(download_successes)}/{len(resolvable)}\n" \
|
||||
f"Best stream download speed: {round(max(download_speeds), 2)}mb/s\n" \
|
||||
f"Worst stream download speed: {round(min(download_speeds), 2)}mb/s\n" \
|
||||
f"95% confidence download speed: {confidence(download_speeds, 1.984, False)}mb/s\n" \
|
||||
f"99% confidence download speed: {confidence(download_speeds, 2.626, False)}mb/s\n"
|
||||
f"Variance: {variance(first_byte_times)}\n"
|
||||
if not cmd_args.head_blob_only:
|
||||
result += f"Downloaded {len(download_successes)}/{len(resolvable)}\n" \
|
||||
f"Best stream download speed: {round(max(download_speeds), 2)}mb/s\n" \
|
||||
f"Worst stream download speed: {round(min(download_speeds), 2)}mb/s\n" \
|
||||
f"95% confidence download speed: {confidence(download_speeds, 1.984, False)}mb/s\n" \
|
||||
f"99% confidence download speed: {confidence(download_speeds, 2.626, False)}mb/s\n"
|
||||
|
||||
if failed_to_start:
|
||||
result += "\nFailed to start:" + "\n".join([f for f in failed_to_start])
|
||||
@@ -159,9 +166,12 @@ async def main(uris=None, allow_fees=False):
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--data_dir")
|
||||
parser.add_argument("--wallet_dir")
|
||||
parser.add_argument("--download_directory")
|
||||
#parser.add_argument("--data_dir")
|
||||
#parser.add_argument("--wallet_dir")
|
||||
#parser.add_argument("--download_directory")
|
||||
parser.add_argument("--allow_fees", action='store_true')
|
||||
args = parser.parse_args()
|
||||
asyncio.run(main(allow_fees=args.allow_fees))
|
||||
parser.add_argument("--exit_on_error", action='store_true')
|
||||
parser.add_argument("--stall_download_timeout", default=10, type=int)
|
||||
parser.add_argument("--delete_after_download", action='store_true')
|
||||
parser.add_argument("--head_blob_only", action='store_true')
|
||||
asyncio.run(main(cmd_args=parser.parse_args()))
|
||||
@@ -23,6 +23,7 @@ setup(
|
||||
'console_scripts': 'lbrynet=lbrynet.extras.cli:main'
|
||||
},
|
||||
install_requires=[
|
||||
'torba',
|
||||
'aiohttp==3.5.4',
|
||||
'aioupnp',
|
||||
'appdirs==1.4.3',
|
||||
@@ -35,7 +36,6 @@ setup(
|
||||
'protobuf==3.6.1',
|
||||
'msgpack==0.6.1',
|
||||
'ecdsa==0.13',
|
||||
'torba',
|
||||
'pyyaml==4.2b1',
|
||||
'docopt==0.6.2',
|
||||
'hachoir',
|
||||
|
||||
@@ -61,6 +61,7 @@ def mock_network_loop(loop: asyncio.BaseEventLoop):
|
||||
|
||||
protocol = proto_lam()
|
||||
transport = asyncio.DatagramTransport(extra={'socket': mock_sock})
|
||||
transport.is_closing = lambda: False
|
||||
transport.close = lambda: mock_sock.close()
|
||||
mock_sock.sendto = sendto
|
||||
transport.sendto = mock_sock.sendto
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from integration.testcase import CommandTestCase
|
||||
from lbrynet.testcase import CommandTestCase
|
||||
|
||||
|
||||
class AccountManagement(CommandTestCase):
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from integration.testcase import CommandTestCase
|
||||
from lbrynet.testcase import CommandTestCase
|
||||
|
||||
|
||||
class EpicAdventuresOfChris45(CommandTestCase):
|
||||
@@ -81,7 +81,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
|
||||
|
||||
# After some soul searching Chris decides that his story needs more
|
||||
# heart and a better ending. He takes down the story and begins the rewrite.
|
||||
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(claim_id, blocking=False))
|
||||
abandon = await self.out(self.daemon.jsonrpc_stream_abandon(claim_id, blocking=False))
|
||||
self.assertEqual(abandon['inputs'][0]['claim_id'], claim_id)
|
||||
await self.confirm_tx(abandon['txid'])
|
||||
|
||||
@@ -92,12 +92,12 @@ class EpicAdventuresOfChris45(CommandTestCase):
|
||||
# After abandoning he just waits for his LBCs to be returned to his account
|
||||
await self.generate(5)
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
self.assertEqual(result, '8.969379')
|
||||
self.assertEqual(result, '8.9693455')
|
||||
|
||||
# Amidst all this Chris receives a call from his friend Ramsey
|
||||
# who says that it is of utmost urgency that Chris transfer him
|
||||
# 1 LBC to which Chris readily obliges
|
||||
ramsey_account_id = (await self.daemon.jsonrpc_account_create("Ramsey"))['id']
|
||||
ramsey_account_id = (await self.out(self.daemon.jsonrpc_account_create("Ramsey")))['id']
|
||||
ramsey_address = await self.daemon.jsonrpc_address_unused(ramsey_account_id)
|
||||
result = await self.out(self.daemon.jsonrpc_account_send('1.0', ramsey_address))
|
||||
self.assertIn("txid", result)
|
||||
@@ -108,7 +108,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
|
||||
await self.generate(5)
|
||||
result = await self.daemon.jsonrpc_account_balance()
|
||||
# Chris' balance was correct
|
||||
self.assertEqual(result, '7.969255')
|
||||
self.assertEqual(result, '7.9692215')
|
||||
|
||||
# Ramsey too assured him that he had received the 1 LBC and thanks him
|
||||
result = await self.daemon.jsonrpc_account_balance(ramsey_account_id)
|
||||
@@ -183,7 +183,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
|
||||
|
||||
# But sadly Ramsey wasn't so pleased. It was hard for him to tell Chris...
|
||||
# Chris, though a bit heartbroken, abandoned the claim for now, but instantly started working on new hit lyrics
|
||||
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=tx['txid'], nout=0, blocking=False))
|
||||
abandon = await self.out(self.daemon.jsonrpc_stream_abandon(txid=tx['txid'], nout=0, blocking=False))
|
||||
self.assertTrue(abandon['inputs'][0]['txid'], tx['txid'])
|
||||
await self.confirm_tx(abandon['txid'])
|
||||
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import os.path
|
||||
import hashlib
|
||||
import tempfile
|
||||
import logging
|
||||
from binascii import unhexlify
|
||||
from urllib.request import urlopen
|
||||
|
||||
import ecdsa
|
||||
|
||||
@@ -8,10 +11,13 @@ from lbrynet.wallet.transaction import Transaction, Output
|
||||
from torba.client.errors import InsufficientFundsError
|
||||
from lbrynet.schema.compat import OldClaimMessage
|
||||
|
||||
from integration.testcase import CommandTestCase
|
||||
from lbrynet.testcase import CommandTestCase
|
||||
from torba.client.hash import sha256, Base58
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ChannelCommands(CommandTestCase):
|
||||
|
||||
async def test_create_channel_names(self):
|
||||
@@ -69,74 +75,65 @@ class ChannelCommands(CommandTestCase):
|
||||
|
||||
async def test_setting_channel_fields(self):
|
||||
values = {
|
||||
'tags': ["cool", "awesome"],
|
||||
'title': "Cool Channel",
|
||||
'description': "Best channel on LBRY.",
|
||||
'thumbnail_url': "https://co.ol/thumbnail.png",
|
||||
'language': "en",
|
||||
'contact_email': "human@email.com",
|
||||
'homepage_url': "https://co.ol",
|
||||
'tags': ["cool", "awesome"],
|
||||
'languages': ["en-US"],
|
||||
'locations': ['US::Manchester'],
|
||||
'email': "human@email.com",
|
||||
'website_url': "https://co.ol",
|
||||
'cover_url': "https://co.ol/cover.png",
|
||||
'featured': ['cafe']
|
||||
}
|
||||
fixed_values = values.copy()
|
||||
del fixed_values['language']
|
||||
fixed_values['languages'] = [{'language': 'en'}]
|
||||
fixed_values['thumbnail'] = {'url': fixed_values.pop('thumbnail_url')}
|
||||
fixed_values['locations'] = [{'country': 'US', 'city': 'Manchester'}]
|
||||
fixed_values['cover'] = {'url': fixed_values.pop('cover_url')}
|
||||
|
||||
# create new channel with all fields set
|
||||
tx = await self.out(self.channel_create('@bigchannel', **values))
|
||||
txo = tx['outputs'][0]
|
||||
self.assertEqual(
|
||||
txo['value']['channel'],
|
||||
{'public_key': txo['value']['channel']['public_key'], **fixed_values}
|
||||
)
|
||||
channel = tx['outputs'][0]['value']
|
||||
self.assertEqual(channel, {'public_key': channel['public_key'], **fixed_values})
|
||||
|
||||
# create channel with nothing set
|
||||
tx = await self.out(self.channel_create('@lightchannel'))
|
||||
txo = tx['outputs'][0]
|
||||
self.assertEqual(
|
||||
txo['value']['channel'],
|
||||
{'public_key': txo['value']['channel']['public_key']}
|
||||
)
|
||||
channel = tx['outputs'][0]['value']
|
||||
self.assertEqual(channel, {'public_key': channel['public_key']})
|
||||
|
||||
# create channel with just some tags
|
||||
tx = await self.out(self.channel_create('@updatedchannel', tags='blah'))
|
||||
# create channel with just a featured claim
|
||||
tx = await self.out(self.channel_create('@featurechannel', featured='beef'))
|
||||
txo = tx['outputs'][0]
|
||||
claim_id = txo['claim_id']
|
||||
public_key = txo['value']['channel']['public_key']
|
||||
self.assertEqual(
|
||||
txo['value']['channel'],
|
||||
{'public_key': public_key, 'tags': ['blah']}
|
||||
)
|
||||
claim_id, channel = txo['claim_id'], txo['value']
|
||||
fixed_values['public_key'] = channel['public_key']
|
||||
self.assertEqual(channel, {'public_key': fixed_values['public_key'], 'featured': ['beef']})
|
||||
|
||||
# update channel setting all fields
|
||||
tx = await self.out(self.channel_update(claim_id, **values))
|
||||
txo = tx['outputs'][0]
|
||||
fixed_values['public_key'] = public_key
|
||||
fixed_values['tags'].insert(0, 'blah') # existing tag
|
||||
self.assertEqual(
|
||||
txo['value']['channel'],
|
||||
fixed_values
|
||||
)
|
||||
channel = tx['outputs'][0]['value']
|
||||
fixed_values['featured'].insert(0, 'beef') # existing featured claim
|
||||
self.assertEqual(channel, fixed_values)
|
||||
|
||||
# clearing and settings tags
|
||||
tx = await self.out(self.channel_update(claim_id, tags='single', clear_tags=True))
|
||||
txo = tx['outputs'][0]
|
||||
fixed_values['tags'] = ['single']
|
||||
self.assertEqual(
|
||||
txo['value']['channel'],
|
||||
fixed_values
|
||||
)
|
||||
# clearing and settings featured content
|
||||
tx = await self.out(self.channel_update(claim_id, featured='beefcafe', clear_featured=True))
|
||||
channel = tx['outputs'][0]['value']
|
||||
fixed_values['featured'] = ['beefcafe']
|
||||
self.assertEqual(channel, fixed_values)
|
||||
|
||||
# reset signing key
|
||||
tx = await self.out(self.channel_update(claim_id, new_signing_key=True))
|
||||
txo = tx['outputs'][0]
|
||||
self.assertNotEqual(
|
||||
txo['value']['channel']['public_key'],
|
||||
fixed_values['public_key']
|
||||
channel = tx['outputs'][0]['value']
|
||||
self.assertNotEqual(channel['public_key'], fixed_values['public_key'])
|
||||
|
||||
# replace mode (clears everything except public_key)
|
||||
tx = await self.out(self.channel_update(claim_id, replace=True, title='foo', email='new@email.com'))
|
||||
self.assertEqual(
|
||||
tx['outputs'][0]['value'],
|
||||
{'public_key': channel['public_key'], 'title': 'foo', 'email': 'new@email.com'}
|
||||
)
|
||||
|
||||
# send channel to someone else
|
||||
new_account = await self.daemon.jsonrpc_account_create('second account')
|
||||
new_account = await self.out(self.daemon.jsonrpc_account_create('second account'))
|
||||
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
|
||||
|
||||
# before sending
|
||||
@@ -165,6 +162,19 @@ class ChannelCommands(CommandTestCase):
|
||||
|
||||
class StreamCommands(CommandTestCase):
|
||||
|
||||
files_directory = os.path.join(os.path.dirname(__file__), 'files')
|
||||
video_file_url = 'http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerEscapes.mp4'
|
||||
video_file_name = os.path.join(files_directory, 'ForBiggerEscapes.mp4')
|
||||
|
||||
def setUp(self):
|
||||
if not os.path.exists(self.video_file_name):
|
||||
if not os.path.exists(self.files_directory):
|
||||
os.mkdir(self.files_directory)
|
||||
log.info(f'downloading test video from {self.video_file_name}')
|
||||
with urlopen(self.video_file_url) as response,\
|
||||
open(self.video_file_name, 'wb') as video_file:
|
||||
video_file.write(response.read())
|
||||
|
||||
async def test_create_stream_names(self):
|
||||
# claim new name
|
||||
await self.stream_create('foo')
|
||||
@@ -204,24 +214,24 @@ class StreamCommands(CommandTestCase):
|
||||
tx = await self.stream_update(claim_id, bid='3.0')
|
||||
self.assertEqual(tx['outputs'][0]['amount'], '3.0')
|
||||
|
||||
await self.assertBalance(self.account, '6.993384')
|
||||
await self.assertBalance(self.account, '6.993319')
|
||||
|
||||
# not enough funds
|
||||
with self.assertRaisesRegex(
|
||||
InsufficientFundsError, "Not enough funds to cover this transaction."):
|
||||
await self.stream_create('foo2', '9.0')
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 1)
|
||||
await self.assertBalance(self.account, '6.993384')
|
||||
await self.assertBalance(self.account, '6.993319')
|
||||
|
||||
# spend exactly amount available, no change
|
||||
tx = await self.stream_create('foo3', '6.98527700')
|
||||
tx = await self.stream_create('foo3', '6.98523')
|
||||
await self.assertBalance(self.account, '0.0')
|
||||
self.assertEqual(len(tx['outputs']), 1) # no change
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 2)
|
||||
|
||||
async def test_publishing_checks_all_accounts_for_channel(self):
|
||||
account1_id, account1 = self.account.id, self.account
|
||||
new_account = await self.daemon.jsonrpc_account_create('second account')
|
||||
new_account = await self.out(self.daemon.jsonrpc_account_create('second account'))
|
||||
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
|
||||
|
||||
await self.out(self.channel_create('@spam', '1.0'))
|
||||
@@ -248,24 +258,66 @@ class StreamCommands(CommandTestCase):
|
||||
self.assertEqual(channels[0]['name'], '@baz')
|
||||
|
||||
# defaults to using all accounts to lookup channel
|
||||
await self.stream_create('hovercraft1', channel_id=baz_id)
|
||||
await self.stream_create('hovercraft1', '0.1', channel_id=baz_id)
|
||||
self.assertEqual((await self.claim_search('hovercraft1'))[0]['channel_name'], '@baz')
|
||||
# uses only the specific accounts which contains the channel
|
||||
await self.stream_create('hovercraft2', channel_id=baz_id, channel_account_id=[account2_id])
|
||||
# lookup by channel_name in all accounts
|
||||
await self.stream_create('hovercraft2', '0.1', channel_name='@baz')
|
||||
self.assertEqual((await self.claim_search('hovercraft2'))[0]['channel_name'], '@baz')
|
||||
# uses only the specific accounts which contains the channel
|
||||
await self.stream_create('hovercraft3', '0.1', channel_id=baz_id, channel_account_id=[account2_id])
|
||||
self.assertEqual((await self.claim_search('hovercraft3'))[0]['channel_name'], '@baz')
|
||||
# lookup by channel_name in specific account
|
||||
await self.stream_create('hovercraft4', '0.1', channel_name='@baz', channel_account_id=[account2_id])
|
||||
self.assertEqual((await self.claim_search('hovercraft4'))[0]['channel_name'], '@baz')
|
||||
# fails when specifying account which does not contain channel
|
||||
with self.assertRaisesRegex(ValueError, "Couldn't find channel with channel_id"):
|
||||
await self.stream_create(
|
||||
'hovercraft3', channel_id=baz_id, channel_account_id=[account1_id]
|
||||
'hovercraft5', '0.1', channel_id=baz_id, channel_account_id=[account1_id]
|
||||
)
|
||||
# fail with channel_name
|
||||
with self.assertRaisesRegex(ValueError, "Couldn't find channel with channel_name '@baz'"):
|
||||
await self.stream_create(
|
||||
'hovercraft5', '0.1', channel_name='@baz', channel_account_id=[account1_id]
|
||||
)
|
||||
|
||||
async def test_setting_claim_fields(self):
|
||||
async def test_preview_works_with_signed_streams(self):
|
||||
await self.out(self.channel_create('@spam', '1.0'))
|
||||
signed = await self.out(self.stream_create('bar', '1.0', channel_name='@spam', preview=True, confirm=False))
|
||||
self.assertTrue(signed['outputs'][0]['is_channel_signature_valid'])
|
||||
|
||||
async def test_publish_updates_file_list(self):
|
||||
tx = await self.out(self.stream_create(title='created'))
|
||||
txo = tx['outputs'][0]
|
||||
claim_id, expected = txo['claim_id'], txo['value']
|
||||
files = self.sout(self.daemon.jsonrpc_file_list())
|
||||
self.assertEqual(1, len(files))
|
||||
self.assertEqual(tx['txid'], files[0]['txid'])
|
||||
self.assertEqual(expected, files[0]['metadata'])
|
||||
|
||||
# update with metadata-only changes
|
||||
tx = await self.out(self.stream_update(claim_id, title='update 1'))
|
||||
files = self.sout(self.daemon.jsonrpc_file_list())
|
||||
expected['title'] = 'update 1'
|
||||
self.assertEqual(1, len(files))
|
||||
self.assertEqual(tx['txid'], files[0]['txid'])
|
||||
self.assertEqual(expected, files[0]['metadata'])
|
||||
|
||||
# update with new data
|
||||
tx = await self.out(self.stream_update(claim_id, title='update 2', data=b'updated data'))
|
||||
expected = tx['outputs'][0]['value']
|
||||
files = self.sout(self.daemon.jsonrpc_file_list())
|
||||
self.assertEqual(1, len(files))
|
||||
self.assertEqual(tx['txid'], files[0]['txid'])
|
||||
self.assertEqual(expected, files[0]['metadata'])
|
||||
|
||||
async def test_setting_stream_fields(self):
|
||||
values = {
|
||||
'tags': ["cool", "awesome"],
|
||||
'title': "Cool Content",
|
||||
'description': "Best content on LBRY.",
|
||||
'thumbnail_url': "https://co.ol/thumbnail.png",
|
||||
'language': "en",
|
||||
'tags': ["cool", "awesome"],
|
||||
'languages': ["en"],
|
||||
'locations': ['US:NH:Manchester:03101:42.990605:-71.460989'],
|
||||
|
||||
'author': "Jules Verne",
|
||||
'license': 'Public Domain',
|
||||
@@ -275,86 +327,253 @@ class StreamCommands(CommandTestCase):
|
||||
'fee_currency': 'usd',
|
||||
'fee_amount': '2.99',
|
||||
'fee_address': 'mmCsWAiXMUVecFQ3fVzUwvpT9XFMXno2Ca',
|
||||
|
||||
'video_width': 800,
|
||||
'video_height': 600
|
||||
}
|
||||
fixed_values = values.copy()
|
||||
del fixed_values['language']
|
||||
fixed_values['languages'] = [{'language': 'en'}]
|
||||
|
||||
# create new channel with all fields set
|
||||
tx = await self.out(self.stream_create('big', **values))
|
||||
txo = tx['outputs'][0]
|
||||
stream = txo['value']['stream']
|
||||
fixed_values['sd_hash'] = stream['sd_hash']
|
||||
fixed_values['file'] = stream['file']
|
||||
fixed_values['media_type'] = 'application/octet-stream'
|
||||
fixed_values['locations'] = [{
|
||||
'country': 'US',
|
||||
'state': 'NH',
|
||||
'city': 'Manchester',
|
||||
'code': '03101',
|
||||
'latitude': '42.990605',
|
||||
'longitude': '-71.460989'
|
||||
}]
|
||||
fixed_values['thumbnail'] = {'url': fixed_values.pop('thumbnail_url')}
|
||||
fixed_values['release_time'] = str(values['release_time'])
|
||||
fixed_values['source'] = {
|
||||
'hash': '56bf5dbae43f77a63d075b0f2ae9c7c3e3098db93779c7f9840da0f4db9c2f8c8454f4edd1373e2b64ee2e68350d916e',
|
||||
'media_type': 'application/octet-stream',
|
||||
'size': '3'
|
||||
}
|
||||
fixed_values['fee'] = {
|
||||
'address': fixed_values.pop('fee_address'),
|
||||
'amount': fixed_values.pop('fee_amount').replace('.', ''),
|
||||
'amount': fixed_values.pop('fee_amount'),
|
||||
'currency': fixed_values.pop('fee_currency').upper()
|
||||
}
|
||||
fixed_values['video'] = {
|
||||
'height': fixed_values.pop('video_height'),
|
||||
'width': fixed_values.pop('video_width')
|
||||
}
|
||||
|
||||
# create new stream with all fields set
|
||||
tx = await self.out(self.stream_create('big', **values))
|
||||
stream = tx['outputs'][0]['value']
|
||||
fixed_values['source']['name'] = stream['source']['name']
|
||||
fixed_values['source']['sd_hash'] = stream['source']['sd_hash']
|
||||
self.assertEqual(stream, fixed_values)
|
||||
|
||||
# create channel with nothing set
|
||||
# create stream with nothing set
|
||||
tx = await self.out(self.stream_create('light'))
|
||||
txo = tx['outputs'][0]
|
||||
stream = tx['outputs'][0]['value']
|
||||
self.assertEqual(
|
||||
txo['value']['stream'], {
|
||||
'file': {'size': '3'},
|
||||
'media_type': 'application/octet-stream',
|
||||
'sd_hash': txo['value']['stream']['sd_hash']
|
||||
stream, {
|
||||
'source': {
|
||||
'size': '3',
|
||||
'media_type': 'application/octet-stream',
|
||||
'name': stream['source']['name'],
|
||||
'hash': '56bf5dbae43f77a63d075b0f2ae9c7c3e3098db93779c7f9840da0f4db9c2f8c8454f4edd1373e2b64ee2e68350d916e',
|
||||
'sd_hash': stream['source']['sd_hash']
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
# create channel with just some tags
|
||||
tx = await self.out(self.stream_create('updated', tags='blah'))
|
||||
# create stream with just some tags, langs and locations
|
||||
tx = await self.out(self.stream_create('updated', tags='blah', languages='uk', locations='UA::Kyiv'))
|
||||
txo = tx['outputs'][0]
|
||||
claim_id = txo['claim_id']
|
||||
fixed_values['sd_hash'] = txo['value']['stream']['sd_hash']
|
||||
claim_id, stream = txo['claim_id'], txo['value']
|
||||
fixed_values['source']['name'] = stream['source']['name']
|
||||
fixed_values['source']['sd_hash'] = stream['source']['sd_hash']
|
||||
self.assertEqual(
|
||||
txo['value']['stream'], {
|
||||
'file': {'size': '3'},
|
||||
'media_type': 'application/octet-stream',
|
||||
'sd_hash': fixed_values['sd_hash'],
|
||||
'tags': ['blah']
|
||||
stream, {
|
||||
'source': {
|
||||
'size': '3',
|
||||
'media_type': 'application/octet-stream',
|
||||
'name': fixed_values['source']['name'],
|
||||
'hash': '56bf5dbae43f77a63d075b0f2ae9c7c3e3098db93779c7f9840da0f4db9c2f8c8454f4edd1373e2b64ee2e68350d916e',
|
||||
'sd_hash': fixed_values['source']['sd_hash'],
|
||||
},
|
||||
'tags': ['blah'],
|
||||
'languages': ['uk'],
|
||||
'locations': [{'country': 'UA', 'city': 'Kyiv'}]
|
||||
}
|
||||
)
|
||||
|
||||
# update channel setting all fields
|
||||
# update stream setting all fields, 'source' doesn't change
|
||||
tx = await self.out(self.stream_update(claim_id, **values))
|
||||
txo = tx['outputs'][0]
|
||||
stream = tx['outputs'][0]['value']
|
||||
fixed_values['tags'].insert(0, 'blah') # existing tag
|
||||
self.assertEqual(txo['value']['stream'], fixed_values)
|
||||
fixed_values['languages'].insert(0, 'uk') # existing language
|
||||
fixed_values['locations'].insert(0, {'country': 'UA', 'city': 'Kyiv'}) # existing location
|
||||
self.assertEqual(stream, fixed_values)
|
||||
|
||||
# clearing and settings tags
|
||||
tx = await self.out(self.stream_update(claim_id, tags='single', clear_tags=True))
|
||||
# clearing and settings tags, languages and locations
|
||||
tx = await self.out(self.stream_update(
|
||||
claim_id, tags='single', clear_tags=True,
|
||||
languages='pt', clear_languages=True,
|
||||
locations='BR', clear_locations=True,
|
||||
))
|
||||
txo = tx['outputs'][0]
|
||||
fixed_values['tags'] = ['single']
|
||||
self.assertEqual(txo['value']['stream'], fixed_values)
|
||||
fixed_values['languages'] = ['pt']
|
||||
fixed_values['locations'] = [{'country': 'BR'}]
|
||||
self.assertEqual(txo['value'], fixed_values)
|
||||
|
||||
# clearing fee
|
||||
tx = await self.out(self.stream_update(claim_id, clear_fee=True))
|
||||
txo = tx['outputs'][0]
|
||||
del fixed_values['fee']
|
||||
self.assertEqual(txo['value'], fixed_values)
|
||||
|
||||
# modifying hash/size/name
|
||||
fixed_values['source']['name'] = 'changed_name'
|
||||
fixed_values['source']['hash'] = 'cafebeef'
|
||||
fixed_values['source']['size'] = '42'
|
||||
tx = await self.out(self.stream_update(
|
||||
claim_id, file_name='changed_name', file_hash='cafebeef', file_size=42
|
||||
))
|
||||
self.assertEqual(tx['outputs'][0]['value'], fixed_values)
|
||||
|
||||
# stream_update re-signs with the same channel
|
||||
channel_id = (await self.channel_create('@chan'))['outputs'][0]['claim_id']
|
||||
tx = await self.stream_update(claim_id, channel_id=channel_id)
|
||||
self.assertEqual(tx['outputs'][0]['signing_channel']['name'], '@chan')
|
||||
tx = await self.stream_update(claim_id, title='channel re-signs')
|
||||
self.assertEqual(tx['outputs'][0]['value']['title'], 'channel re-signs')
|
||||
self.assertEqual(tx['outputs'][0]['signing_channel']['name'], '@chan')
|
||||
|
||||
# send claim to someone else
|
||||
new_account = await self.daemon.jsonrpc_account_create('second account')
|
||||
new_account = await self.out(self.daemon.jsonrpc_account_create('second account'))
|
||||
account2_id, account2 = new_account['id'], self.daemon.get_account_or_error(new_account['id'])
|
||||
|
||||
# before sending
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 3)
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 4)
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list(account_id=account2_id)), 0)
|
||||
|
||||
other_address = await account2.receiving.get_or_create_usable_address()
|
||||
tx = await self.out(self.stream_update(claim_id, claim_address=other_address))
|
||||
|
||||
# after sending
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 2)
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list()), 3)
|
||||
self.assertEqual(len(await self.daemon.jsonrpc_claim_list(account_id=account2_id)), 1)
|
||||
|
||||
async def test_create_update_and_abandon_claim(self):
|
||||
async def test_automatic_type_and_metadata_detection_for_image(self):
|
||||
with tempfile.NamedTemporaryFile(suffix='.png') as file:
|
||||
file.write(unhexlify(
|
||||
b'89504e470d0a1a0a0000000d49484452000000050000000708020000004fc'
|
||||
b'510b9000000097048597300000b1300000b1301009a9c1800000015494441'
|
||||
b'5408d763fcffff3f031260624005d4e603004c45030b5286e9ea000000004'
|
||||
b'9454e44ae426082'
|
||||
))
|
||||
file.flush()
|
||||
tx = await self.out(
|
||||
self.daemon.jsonrpc_stream_create(
|
||||
'blank-image', '1.0', file_path=file.name
|
||||
)
|
||||
)
|
||||
txo = tx['outputs'][0]
|
||||
self.assertEqual(
|
||||
txo['value'], {
|
||||
'source': {
|
||||
'size': '99',
|
||||
'name': os.path.basename(file.name),
|
||||
'media_type': 'image/png',
|
||||
'hash': '6c7df435d412c603390f593ef658c199817c7830ba3f16b7eadd8f99fa50e85dbd0d2b3dc61eadc33fe096e3872d1545',
|
||||
'sd_hash': txo['value']['source']['sd_hash'],
|
||||
},
|
||||
'stream_type': 'image',
|
||||
'image': {
|
||||
'width': 5,
|
||||
'height': 7
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
async def test_automatic_type_and_metadata_detection_for_video(self):
|
||||
tx = await self.out(
|
||||
self.daemon.jsonrpc_stream_create(
|
||||
'chrome', '1.0', file_path=self.video_file_name
|
||||
)
|
||||
)
|
||||
txo = tx['outputs'][0]
|
||||
self.assertEqual(
|
||||
txo['value'], {
|
||||
'source': {
|
||||
'size': '2299653',
|
||||
'name': 'ForBiggerEscapes.mp4',
|
||||
'media_type': 'video/mp4',
|
||||
'hash': '5f6811c83c1616df06f10bf5309ca61edb5ff949a9c1212ce784602d837bfdfc1c3db1e0580ef7bd1dadde41d8acf315',
|
||||
'sd_hash': txo['value']['source']['sd_hash'],
|
||||
},
|
||||
'stream_type': 'video',
|
||||
'video': {
|
||||
'width': 1280,
|
||||
'height': 720,
|
||||
'duration': 15
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
async def test_overriding_automatic_metadata_detection(self):
|
||||
tx = await self.out(
|
||||
self.daemon.jsonrpc_stream_create(
|
||||
'chrome', '1.0', file_path=self.video_file_name, width=99, height=88, duration=9
|
||||
)
|
||||
)
|
||||
txo = tx['outputs'][0]
|
||||
self.assertEqual(
|
||||
txo['value'], {
|
||||
'source': {
|
||||
'size': '2299653',
|
||||
'name': 'ForBiggerEscapes.mp4',
|
||||
'media_type': 'video/mp4',
|
||||
'hash': '5f6811c83c1616df06f10bf5309ca61edb5ff949a9c1212ce784602d837bfdfc1c3db1e0580ef7bd1dadde41d8acf315',
|
||||
'sd_hash': txo['value']['source']['sd_hash'],
|
||||
},
|
||||
'stream_type': 'video',
|
||||
'video': {
|
||||
'width': 99,
|
||||
'height': 88,
|
||||
'duration': 9
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
async def test_replace_mode_preserves_source_and_type(self):
|
||||
expected = {
|
||||
'tags': ['blah'],
|
||||
'languages': ['uk'],
|
||||
'locations': [{'country': 'UA', 'city': 'Kyiv'}],
|
||||
'source': {
|
||||
'size': '2299653',
|
||||
'name': 'ForBiggerEscapes.mp4',
|
||||
'media_type': 'video/mp4',
|
||||
'hash': '5f6811c83c1616df06f10bf5309ca61edb5ff949a9c1212ce784602d837bfdfc1c3db1e0580ef7bd1dadde41d8acf315',
|
||||
},
|
||||
'stream_type': 'video',
|
||||
'video': {
|
||||
'width': 1280,
|
||||
'height': 720,
|
||||
'duration': 15
|
||||
}
|
||||
}
|
||||
channel = await self.channel_create('@chan')
|
||||
tx = await self.out(self.daemon.jsonrpc_stream_create(
|
||||
'chrome', '1.0', file_path=self.video_file_name,
|
||||
tags='blah', languages='uk', locations='UA::Kyiv',
|
||||
channel_id=channel['outputs'][0]['claim_id']
|
||||
))
|
||||
await self.on_transaction_dict(tx)
|
||||
txo = tx['outputs'][0]
|
||||
expected['source']['sd_hash'] = txo['value']['source']['sd_hash']
|
||||
self.assertEqual(txo['value'], expected)
|
||||
self.assertEqual(txo['signing_channel']['name'], '@chan')
|
||||
tx = await self.out(self.daemon.jsonrpc_stream_update(
|
||||
txo['claim_id'], title='new title', replace=True
|
||||
))
|
||||
txo = tx['outputs'][0]
|
||||
expected['title'] = 'new title'
|
||||
del expected['tags']
|
||||
del expected['languages']
|
||||
del expected['locations']
|
||||
self.assertEqual(txo['value'], expected)
|
||||
self.assertNotIn('signing_channel', txo)
|
||||
|
||||
async def test_create_update_and_abandon_stream(self):
|
||||
await self.assertBalance(self.account, '10.0')
|
||||
|
||||
tx = await self.stream_create(bid='2.5') # creates new claim
|
||||
@@ -367,6 +586,10 @@ class StreamCommands(CommandTestCase):
|
||||
self.assertEqual(txs[0]['value'], '0.0')
|
||||
self.assertEqual(txs[0]['fee'], '-0.020107')
|
||||
await self.assertBalance(self.account, '7.479893')
|
||||
self.assertEqual(1, len(self.daemon.jsonrpc_file_list()))
|
||||
|
||||
await self.daemon.jsonrpc_file_delete(delete_all=True)
|
||||
self.assertEqual(0, len(self.daemon.jsonrpc_file_list()))
|
||||
|
||||
await self.stream_update(claim_id, bid='1.0') # updates previous claim
|
||||
txs = await self.out(self.daemon.jsonrpc_transaction_list())
|
||||
@@ -374,23 +597,23 @@ class StreamCommands(CommandTestCase):
|
||||
self.assertEqual(txs[0]['update_info'][0]['balance_delta'], '1.5')
|
||||
self.assertEqual(txs[0]['update_info'][0]['claim_id'], claim_id)
|
||||
self.assertEqual(txs[0]['value'], '0.0')
|
||||
self.assertEqual(txs[0]['fee'], '-0.000184')
|
||||
await self.assertBalance(self.account, '8.979709')
|
||||
self.assertEqual(txs[0]['fee'], '-0.0002165')
|
||||
await self.assertBalance(self.account, '8.9796765')
|
||||
|
||||
await self.claim_abandon(claim_id)
|
||||
await self.stream_abandon(claim_id)
|
||||
txs = await self.out(self.daemon.jsonrpc_transaction_list())
|
||||
self.assertEqual(len(txs[0]['abandon_info']), 1)
|
||||
self.assertEqual(txs[0]['abandon_info'][0]['balance_delta'], '1.0')
|
||||
self.assertEqual(txs[0]['abandon_info'][0]['claim_id'], claim_id)
|
||||
self.assertEqual(txs[0]['value'], '0.0')
|
||||
self.assertEqual(txs[0]['fee'], '-0.000107')
|
||||
await self.assertBalance(self.account, '9.979602')
|
||||
await self.assertBalance(self.account, '9.9795695')
|
||||
|
||||
async def test_abandoning_claim_at_loss(self):
|
||||
async def test_abandoning_stream_at_loss(self):
|
||||
await self.assertBalance(self.account, '10.0')
|
||||
tx = await self.stream_create(bid='0.0001')
|
||||
await self.assertBalance(self.account, '9.979793')
|
||||
await self.claim_abandon(tx['outputs'][0]['claim_id'])
|
||||
await self.stream_abandon(tx['outputs'][0]['claim_id'])
|
||||
await self.assertBalance(self.account, '9.97968399')
|
||||
|
||||
async def test_publish(self):
|
||||
@@ -408,8 +631,12 @@ class StreamCommands(CommandTestCase):
|
||||
file.flush()
|
||||
tx1 = await self.publish('foo', bid='1.0', file_path=file.name)
|
||||
|
||||
self.assertEqual(1, len(self.daemon.jsonrpc_file_list()))
|
||||
|
||||
# doesn't error on missing arguments when doing an update stream
|
||||
tx2 = await self.publish('foo', tags='updated')
|
||||
|
||||
self.assertEqual(1, len(self.daemon.jsonrpc_file_list()))
|
||||
self.assertEqual(
|
||||
tx1['outputs'][0]['claim_id'],
|
||||
tx2['outputs'][0]['claim_id']
|
||||
@@ -420,18 +647,29 @@ class StreamCommands(CommandTestCase):
|
||||
with self.assertRaisesRegex(Exception, "There are 2 claims for 'foo'"):
|
||||
await self.daemon.jsonrpc_publish('foo')
|
||||
|
||||
# abandon duplicate channel
|
||||
await self.claim_abandon(tx3['outputs'][0]['claim_id'])
|
||||
self.assertEqual(2, len(self.daemon.jsonrpc_file_list()))
|
||||
# abandon duplicate stream
|
||||
await self.stream_abandon(tx3['outputs'][0]['claim_id'])
|
||||
|
||||
# publish to a channel
|
||||
await self.channel_create('@abc')
|
||||
tx3 = await self.publish('foo', channel_name='@abc')
|
||||
self.assertEqual(2, len(self.daemon.jsonrpc_file_list()))
|
||||
r = await self.resolve('lbry://@abc/foo')
|
||||
self.assertEqual(
|
||||
r['lbry://@abc/foo']['claim']['claim_id'],
|
||||
tx3['outputs'][0]['claim_id']
|
||||
)
|
||||
|
||||
# publishing again clears channel
|
||||
tx4 = await self.publish('foo', languages='uk-UA')
|
||||
self.assertEqual(2, len(self.daemon.jsonrpc_file_list()))
|
||||
r = await self.resolve('lbry://foo')
|
||||
claim = r['lbry://foo']['claim']
|
||||
self.assertEqual(claim['txid'], tx4['outputs'][0]['txid'])
|
||||
self.assertNotIn('channel_name', claim)
|
||||
self.assertEqual(claim['value']['languages'], ['uk-UA'])
|
||||
|
||||
async def test_claim_search(self):
|
||||
# search for channel claim
|
||||
channel = await self.channel_create('@abc', '1.0')
|
||||
@@ -447,7 +685,7 @@ class StreamCommands(CommandTestCase):
|
||||
claims = await self.claim_search(claim_id=channel_id)
|
||||
self.assertEqual(claims[0]['value'], value)
|
||||
|
||||
await self.claim_abandon(txid=txid, nout=0)
|
||||
await self.channel_abandon(txid=txid, nout=0)
|
||||
self.assertEqual(len(await self.claim_search(txid=txid, nout=0)), 0)
|
||||
|
||||
# search stream claims
|
||||
@@ -469,10 +707,17 @@ class StreamCommands(CommandTestCase):
|
||||
|
||||
claims = await self.claim_search(channel_id=channel_id)
|
||||
self.assertEqual(len(claims), 3)
|
||||
# same is expected using name or name#claim_id urls
|
||||
claims = await self.claim_search(channel_name="@abc")
|
||||
self.assertEqual(len(claims), 3)
|
||||
claims = await self.claim_search(channel_name="@abc", channel_id=channel_id)
|
||||
self.assertEqual(len(claims), 3)
|
||||
claims = await self.claim_search(channel_name=f"@abc#{channel_id}")
|
||||
self.assertEqual(len(claims), 3)
|
||||
|
||||
await self.claim_abandon(claim_id=claims[0]['claim_id'])
|
||||
await self.claim_abandon(claim_id=claims[1]['claim_id'])
|
||||
await self.claim_abandon(claim_id=claims[2]['claim_id'])
|
||||
await self.stream_abandon(claim_id=claims[0]['claim_id'])
|
||||
await self.stream_abandon(claim_id=claims[1]['claim_id'])
|
||||
await self.stream_abandon(claim_id=claims[2]['claim_id'])
|
||||
|
||||
claims = await self.claim_search(channel_id=channel_id)
|
||||
self.assertEqual(len(claims), 0)
|
||||
@@ -507,7 +752,7 @@ class StreamCommands(CommandTestCase):
|
||||
async def test_abandoned_channel_with_signed_claims(self):
|
||||
channel = (await self.channel_create('@abc', '1.0'))['outputs'][0]
|
||||
orphan_claim = await self.stream_create('on-channel-claim', '0.0001', channel_id=channel['claim_id'])
|
||||
await self.claim_abandon(txid=channel['txid'], nout=0)
|
||||
await self.channel_abandon(txid=channel['txid'], nout=0)
|
||||
channel = (await self.channel_create('@abc', '1.0'))['outputs'][0]
|
||||
orphan_claim_id = orphan_claim['outputs'][0]['claim_id']
|
||||
|
||||
@@ -516,11 +761,11 @@ class StreamCommands(CommandTestCase):
|
||||
response = await self.resolve('lbry://@abc/on-channel-claim')
|
||||
self.assertNotIn('claim', response['lbry://@abc/on-channel-claim'])
|
||||
response = await self.resolve('lbry://on-channel-claim')
|
||||
self.assertFalse(response['lbry://on-channel-claim']['claim']['signature_is_valid'])
|
||||
self.assertIs(False, response['lbry://on-channel-claim']['claim']['signature_is_valid'])
|
||||
direct_uri = 'lbry://on-channel-claim#' + orphan_claim_id
|
||||
response = await self.resolve(direct_uri)
|
||||
self.assertFalse(response[direct_uri]['claim']['signature_is_valid'])
|
||||
await self.claim_abandon(claim_id=orphan_claim_id)
|
||||
self.assertIs(False, response[direct_uri]['claim']['signature_is_valid'])
|
||||
await self.stream_abandon(claim_id=orphan_claim_id)
|
||||
|
||||
uri = 'lbry://@abc/on-channel-claim'
|
||||
# now, claim something on this channel (it will update the invalid claim, but we save and forcefully restore)
|
||||
@@ -560,8 +805,7 @@ class StreamCommands(CommandTestCase):
|
||||
self.assertNotEqual(r1c, r2c)
|
||||
|
||||
await self.generate(50)
|
||||
head = await self.daemon.jsonrpc_block_show()
|
||||
self.assertTrue(head['height'] > 250)
|
||||
self.assertTrue(self.ledger.headers.height > 250)
|
||||
|
||||
r3 = await self.daemon.jsonrpc_resolve(urls='lbry://ΣίσυφοςfiÆ')
|
||||
r4 = await self.daemon.jsonrpc_resolve(urls='lbry://ΣΊΣΥΦΟσFIæ')
|
||||
@@ -580,7 +824,7 @@ class StreamCommands(CommandTestCase):
|
||||
channel = await self.daemon.jsonrpc_channel_create('@olds', '1.0')
|
||||
await self.confirm_tx(channel.id)
|
||||
address = channel.outputs[0].get_address(self.account.ledger)
|
||||
claim = generate_signed_legacy('example', address, channel.outputs[0])
|
||||
claim = generate_signed_legacy(address, channel.outputs[0])
|
||||
tx = await Transaction.claim_create('example', claim.SerializeToString(), 1, address, [self.account], self.account)
|
||||
await tx.sign([self.account])
|
||||
await self.broadcast(tx)
|
||||
@@ -598,12 +842,12 @@ class StreamCommands(CommandTestCase):
|
||||
await self.confirm_tx(tx.id)
|
||||
|
||||
response = await self.daemon.jsonrpc_resolve(urls='bad_example')
|
||||
self.assertFalse(response['bad_example']['claim']['signature_is_valid'], response)
|
||||
self.assertIs(False, response['bad_example']['claim']['signature_is_valid'], response)
|
||||
response = await self.daemon.jsonrpc_resolve(urls='@olds/bad_example')
|
||||
self.assertEqual('URI lbry://@olds/bad_example cannot be resolved', response['@olds/bad_example']['error'])
|
||||
|
||||
|
||||
def generate_signed_legacy(name: str, address: bytes, output: Output):
|
||||
def generate_signed_legacy(address: bytes, output: Output):
|
||||
decoded_address = Base58.decode(address)
|
||||
claim = OldClaimMessage()
|
||||
claim.ParseFromString(unhexlify(
|
||||
@@ -629,14 +873,14 @@ def generate_signed_legacy(name: str, address: bytes, output: Output):
|
||||
digest = sha256(b''.join([
|
||||
decoded_address,
|
||||
claim.SerializeToString(),
|
||||
output.claim_hash
|
||||
output.claim_hash[::-1]
|
||||
]))
|
||||
private_key = ecdsa.SigningKey.from_pem(output.private_key, hashfunc=hashlib.sha256)
|
||||
signature = private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
|
||||
claim.publisherSignature.version = 1
|
||||
claim.publisherSignature.signatureType = 1
|
||||
claim.publisherSignature.signature = signature
|
||||
claim.publisherSignature.certificateId = output.claim_hash
|
||||
claim.publisherSignature.certificateId = output.claim_hash[::-1]
|
||||
return claim
|
||||
|
||||
|
||||
@@ -644,7 +888,7 @@ class SupportCommands(CommandTestCase):
|
||||
|
||||
async def test_regular_supports_and_tip_supports(self):
|
||||
# account2 will be used to send tips and supports to account1
|
||||
account2_id = (await self.daemon.jsonrpc_account_create('second account'))['id']
|
||||
account2_id = (await self.out(self.daemon.jsonrpc_account_create('second account')))['id']
|
||||
account2 = self.daemon.get_account_or_error(account2_id)
|
||||
|
||||
# send account2 5 LBC out of the 10 LBC in account1
|
||||
|
||||
@@ -29,7 +29,7 @@ class CLIIntegrationTest(AsyncioTestCase):
|
||||
await self.daemon.start()
|
||||
|
||||
async def asyncTearDown(self):
|
||||
await self.daemon.stop()
|
||||
await self.daemon.stop(shutdown_runner=False)
|
||||
|
||||
def test_cli_status_command_with_auth(self):
|
||||
actual_output = StringIO()
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
import logging
|
||||
|
||||
import typing
|
||||
import random
|
||||
import asyncio
|
||||
from aiohttp import web
|
||||
|
||||
from lbrynet.testcase import CommandTestCase
|
||||
|
||||
import lbrynet.schema
|
||||
lbrynet.schema.BLOCKCHAIN_NAME = 'lbrycrd_regtest'
|
||||
|
||||
|
||||
class FakedCommentServer:
|
||||
ERRORS = {
|
||||
'INVALID_URI': {'code': 1, 'message': 'Invalid claim URI'},
|
||||
'INVALID_PARAMS': {'code': -32602, 'message': 'Invalid parameters'},
|
||||
'INTERNAL': {'code': -32603, 'message': 'An internal error'},
|
||||
'UNKNOWN': {'code': -1, 'message': 'An unknown or very miscellaneous error'},
|
||||
}
|
||||
|
||||
def __init__(self, port=2903):
|
||||
self.port = port
|
||||
self.app = web.Application(debug=True)
|
||||
self.app.add_routes([web.post('/api', self.api)])
|
||||
self.runner = None
|
||||
self.server = None
|
||||
|
||||
def get_claim_comments(self, uri: str, better_keys: bool) -> typing.Union[dict, list, None]:
|
||||
if not uri.startswith('lbry://'): # Very basic error case
|
||||
return {'error': self.ERRORS['INVALID_URI']}
|
||||
return [self.get_comment(i) for i in range(75)]
|
||||
|
||||
def get_comment(self, comment_id: int, parent_id: int = None) -> dict:
|
||||
return {
|
||||
'comment_id': comment_id,
|
||||
'parent_id': parent_id,
|
||||
'author': f'Person{comment_id}',
|
||||
'message': f'comment {comment_id}',
|
||||
'claim_id': random.randint(1, 2**16),
|
||||
'time_posted': random.randint(2**16, 2**32 - 1),
|
||||
'upvotes': random.randint(0, 9999), 'downvotes': random.randint(0, 9999)
|
||||
}
|
||||
|
||||
def comment(self, uri: str, poster: str, message: str) -> typing.Union[int, dict, None]:
|
||||
if not uri.startswith('lbry://'):
|
||||
return {'error': self.ERRORS['INVALID_URI']}
|
||||
return random.randint(1, 9999)
|
||||
|
||||
def reply(self, parent_id: int, poster: str, message: str) -> dict:
|
||||
if 2 <= len(message) <= 2000 and 2 <= len(poster) <= 127 and parent_id > 0:
|
||||
return random.randint(parent_id + 1, 2**32 - 1)
|
||||
return {'error': self.ERRORS['INVALID_PARAMS']}
|
||||
|
||||
def get_comment_data(self, comm_index: int, better_keys: bool = False) -> typing.Union[dict, None]:
|
||||
return self.get_comment(comm_index)
|
||||
|
||||
def get_comment_replies(self, comm_index: int) -> typing.Union[list, None]:
|
||||
return [random.randint(comm_index, comm_index+250) for _ in range(75)]
|
||||
|
||||
methods = {
|
||||
'get_claim_comments': get_claim_comments,
|
||||
'get_comment_data': get_comment_data,
|
||||
'get_comment_replies': get_comment_replies,
|
||||
'comment': comment,
|
||||
'reply': reply
|
||||
}
|
||||
|
||||
def process_json(self, body) -> dict:
|
||||
response = {'jsonrpc': '2.0', 'id': body['id']}
|
||||
if body['method'] in self.methods:
|
||||
params = body.get('params', {})
|
||||
result = self.methods[body['method']](self, **params)
|
||||
if type(result) is dict and 'error' in result:
|
||||
response['error'] = result['error']
|
||||
else:
|
||||
response['result'] = result
|
||||
else:
|
||||
response['error'] = self.ERRORS['UNKNOWN']
|
||||
return response
|
||||
|
||||
async def _start(self):
|
||||
self.runner = web.AppRunner(self.app)
|
||||
await self.runner.setup()
|
||||
self.server = web.TCPSite(self.runner, 'localhost', self.port)
|
||||
await self.server.start()
|
||||
|
||||
async def _stop(self):
|
||||
await self.runner.cleanup()
|
||||
|
||||
async def run(self, max_timeout=3600):
|
||||
try:
|
||||
await self._start()
|
||||
await asyncio.sleep(max_timeout)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
finally:
|
||||
await self._stop()
|
||||
|
||||
async def api(self, request):
|
||||
body = await request.json()
|
||||
if type(body) is list or type(body) is dict:
|
||||
if type(body) is list:
|
||||
response = [self.process_json(part) for part in body]
|
||||
else:
|
||||
response = self.process_json(body)
|
||||
return web.json_response(response)
|
||||
else:
|
||||
return web.json_response({'error': self.ERRORS['UNKNOWN']})
|
||||
|
||||
|
||||
class CommentCommands(CommandTestCase):
|
||||
|
||||
VERBOSITY = logging.WARN
|
||||
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
self.daemon.conf.comment_server = 'http://localhost:2903/api'
|
||||
self.server = FakedCommentServer(2903)
|
||||
self.server_task = asyncio.create_task(self.server.run(self.timeout))
|
||||
|
||||
async def asyncTearDown(self):
|
||||
await super().asyncTearDown()
|
||||
self.server_task.cancel()
|
||||
if not self.server_task.cancelled():
|
||||
await self.server_task
|
||||
|
||||
async def test_comment_create(self):
|
||||
claim = await self.stream_create(name='doge', bid='0.001', data=b'loool')
|
||||
self.assertIn('outputs', claim)
|
||||
comment = await self.daemon.jsonrpc_comment_create(
|
||||
claim_id=claim['outputs'][0]['claim_id'],
|
||||
channel_id='Jimmy Buffett',
|
||||
message="It's 5 O'Clock Somewhere"
|
||||
)
|
||||
self.assertIs(type(comment), dict, msg=f"Response type ({type(comment)})is not dict: {comment}")
|
||||
self.assertIn('message', comment, msg=f"Response {comment} doesn't contain message")
|
||||
self.assertIn('author', comment)
|
||||
|
||||
async def test_comment_create_reply(self):
|
||||
claim = await self.stream_create(name='doge', bid='0.001')
|
||||
self.assertIn('outputs', claim)
|
||||
reply = await self.daemon.jsonrpc_comment_create(
|
||||
claim_id=claim['outputs'][0]['claim_id'],
|
||||
channel_id='Jimmy Buffett',
|
||||
message='Let\'s all go to Margaritaville',
|
||||
parent_comment_id=42
|
||||
)
|
||||
self.assertIs(type(reply), dict, msg=f'Response {type(reply)} is not dict\nResponse: {reply}')
|
||||
self.assertIn('author', reply)
|
||||
|
||||
async def test_comment_list_root_level(self):
|
||||
claim = await self.stream_create(name='doge', bid='0.001')
|
||||
self.assertIn('outputs', claim)
|
||||
claim_id = claim['outputs'][0]['claim_id']
|
||||
comments = await self.daemon.jsonrpc_comment_list(claim_id)
|
||||
self.assertIsNotNone(type(comments))
|
||||
self.assertIs(type(comments), dict)
|
||||
self.assertIn('comments', comments, f"'comments' field was not found in returned dict: {comments}")
|
||||
self.assertIs(type(comments['comments']), list, msg=f'comment_list: {comments}')
|
||||
comments = await self.daemon.jsonrpc_comment_list(claim_id, page_size=50)
|
||||
self.assertIsNotNone(comments)
|
||||
self.assertIs(type(comments), dict)
|
||||
self.assertIn('comments', comments, f"'comments' field was not found in returned dict: {comments}")
|
||||
comment_list = comments['comments']
|
||||
self.assertEqual(len(comment_list), 50, msg=f'comment_list incorrect size {len(comment_list)}: {comment_list}')
|
||||
comments = await self.daemon.jsonrpc_comment_list(claim_id, page_size=50, page=2)
|
||||
self.assertEqual(len(comments['comments']), 25, msg=f'comment list page 2: {comments["comments"]}')
|
||||
comments = await self.daemon.jsonrpc_comment_list(claim_id, page_size=50, page=3)
|
||||
self.assertEqual(len(comments['comments']), 0, msg=f'comment list is non-zero: {comments["comments"]}')
|
||||
|
||||
async def test_comment_list_replies(self):
|
||||
claim = await self.stream_create(name='doge', bid='0.001')
|
||||
self.assertIn('outputs', claim)
|
||||
claim_id = claim['outputs'][0]['claim_id']
|
||||
replies = await self.daemon.jsonrpc_comment_list(claim_id, parent_comment_id=23)
|
||||
self.assertIsInstance(replies['comments'], list, msg=f'Invalid type: {replies["comments"]} should be list')
|
||||
self.assertGreater(len(replies['comments']), 0, msg='Returned replies are empty')
|
||||
replies = (await self.daemon.jsonrpc_comment_list(claim_id, parent_comment_id=25, page_size=50))['comments']
|
||||
self.assertEqual(len(replies), 50, f'Replies invalid length ({len(replies)})')
|
||||
replies = (await self.daemon.jsonrpc_comment_list(claim_id, parent_comment_id=67,
|
||||
page_size=23, page=5))['comments']
|
||||
self.assertEqual(len(replies), 0, f'replies {replies} not 23: {len(replies)}')
|
||||
replies = (await self.daemon.jsonrpc_comment_list(claim_id, parent_comment_id=79,
|
||||
page_size=60, page=2))['comments']
|
||||
self.assertEqual(len(replies), 15, f'Size of replies is incorrect, should be 15: {replies}')
|
||||
|
||||
async def test_comment_list_flatness_flatness_LA(self):
|
||||
claim = await self.stream_create(name='doge', bid='0.001')
|
||||
self.assertIn('outputs', claim)
|
||||
claim_id = claim['outputs'][0]['claim_id']
|
||||
replies = await self.daemon.jsonrpc_comment_list(claim_id, parent_comment_id=23, flat=True)
|
||||
self.assertIsInstance(replies['comments'], list, msg=f'Invalid type: {replies["comments"]} should be list')
|
||||
self.assertGreater(len(replies['comments']), 0, msg='Returned replies are empty')
|
||||
replies = (await self.daemon.jsonrpc_comment_list(claim_id, parent_comment_id=25, flat=True,
|
||||
max_replies_shown=0, page_size=50))['comments']
|
||||
self.assertEqual(len(replies), 50, f'Replies invalid length ({len(replies)})')
|
||||
replies = (await self.daemon.jsonrpc_comment_list(claim_id, parent_comment_id=67,
|
||||
flat=True, page_size=23, page=5))['comments']
|
||||
self.assertEqual(len(replies), 0, f'replies {replies} not 23: {len(replies)}')
|
||||
replies = (await self.daemon.jsonrpc_comment_list(claim_id, parent_comment_id=79,
|
||||
page_size=60, page=2))['comments']
|
||||
self.assertGreaterEqual(len(replies), 15, f'Size of replies is incorrect, should be 15: {replies}')
|
||||
@@ -1,8 +1,10 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from binascii import unhexlify, hexlify
|
||||
|
||||
from integration.testcase import CommandTestCase
|
||||
from lbrynet.schema import Claim
|
||||
from lbrynet.testcase import CommandTestCase
|
||||
from lbrynet.blob_exchange.downloader import BlobDownloader
|
||||
|
||||
|
||||
@@ -14,7 +16,7 @@ class FileCommands(CommandTestCase):
|
||||
await self.stream_create('foo', '0.01')
|
||||
await self.stream_create('foo2', '0.01')
|
||||
|
||||
file1, file2 = self.daemon.jsonrpc_file_list('claim_name')
|
||||
file1, file2 = self.sout(self.daemon.jsonrpc_file_list('claim_name'))
|
||||
self.assertEqual(file1['claim_name'], 'foo')
|
||||
self.assertEqual(file2['claim_name'], 'foo2')
|
||||
|
||||
@@ -26,30 +28,91 @@ class FileCommands(CommandTestCase):
|
||||
await self.daemon.jsonrpc_get('lbry://foo')
|
||||
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
|
||||
|
||||
async def test_announces(self):
|
||||
# announces on publish
|
||||
self.assertEqual(await self.daemon.storage.get_blobs_to_announce(), [])
|
||||
await self.stream_create('foo', '0.01')
|
||||
stream = self.daemon.jsonrpc_file_list()[0]
|
||||
self.assertSetEqual(
|
||||
set(await self.daemon.storage.get_blobs_to_announce()),
|
||||
{stream.sd_hash, stream.descriptor.blobs[0].blob_hash}
|
||||
)
|
||||
self.assertTrue(await self.daemon.jsonrpc_file_delete(delete_all=True))
|
||||
# announces on download
|
||||
self.assertEqual(await self.daemon.storage.get_blobs_to_announce(), [])
|
||||
stream = await self.daemon.jsonrpc_get('foo')
|
||||
self.assertSetEqual(
|
||||
set(await self.daemon.storage.get_blobs_to_announce()),
|
||||
{stream.sd_hash, stream.descriptor.blobs[0].blob_hash}
|
||||
)
|
||||
|
||||
async def test_file_list_fields(self):
|
||||
await self.stream_create('foo', '0.01')
|
||||
file_list = self.sout(self.daemon.jsonrpc_file_list())
|
||||
self.assertEqual(
|
||||
file_list[0]['timestamp'],
|
||||
None
|
||||
)
|
||||
self.assertEqual(file_list[0]['confirmations'], -1)
|
||||
await self.daemon.jsonrpc_resolve('foo')
|
||||
file_list = self.sout(self.daemon.jsonrpc_file_list())
|
||||
self.assertEqual(
|
||||
file_list[0]['timestamp'],
|
||||
self.ledger.headers[file_list[0]['height']]['timestamp']
|
||||
)
|
||||
self.assertEqual(file_list[0]['confirmations'], 1)
|
||||
|
||||
async def test_get_doesnt_touch_user_written_files_between_calls(self):
|
||||
await self.stream_create('foo', '0.01', data=bytes([0] * (2 << 23)))
|
||||
self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='foo'))
|
||||
first_path = (await self.daemon.jsonrpc_get('lbry://foo', save_file=True)).full_path
|
||||
await self.wait_files_to_complete()
|
||||
self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='foo'))
|
||||
with open(first_path, 'wb') as f:
|
||||
f.write(b' ')
|
||||
f.flush()
|
||||
second_path = await self.daemon.jsonrpc_get('lbry://foo', save_file=True)
|
||||
await self.wait_files_to_complete()
|
||||
self.assertNotEquals(first_path, second_path)
|
||||
|
||||
async def test_file_list_updated_metadata_on_resolve(self):
|
||||
await self.stream_create('foo', '0.01')
|
||||
claim = await self.daemon.resolve('lbry://foo')
|
||||
claim = claim['lbry://foo']['claim']['protobuf'].decode()
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo')
|
||||
txid = await self.blockchain_claim_name('bar', claim, '0.01')
|
||||
await self.daemon.jsonrpc_get('lbry://bar')
|
||||
claim = Claim.from_bytes(unhexlify(claim))
|
||||
claim.stream.description = "fix typos, fix the world"
|
||||
await self.blockchain_update_name(txid, hexlify(claim.to_bytes()).decode(), '0.01')
|
||||
await self.daemon.jsonrpc_resolve('lbry://bar')
|
||||
file_list = self.daemon.jsonrpc_file_list()
|
||||
self.assertEqual(file_list[0].stream_claim_info.claim.stream.description, claim.stream.description)
|
||||
|
||||
async def test_download_different_timeouts(self):
|
||||
tx = await self.stream_create('foo', '0.01')
|
||||
sd_hash = tx['outputs'][0]['value']['stream']['sd_hash']
|
||||
sd_hash = tx['outputs'][0]['value']['source']['sd_hash']
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo')
|
||||
all_except_sd = [
|
||||
blob_hash for blob_hash in self.server.blob_manager.completed_blob_hashes if blob_hash != sd_hash
|
||||
]
|
||||
await self.server.blob_manager.delete_blobs(all_except_sd)
|
||||
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2)
|
||||
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2, save_file=True)
|
||||
self.assertIn('error', resp)
|
||||
self.assertEquals('Failed to download data blobs for sd hash %s within timeout' % sd_hash, resp['error'])
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo')
|
||||
self.assertEqual('Failed to download data blobs for sd hash %s within timeout' % sd_hash, resp['error'])
|
||||
self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='foo'), "data timeout didnt create a file")
|
||||
await self.server.blob_manager.delete_blobs([sd_hash])
|
||||
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2)
|
||||
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2, save_file=True)
|
||||
self.assertIn('error', resp)
|
||||
self.assertEquals('Failed to download sd blob %s within timeout' % sd_hash, resp['error'])
|
||||
self.assertEqual('Failed to download sd blob %s within timeout' % sd_hash, resp['error'])
|
||||
|
||||
async def wait_files_to_complete(self):
|
||||
while self.daemon.jsonrpc_file_list(status='running'):
|
||||
while self.sout(self.daemon.jsonrpc_file_list(status='running')):
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
async def test_filename_conflicts_management_on_resume_download(self):
|
||||
await self.stream_create('foo', '0.01', data=bytes([0] * (1 << 23)))
|
||||
file_info = self.daemon.jsonrpc_file_list()[0]
|
||||
file_info = self.sout(self.daemon.jsonrpc_file_list())[0]
|
||||
original_path = os.path.join(self.daemon.conf.download_dir, file_info['file_name'])
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo')
|
||||
await self.daemon.jsonrpc_get('lbry://foo')
|
||||
@@ -59,43 +122,42 @@ class FileCommands(CommandTestCase):
|
||||
await self.daemon.stream_manager.start()
|
||||
await asyncio.wait_for(self.wait_files_to_complete(), timeout=5) # if this hangs, file didnt get set completed
|
||||
# check that internal state got through up to the file list API
|
||||
downloader = self.daemon.stream_manager.get_stream_by_stream_hash(file_info['stream_hash']).downloader
|
||||
file_info = self.daemon.jsonrpc_file_list()[0]
|
||||
self.assertEqual(downloader.output_file_name, file_info['file_name'])
|
||||
stream = self.daemon.stream_manager.get_stream_by_stream_hash(file_info['stream_hash'])
|
||||
file_info = self.sout(self.daemon.jsonrpc_file_list()[0])
|
||||
self.assertEqual(stream.file_name, file_info['file_name'])
|
||||
# checks if what the API shows is what he have at the very internal level.
|
||||
self.assertEqual(downloader.output_path, file_info['download_path'])
|
||||
# if you got here refactoring just change above, but ensure what gets set internally gets reflected externally!
|
||||
self.assertTrue(downloader.output_path.endswith(downloader.output_file_name))
|
||||
# this used to be inconsistent, if it becomes again it would create weird bugs, so worth checking
|
||||
self.assertEqual(stream.full_path, file_info['download_path'])
|
||||
|
||||
async def test_incomplete_downloads_erases_output_file_on_stop(self):
|
||||
tx = await self.stream_create('foo', '0.01')
|
||||
sd_hash = tx['outputs'][0]['value']['stream']['sd_hash']
|
||||
file_info = self.daemon.jsonrpc_file_list()[0]
|
||||
tx = await self.stream_create('foo', '0.01', data=b'deadbeef' * 1000000)
|
||||
sd_hash = tx['outputs'][0]['value']['source']['sd_hash']
|
||||
file_info = self.sout(self.daemon.jsonrpc_file_list())[0]
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo')
|
||||
blobs = await self.server_storage.get_blobs_for_stream(
|
||||
await self.server_storage.get_stream_hash_for_sd_hash(sd_hash)
|
||||
)
|
||||
all_except_sd_and_head = [
|
||||
blob.blob_hash for blob in blobs[1:] if blob.blob_hash
|
||||
blob.blob_hash for blob in blobs[1:-1]
|
||||
]
|
||||
await self.server.blob_manager.delete_blobs(all_except_sd_and_head)
|
||||
self.assertFalse(os.path.isfile(os.path.join(self.daemon.conf.download_dir, file_info['file_name'])))
|
||||
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2)
|
||||
path = os.path.join(self.daemon.conf.download_dir, file_info['file_name'])
|
||||
self.assertFalse(os.path.isfile(path))
|
||||
resp = await self.out(self.daemon.jsonrpc_get('lbry://foo', timeout=2))
|
||||
self.assertNotIn('error', resp)
|
||||
self.assertTrue(os.path.isfile(os.path.join(self.daemon.conf.download_dir, file_info['file_name'])))
|
||||
self.assertTrue(os.path.isfile(path))
|
||||
self.daemon.stream_manager.stop()
|
||||
self.assertFalse(os.path.isfile(os.path.join(self.daemon.conf.download_dir, file_info['file_name'])))
|
||||
await asyncio.sleep(0.01, loop=self.loop) # FIXME: this sleep should not be needed
|
||||
self.assertFalse(os.path.isfile(path))
|
||||
|
||||
async def test_incomplete_downloads_retry(self):
|
||||
tx = await self.stream_create('foo', '0.01')
|
||||
sd_hash = tx['outputs'][0]['value']['stream']['sd_hash']
|
||||
tx = await self.stream_create('foo', '0.01', data=b'deadbeef' * 1000000)
|
||||
sd_hash = tx['outputs'][0]['value']['source']['sd_hash']
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo')
|
||||
blobs = await self.server_storage.get_blobs_for_stream(
|
||||
await self.server_storage.get_stream_hash_for_sd_hash(sd_hash)
|
||||
)
|
||||
all_except_sd_and_head = [
|
||||
blob.blob_hash for blob in blobs[1:] if blob.blob_hash
|
||||
blob.blob_hash for blob in blobs[1:-1]
|
||||
]
|
||||
|
||||
# backup server blobs
|
||||
@@ -107,10 +169,10 @@ class FileCommands(CommandTestCase):
|
||||
await self.server.blob_manager.delete_blobs(all_except_sd_and_head)
|
||||
|
||||
# start the download
|
||||
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2)
|
||||
resp = await self.out(self.daemon.jsonrpc_get('lbry://foo', timeout=2))
|
||||
self.assertNotIn('error', resp)
|
||||
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
|
||||
self.assertEqual('running', self.daemon.jsonrpc_file_list()[0]['status'])
|
||||
self.assertEqual('running', self.sout(self.daemon.jsonrpc_file_list())[0]['status'])
|
||||
await self.daemon.jsonrpc_file_set_status('stop', claim_name='foo')
|
||||
|
||||
# recover blobs
|
||||
@@ -122,14 +184,14 @@ class FileCommands(CommandTestCase):
|
||||
|
||||
await self.daemon.jsonrpc_file_set_status('start', claim_name='foo')
|
||||
await asyncio.wait_for(self.wait_files_to_complete(), timeout=5)
|
||||
file_info = self.daemon.jsonrpc_file_list()[0]
|
||||
file_info = self.sout(self.daemon.jsonrpc_file_list())[0]
|
||||
self.assertEqual(file_info['blobs_completed'], file_info['blobs_in_stream'])
|
||||
self.assertEqual('finished', file_info['status'])
|
||||
|
||||
async def test_unban_recovers_stream(self):
|
||||
BlobDownloader.BAN_TIME = .5 # fixme: temporary field, will move to connection manager or a conf
|
||||
BlobDownloader.BAN_FACTOR = .5 # fixme: temporary field, will move to connection manager or a conf
|
||||
tx = await self.stream_create('foo', '0.01', data=bytes([0] * (1 << 23)))
|
||||
sd_hash = tx['outputs'][0]['value']['stream']['sd_hash']
|
||||
sd_hash = tx['outputs'][0]['value']['source']['sd_hash']
|
||||
missing_blob_hash = (await self.daemon.jsonrpc_blob_list(sd_hash=sd_hash))[-2]
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo')
|
||||
# backup blob
|
||||
@@ -143,7 +205,7 @@ class FileCommands(CommandTestCase):
|
||||
os.rename(missing_blob.file_path + '__', missing_blob.file_path)
|
||||
self.server_blob_manager.blobs.clear()
|
||||
missing_blob = self.server_blob_manager.get_blob(missing_blob_hash)
|
||||
await self.server_blob_manager.blob_completed(missing_blob)
|
||||
self.server_blob_manager.blob_completed(missing_blob)
|
||||
await asyncio.wait_for(self.wait_files_to_complete(), timeout=1)
|
||||
|
||||
async def test_paid_download(self):
|
||||
@@ -155,7 +217,7 @@ class FileCommands(CommandTestCase):
|
||||
fee_currency='LBC', fee_amount='11.0', fee_address=target_address
|
||||
)
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='expensive')
|
||||
response = await self.daemon.jsonrpc_get('lbry://expensive')
|
||||
response = await self.out(self.daemon.jsonrpc_get('lbry://expensive'))
|
||||
self.assertEqual(response['error'], 'fee of 11.00000 exceeds max available balance')
|
||||
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 0)
|
||||
|
||||
@@ -165,7 +227,7 @@ class FileCommands(CommandTestCase):
|
||||
fee_currency='LBC', fee_amount='111.0', fee_address=target_address
|
||||
)
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='maxkey')
|
||||
response = await self.daemon.jsonrpc_get('lbry://maxkey')
|
||||
response = await self.out(self.daemon.jsonrpc_get('lbry://maxkey'))
|
||||
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 0)
|
||||
self.assertEqual(response['error'], 'fee of 111.00000 exceeds max configured to allow of 50.00000')
|
||||
|
||||
@@ -177,8 +239,8 @@ class FileCommands(CommandTestCase):
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='icanpay')
|
||||
await self.assertBalance(self.account, '9.925679')
|
||||
response = await self.daemon.jsonrpc_get('lbry://icanpay')
|
||||
self.assertNotIn('error', response)
|
||||
await self.ledger.wait(response['tx'])
|
||||
raw_content_fee = response.content_fee.raw
|
||||
await self.ledger.wait(response.content_fee)
|
||||
await self.assertBalance(self.account, '8.925555')
|
||||
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
|
||||
|
||||
@@ -191,3 +253,10 @@ class FileCommands(CommandTestCase):
|
||||
self.assertEqual(
|
||||
await self.blockchain.get_balance(), starting_balance + block_reward_and_claim_fee
|
||||
)
|
||||
|
||||
# restart the daemon and make sure the fee is still there
|
||||
|
||||
self.daemon.stream_manager.stop()
|
||||
await self.daemon.stream_manager.start()
|
||||
self.assertEqual(len(self.daemon.jsonrpc_file_list()), 1)
|
||||
self.assertEqual(self.daemon.jsonrpc_file_list()[0].content_fee.raw, raw_content_fee)
|
||||
@@ -36,7 +36,7 @@ class BasicTransactionTest(IntegrationTestCase):
|
||||
channel_tx = await Transaction.create([], [channel_txo], [self.account], self.account)
|
||||
|
||||
stream = Claim()
|
||||
stream.stream.media_type = "video/mp4"
|
||||
stream.stream.source.media_type = "video/mp4"
|
||||
stream_txo = Output.pay_claim_name_pubkey_hash(
|
||||
l2d('1.0'), 'foo', stream, self.account.ledger.address_to_hash160(address1)
|
||||
)
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
from integration.testcase import CommandTestCase
|
||||
import json
|
||||
from binascii import hexlify
|
||||
from lbrynet.testcase import CommandTestCase
|
||||
|
||||
|
||||
class ResolveCommand(CommandTestCase):
|
||||
@@ -36,6 +38,14 @@ class ResolveCommand(CommandTestCase):
|
||||
self.assertEqual(claim['claim']['channel_name'], '@abc')
|
||||
self.assertEqual(claim['certificate']['name'], '@abc')
|
||||
self.assertEqual(claim['claims_in_channel'], 0)
|
||||
self.assertEqual(
|
||||
claim['claim']['timestamp'],
|
||||
self.ledger.headers[claim['claim']['height']]['timestamp']
|
||||
)
|
||||
self.assertEqual(
|
||||
claim['certificate']['timestamp'],
|
||||
self.ledger.headers[claim['certificate']['height']]['timestamp']
|
||||
)
|
||||
|
||||
# resolving claim foo by itself
|
||||
response = await self.resolve('lbry://foo')
|
||||
@@ -48,6 +58,10 @@ class ResolveCommand(CommandTestCase):
|
||||
self.assertEqual(claim['certificate']['name'], '@abc')
|
||||
self.assertEqual(claim['claims_in_channel'], 0)
|
||||
|
||||
# resolving from the given permanent url
|
||||
new_response = await self.resolve(claim['claim']['permanent_url'])
|
||||
self.assertEqual(new_response[claim['claim']['permanent_url']], claim)
|
||||
|
||||
# resolving multiple at once
|
||||
response = await self.resolve(['lbry://foo', 'lbry://foo2'])
|
||||
self.assertSetEqual({'lbry://foo', 'lbry://foo2'}, set(response))
|
||||
@@ -58,3 +72,19 @@ class ResolveCommand(CommandTestCase):
|
||||
self.assertEqual(claim['claim']['channel_name'], '@abc')
|
||||
self.assertEqual(claim['certificate']['name'], '@abc')
|
||||
self.assertEqual(claim['claims_in_channel'], 0)
|
||||
|
||||
# resolve has correct confirmations
|
||||
tx_details = await self.blockchain.get_raw_transaction(claim['claim']['txid'])
|
||||
self.assertEqual(claim['claim']['confirmations'], json.loads(tx_details)['confirmations'])
|
||||
|
||||
# resolve handles invalid data
|
||||
txid = await self.blockchain_claim_name(
|
||||
"gibberish", hexlify(b"{'invalid':'json'}").decode(), "0.1")
|
||||
response = await self.resolve("lbry://gibberish")
|
||||
self.assertSetEqual({'lbry://gibberish'}, set(response))
|
||||
claim = response['lbry://gibberish']['claim']
|
||||
self.assertEqual(claim['name'], 'gibberish')
|
||||
self.assertEqual(claim['protobuf'], hexlify(b"{'invalid':'json'}").decode())
|
||||
self.assertFalse(claim['decoded_claim'])
|
||||
self.assertEqual(claim['txid'], txid)
|
||||
self.assertEqual(claim['effective_amount'], "0.1")
|
||||
@@ -0,0 +1,366 @@
|
||||
import os
|
||||
import hashlib
|
||||
import aiohttp
|
||||
import aiohttp.web
|
||||
|
||||
from lbrynet.utils import aiohttp_request
|
||||
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
|
||||
from lbrynet.testcase import CommandTestCase
|
||||
|
||||
|
||||
def get_random_bytes(n: int) -> bytes:
|
||||
result = b''.join(hashlib.sha256(os.urandom(4)).digest() for _ in range(n // 16))
|
||||
if len(result) < n:
|
||||
result += os.urandom(n - len(result))
|
||||
elif len(result) > n:
|
||||
result = result[:-(len(result) - n)]
|
||||
assert len(result) == n, (n, len(result))
|
||||
return result
|
||||
|
||||
|
||||
class RangeRequests(CommandTestCase):
|
||||
async def _restart_stream_manager(self):
|
||||
self.daemon.stream_manager.stop()
|
||||
await self.daemon.stream_manager.start()
|
||||
return
|
||||
|
||||
async def _setup_stream(self, data: bytes, save_blobs: bool = True, save_files: bool = False, file_size=0):
|
||||
self.daemon.conf.save_blobs = save_blobs
|
||||
self.daemon.conf.save_files = save_files
|
||||
self.data = data
|
||||
await self.stream_create('foo', '0.01', data=self.data, file_size=file_size)
|
||||
if save_blobs:
|
||||
self.assertTrue(len(os.listdir(self.daemon.blob_manager.blob_dir)) > 1)
|
||||
await self.daemon.jsonrpc_file_list()[0].fully_reflected.wait()
|
||||
await self.daemon.jsonrpc_file_delete(delete_from_download_dir=True, claim_name='foo')
|
||||
self.assertEqual(0, len(os.listdir(self.daemon.blob_manager.blob_dir)))
|
||||
# await self._restart_stream_manager()
|
||||
await self.daemon.runner.setup()
|
||||
site = aiohttp.web.TCPSite(self.daemon.runner, self.daemon.conf.api_host, self.daemon.conf.api_port)
|
||||
await site.start()
|
||||
self.assertListEqual(self.daemon.jsonrpc_file_list(), [])
|
||||
|
||||
async def _test_range_requests(self):
|
||||
name = 'foo'
|
||||
url = f'http://{self.daemon.conf.api_host}:{self.daemon.conf.api_port}/get/{name}'
|
||||
|
||||
async with aiohttp_request('get', url) as req:
|
||||
self.assertEqual(req.headers.get('Content-Type'), 'application/octet-stream')
|
||||
content_range = req.headers.get('Content-Range')
|
||||
content_length = int(req.headers.get('Content-Length'))
|
||||
streamed_bytes = await req.content.read()
|
||||
self.assertEqual(content_length, len(streamed_bytes))
|
||||
return streamed_bytes, content_range, content_length
|
||||
|
||||
async def test_range_requests_2_byte(self):
|
||||
self.data = b'hi'
|
||||
await self._setup_stream(self.data)
|
||||
streamed, content_range, content_length = await self._test_range_requests()
|
||||
self.assertEqual(15, content_length)
|
||||
self.assertEqual(b'hi\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', streamed)
|
||||
self.assertEqual('bytes 0-14/15', content_range)
|
||||
|
||||
async def test_range_requests_15_byte(self):
|
||||
self.data = b'123456789abcdef'
|
||||
await self._setup_stream(self.data)
|
||||
streamed, content_range, content_length = await self._test_range_requests()
|
||||
self.assertEqual(15, content_length)
|
||||
self.assertEqual(15, len(streamed))
|
||||
self.assertEqual(self.data, streamed)
|
||||
self.assertEqual('bytes 0-14/15', content_range)
|
||||
|
||||
async def test_range_requests_0_padded_bytes(self, size: int = (MAX_BLOB_SIZE - 1) * 4,
|
||||
expected_range: str = 'bytes 0-8388603/8388604', padding=b'',
|
||||
file_size=0):
|
||||
self.data = get_random_bytes(size)
|
||||
await self._setup_stream(self.data, file_size=file_size)
|
||||
streamed, content_range, content_length = await self._test_range_requests()
|
||||
self.assertEqual(len(self.data + padding), content_length)
|
||||
self.assertEqual(streamed, self.data + padding)
|
||||
self.assertEqual(expected_range, content_range)
|
||||
|
||||
async def test_range_requests_1_padded_bytes(self):
|
||||
await self.test_range_requests_0_padded_bytes(
|
||||
((MAX_BLOB_SIZE - 1) * 4) - 1, padding=b'\x00'
|
||||
)
|
||||
|
||||
async def test_range_requests_2_padded_bytes(self):
|
||||
await self.test_range_requests_0_padded_bytes(
|
||||
((MAX_BLOB_SIZE - 1) * 4) - 2, padding=b'\x00' * 2
|
||||
)
|
||||
|
||||
async def test_range_requests_14_padded_bytes(self):
|
||||
await self.test_range_requests_0_padded_bytes(
|
||||
((MAX_BLOB_SIZE - 1) * 4) - 14, padding=b'\x00' * 14
|
||||
)
|
||||
|
||||
async def test_range_requests_no_padding_size_from_claim(self):
|
||||
size = ((MAX_BLOB_SIZE - 1) * 4) - 14
|
||||
await self.test_range_requests_0_padded_bytes(size, padding=b'', file_size=size,
|
||||
expected_range=f"bytes 0-{size-1}/{size}")
|
||||
|
||||
async def test_range_requests_15_padded_bytes(self):
|
||||
await self.test_range_requests_0_padded_bytes(
|
||||
((MAX_BLOB_SIZE - 1) * 4) - 15, padding=b'\x00' * 15
|
||||
)
|
||||
|
||||
async def test_range_requests_last_block_of_last_blob_padding(self):
|
||||
self.data = get_random_bytes(((MAX_BLOB_SIZE - 1) * 4) - 16)
|
||||
await self._setup_stream(self.data)
|
||||
streamed, content_range, content_length = await self._test_range_requests()
|
||||
self.assertEqual(len(self.data), content_length)
|
||||
self.assertEqual(streamed, self.data)
|
||||
self.assertEqual('bytes 0-8388587/8388588', content_range)
|
||||
|
||||
async def test_streaming_only_with_blobs(self):
|
||||
self.data = get_random_bytes((MAX_BLOB_SIZE - 1) * 4)
|
||||
await self._setup_stream(self.data)
|
||||
|
||||
await self._test_range_requests()
|
||||
stream = self.daemon.jsonrpc_file_list()[0]
|
||||
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
|
||||
self.assertIsNone(stream.download_directory)
|
||||
self.assertIsNone(stream.full_path)
|
||||
files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
|
||||
|
||||
# test that repeated range requests do not create duplicate files
|
||||
for _ in range(3):
|
||||
await self._test_range_requests()
|
||||
stream = self.daemon.jsonrpc_file_list()[0]
|
||||
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
|
||||
self.assertIsNone(stream.download_directory)
|
||||
self.assertIsNone(stream.full_path)
|
||||
current_files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
|
||||
self.assertEqual(
|
||||
len(files_in_download_dir), len(current_files_in_download_dir)
|
||||
)
|
||||
|
||||
# test that a range request after restart does not create a duplicate file
|
||||
await self._restart_stream_manager()
|
||||
|
||||
current_files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
|
||||
self.assertEqual(
|
||||
len(files_in_download_dir), len(current_files_in_download_dir)
|
||||
)
|
||||
stream = self.daemon.jsonrpc_file_list()[0]
|
||||
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
|
||||
self.assertIsNone(stream.download_directory)
|
||||
self.assertIsNone(stream.full_path)
|
||||
|
||||
await self._test_range_requests()
|
||||
stream = self.daemon.jsonrpc_file_list()[0]
|
||||
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
|
||||
self.assertIsNone(stream.download_directory)
|
||||
self.assertIsNone(stream.full_path)
|
||||
current_files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
|
||||
self.assertEqual(
|
||||
len(files_in_download_dir), len(current_files_in_download_dir)
|
||||
)
|
||||
|
||||
async def test_streaming_only_without_blobs(self):
|
||||
self.data = get_random_bytes((MAX_BLOB_SIZE - 1) * 4)
|
||||
await self._setup_stream(self.data, save_blobs=False)
|
||||
await self._test_range_requests()
|
||||
stream = self.daemon.jsonrpc_file_list()[0]
|
||||
self.assertIsNone(stream.download_directory)
|
||||
self.assertIsNone(stream.full_path)
|
||||
files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
|
||||
|
||||
# test that repeated range requests do not create duplicate files
|
||||
for _ in range(3):
|
||||
await self._test_range_requests()
|
||||
stream = self.daemon.jsonrpc_file_list()[0]
|
||||
self.assertIsNone(stream.download_directory)
|
||||
self.assertIsNone(stream.full_path)
|
||||
current_files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
|
||||
self.assertEqual(
|
||||
len(files_in_download_dir), len(current_files_in_download_dir)
|
||||
)
|
||||
|
||||
# test that a range request after restart does not create a duplicate file
|
||||
await self._restart_stream_manager()
|
||||
|
||||
current_files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
|
||||
self.assertEqual(
|
||||
len(files_in_download_dir), len(current_files_in_download_dir)
|
||||
)
|
||||
stream = self.daemon.jsonrpc_file_list()[0]
|
||||
self.assertIsNone(stream.download_directory)
|
||||
self.assertIsNone(stream.full_path)
|
||||
|
||||
await self._test_range_requests()
|
||||
stream = self.daemon.jsonrpc_file_list()[0]
|
||||
self.assertIsNone(stream.download_directory)
|
||||
self.assertIsNone(stream.full_path)
|
||||
current_files_in_download_dir = list(os.scandir(os.path.dirname(self.daemon.conf.data_dir)))
|
||||
self.assertEqual(
|
||||
len(files_in_download_dir), len(current_files_in_download_dir)
|
||||
)
|
||||
|
||||
async def test_stream_and_save_file_with_blobs(self):
|
||||
self.data = get_random_bytes((MAX_BLOB_SIZE - 1) * 4)
|
||||
await self._setup_stream(self.data, save_files=True)
|
||||
|
||||
await self._test_range_requests()
|
||||
streams = self.daemon.jsonrpc_file_list()
|
||||
self.assertEqual(1, len(streams))
|
||||
stream = streams[0]
|
||||
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
|
||||
self.assertTrue(os.path.isdir(stream.download_directory))
|
||||
self.assertTrue(os.path.isfile(stream.full_path))
|
||||
full_path = stream.full_path
|
||||
files_in_download_dir = list(os.scandir(os.path.dirname(full_path)))
|
||||
|
||||
for _ in range(3):
|
||||
await self._test_range_requests()
|
||||
streams = self.daemon.jsonrpc_file_list()
|
||||
self.assertEqual(1, len(streams))
|
||||
stream = streams[0]
|
||||
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
|
||||
self.assertTrue(os.path.isdir(stream.download_directory))
|
||||
self.assertTrue(os.path.isfile(stream.full_path))
|
||||
current_files_in_download_dir = list(os.scandir(os.path.dirname(full_path)))
|
||||
self.assertEqual(
|
||||
len(files_in_download_dir), len(current_files_in_download_dir)
|
||||
)
|
||||
|
||||
await self._restart_stream_manager()
|
||||
|
||||
current_files_in_download_dir = list(os.scandir(os.path.dirname(full_path)))
|
||||
self.assertEqual(
|
||||
len(files_in_download_dir), len(current_files_in_download_dir)
|
||||
)
|
||||
streams = self.daemon.jsonrpc_file_list()
|
||||
self.assertEqual(1, len(streams))
|
||||
stream = streams[0]
|
||||
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
|
||||
self.assertTrue(os.path.isdir(stream.download_directory))
|
||||
self.assertTrue(os.path.isfile(stream.full_path))
|
||||
|
||||
await self._test_range_requests()
|
||||
streams = self.daemon.jsonrpc_file_list()
|
||||
self.assertEqual(1, len(streams))
|
||||
stream = streams[0]
|
||||
self.assertTrue(os.path.isfile(self.daemon.blob_manager.get_blob(stream.sd_hash).file_path))
|
||||
self.assertTrue(os.path.isdir(stream.download_directory))
|
||||
self.assertTrue(os.path.isfile(stream.full_path))
|
||||
current_files_in_download_dir = list(os.scandir(os.path.dirname(full_path)))
|
||||
self.assertEqual(
|
||||
len(files_in_download_dir), len(current_files_in_download_dir)
|
||||
)
|
||||
with open(stream.full_path, 'rb') as f:
|
||||
self.assertEqual(self.data, f.read())
|
||||
|
||||
async def test_stream_and_save_file_without_blobs(self):
|
||||
self.data = get_random_bytes((MAX_BLOB_SIZE - 1) * 4)
|
||||
await self._setup_stream(self.data, save_files=True)
|
||||
self.daemon.conf.save_blobs = False
|
||||
|
||||
await self._test_range_requests()
|
||||
stream = self.daemon.jsonrpc_file_list()[0]
|
||||
self.assertTrue(os.path.isdir(stream.download_directory))
|
||||
self.assertTrue(os.path.isfile(stream.full_path))
|
||||
full_path = stream.full_path
|
||||
files_in_download_dir = list(os.scandir(os.path.dirname(full_path)))
|
||||
|
||||
for _ in range(3):
|
||||
await self._test_range_requests()
|
||||
stream = self.daemon.jsonrpc_file_list()[0]
|
||||
self.assertTrue(os.path.isdir(stream.download_directory))
|
||||
self.assertTrue(os.path.isfile(stream.full_path))
|
||||
current_files_in_download_dir = list(os.scandir(os.path.dirname(full_path)))
|
||||
self.assertEqual(
|
||||
len(files_in_download_dir), len(current_files_in_download_dir)
|
||||
)
|
||||
|
||||
await self._restart_stream_manager()
|
||||
current_files_in_download_dir = list(os.scandir(os.path.dirname(full_path)))
|
||||
self.assertEqual(
|
||||
len(files_in_download_dir), len(current_files_in_download_dir)
|
||||
)
|
||||
streams = self.daemon.jsonrpc_file_list()
|
||||
self.assertEqual(1, len(streams))
|
||||
stream = streams[0]
|
||||
self.assertTrue(os.path.isdir(stream.download_directory))
|
||||
self.assertTrue(os.path.isfile(stream.full_path))
|
||||
|
||||
await self._test_range_requests()
|
||||
streams = self.daemon.jsonrpc_file_list()
|
||||
self.assertEqual(1, len(streams))
|
||||
stream = streams[0]
|
||||
self.assertTrue(os.path.isdir(stream.download_directory))
|
||||
self.assertTrue(os.path.isfile(stream.full_path))
|
||||
current_files_in_download_dir = list(os.scandir(os.path.dirname(full_path)))
|
||||
self.assertEqual(
|
||||
len(files_in_download_dir), len(current_files_in_download_dir)
|
||||
)
|
||||
|
||||
with open(stream.full_path, 'rb') as f:
|
||||
self.assertEqual(self.data, f.read())
|
||||
|
||||
async def test_switch_save_blobs_while_running(self):
|
||||
await self.test_streaming_only_without_blobs()
|
||||
self.daemon.conf.save_blobs = True
|
||||
blobs_in_stream = self.daemon.jsonrpc_file_list()[0].blobs_in_stream
|
||||
sd_hash = self.daemon.jsonrpc_file_list()[0].sd_hash
|
||||
start_file_count = len(os.listdir(self.daemon.blob_manager.blob_dir))
|
||||
await self._test_range_requests()
|
||||
self.assertEqual(start_file_count + blobs_in_stream, len(os.listdir(self.daemon.blob_manager.blob_dir)))
|
||||
self.assertEqual(0, self.daemon.jsonrpc_file_list()[0].blobs_remaining)
|
||||
|
||||
# switch back
|
||||
self.daemon.conf.save_blobs = False
|
||||
await self._test_range_requests()
|
||||
self.assertEqual(start_file_count + blobs_in_stream, len(os.listdir(self.daemon.blob_manager.blob_dir)))
|
||||
self.assertEqual(0, self.daemon.jsonrpc_file_list()[0].blobs_remaining)
|
||||
await self.daemon.jsonrpc_file_delete(delete_from_download_dir=True, sd_hash=sd_hash)
|
||||
self.assertEqual(start_file_count, len(os.listdir(self.daemon.blob_manager.blob_dir)))
|
||||
await self._test_range_requests()
|
||||
self.assertEqual(start_file_count, len(os.listdir(self.daemon.blob_manager.blob_dir)))
|
||||
self.assertEqual(blobs_in_stream, self.daemon.jsonrpc_file_list()[0].blobs_remaining)
|
||||
|
||||
async def test_file_save_streaming_only_save_blobs(self):
|
||||
await self.test_streaming_only_with_blobs()
|
||||
stream = self.daemon.jsonrpc_file_list()[0]
|
||||
self.assertIsNone(stream.full_path)
|
||||
self.server.stop_server()
|
||||
await self.daemon.jsonrpc_file_save('test', self.daemon.conf.data_dir)
|
||||
stream = self.daemon.jsonrpc_file_list()[0]
|
||||
self.assertIsNotNone(stream.full_path)
|
||||
await stream.finished_writing.wait()
|
||||
with open(stream.full_path, 'rb') as f:
|
||||
self.assertEqual(self.data, f.read())
|
||||
await self.daemon.jsonrpc_file_delete(delete_from_download_dir=True, sd_hash=stream.sd_hash)
|
||||
|
||||
async def test_file_save_stop_before_finished_streaming_only(self, wait_for_start_writing=False):
|
||||
await self.test_streaming_only_with_blobs()
|
||||
stream = self.daemon.jsonrpc_file_list()[0]
|
||||
self.assertIsNone(stream.full_path)
|
||||
self.server.stop_server()
|
||||
await self.daemon.jsonrpc_file_save('test', self.daemon.conf.data_dir)
|
||||
stream = self.daemon.jsonrpc_file_list()[0]
|
||||
path = stream.full_path
|
||||
self.assertIsNotNone(path)
|
||||
if wait_for_start_writing:
|
||||
await stream.started_writing.wait()
|
||||
self.assertTrue(os.path.isfile(path))
|
||||
await self._restart_stream_manager()
|
||||
stream = self.daemon.jsonrpc_file_list()[0]
|
||||
self.assertIsNotNone(stream.full_path)
|
||||
self.assertFalse(os.path.isfile(path))
|
||||
if wait_for_start_writing:
|
||||
await stream.started_writing.wait()
|
||||
self.assertTrue(os.path.isfile(path))
|
||||
|
||||
async def test_file_save_stop_before_finished_streaming_only_wait_for_start(self):
|
||||
return await self.test_file_save_stop_before_finished_streaming_only(wait_for_start_writing=True)
|
||||
|
||||
async def test_file_save_streaming_only_dont_save_blobs(self):
|
||||
await self.test_streaming_only_without_blobs()
|
||||
stream = self.daemon.jsonrpc_file_list()[0]
|
||||
self.assertIsNone(stream.full_path)
|
||||
await self.daemon.jsonrpc_file_save('test', self.daemon.conf.data_dir)
|
||||
stream = self.daemon.jsonrpc_file_list()[0]
|
||||
await stream.finished_writing.wait()
|
||||
with open(stream.full_path, 'rb') as f:
|
||||
self.assertEqual(self.data, f.read())
|
||||
@@ -54,7 +54,7 @@ class AccountSynchronization(AsyncioTestCase):
|
||||
|
||||
async def asyncTearDown(self):
|
||||
self.wallet_component._running = False
|
||||
await self.daemon.stop()
|
||||
await self.daemon.stop(shutdown_runner=False)
|
||||
|
||||
@mock.patch('time.time', mock.Mock(return_value=12345))
|
||||
def test_sync(self):
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from integration.testcase import CommandTestCase
|
||||
from lbrynet.testcase import CommandTestCase
|
||||
|
||||
|
||||
class TransactionCommandsTestCase(CommandTestCase):
|
||||
|
||||
@@ -3,34 +3,223 @@ import tempfile
|
||||
import shutil
|
||||
import os
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from lbrynet.error import InvalidDataError, InvalidBlobHashError
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
from lbrynet.blob.blob_file import BlobFile, BlobBuffer, AbstractBlob
|
||||
|
||||
|
||||
class TestBlobfile(AsyncioTestCase):
|
||||
async def test_create_blob(self):
|
||||
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
|
||||
blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
|
||||
class TestBlob(AsyncioTestCase):
|
||||
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
|
||||
blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
|
||||
|
||||
async def asyncSetUp(self):
|
||||
self.tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(self.tmp_dir))
|
||||
self.loop = asyncio.get_running_loop()
|
||||
self.config = Config()
|
||||
self.storage = SQLiteStorage(self.config, ":memory:", self.loop)
|
||||
self.blob_manager = BlobManager(self.loop, self.tmp_dir, self.storage, self.config)
|
||||
await self.storage.open()
|
||||
|
||||
def _get_blob(self, blob_class=AbstractBlob, blob_directory=None):
|
||||
blob = blob_class(self.loop, self.blob_hash, len(self.blob_bytes), self.blob_manager.blob_completed,
|
||||
blob_directory=blob_directory)
|
||||
self.assertFalse(blob.get_is_verified())
|
||||
self.addCleanup(blob.close)
|
||||
return blob
|
||||
|
||||
async def _test_create_blob(self, blob_class=AbstractBlob, blob_directory=None):
|
||||
blob = self._get_blob(blob_class, blob_directory)
|
||||
writer = blob.get_blob_writer()
|
||||
writer.write(self.blob_bytes)
|
||||
await blob.verified.wait()
|
||||
self.assertTrue(blob.get_is_verified())
|
||||
await asyncio.sleep(0, loop=self.loop) # wait for the db save task
|
||||
return blob
|
||||
|
||||
async def _test_close_writers_on_finished(self, blob_class=AbstractBlob, blob_directory=None):
|
||||
blob = self._get_blob(blob_class, blob_directory=blob_directory)
|
||||
writers = [blob.get_blob_writer('1.2.3.4', port) for port in range(5)]
|
||||
self.assertEqual(5, len(blob.writers))
|
||||
|
||||
# test that writing too much causes the writer to fail with InvalidDataError and to be removed
|
||||
with self.assertRaises(InvalidDataError):
|
||||
writers[1].write(self.blob_bytes * 2)
|
||||
await writers[1].finished
|
||||
await asyncio.sleep(0, loop=self.loop)
|
||||
self.assertEqual(4, len(blob.writers))
|
||||
|
||||
# write the blob
|
||||
other = writers[2]
|
||||
writers[3].write(self.blob_bytes)
|
||||
await blob.verified.wait()
|
||||
|
||||
self.assertTrue(blob.get_is_verified())
|
||||
self.assertEqual(0, len(blob.writers))
|
||||
with self.assertRaises(IOError):
|
||||
other.write(self.blob_bytes)
|
||||
|
||||
def _test_ioerror_if_length_not_set(self, blob_class=AbstractBlob, blob_directory=None):
|
||||
blob = blob_class(
|
||||
self.loop, self.blob_hash, blob_completed_callback=self.blob_manager.blob_completed,
|
||||
blob_directory=blob_directory
|
||||
)
|
||||
self.addCleanup(blob.close)
|
||||
writer = blob.get_blob_writer()
|
||||
with self.assertRaises(IOError):
|
||||
writer.write(b'')
|
||||
|
||||
async def _test_invalid_blob_bytes(self, blob_class=AbstractBlob, blob_directory=None):
|
||||
blob = blob_class(
|
||||
self.loop, self.blob_hash, len(self.blob_bytes), blob_completed_callback=self.blob_manager.blob_completed,
|
||||
blob_directory=blob_directory
|
||||
)
|
||||
self.addCleanup(blob.close)
|
||||
writer = blob.get_blob_writer()
|
||||
writer.write(self.blob_bytes[:-4] + b'fake')
|
||||
with self.assertRaises(InvalidBlobHashError):
|
||||
await writer.finished
|
||||
|
||||
async def test_add_blob_buffer_to_db(self):
|
||||
blob = await self._test_create_blob(BlobBuffer)
|
||||
db_status = await self.storage.get_blob_status(blob.blob_hash)
|
||||
self.assertEqual(db_status, 'pending')
|
||||
|
||||
async def test_add_blob_file_to_db(self):
|
||||
blob = await self._test_create_blob(BlobFile, self.tmp_dir)
|
||||
db_status = await self.storage.get_blob_status(blob.blob_hash)
|
||||
self.assertEqual(db_status, 'finished')
|
||||
|
||||
async def test_invalid_blob_bytes(self):
|
||||
await self._test_invalid_blob_bytes(BlobBuffer)
|
||||
await self._test_invalid_blob_bytes(BlobFile, self.tmp_dir)
|
||||
|
||||
def test_ioerror_if_length_not_set(self):
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
|
||||
self._test_ioerror_if_length_not_set(BlobBuffer)
|
||||
self._test_ioerror_if_length_not_set(BlobFile, tmp_dir)
|
||||
|
||||
async def test_create_blob_file(self):
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
|
||||
blob = await self._test_create_blob(BlobFile, tmp_dir)
|
||||
self.assertIsInstance(blob, BlobFile)
|
||||
self.assertTrue(os.path.isfile(blob.file_path))
|
||||
|
||||
for _ in range(2):
|
||||
with blob.reader_context() as reader:
|
||||
self.assertEqual(self.blob_bytes, reader.read())
|
||||
|
||||
async def test_create_blob_buffer(self):
|
||||
blob = await self._test_create_blob(BlobBuffer)
|
||||
self.assertIsInstance(blob, BlobBuffer)
|
||||
self.assertIsNotNone(blob._verified_bytes)
|
||||
|
||||
# check we can only read the bytes once, and that the buffer is torn down
|
||||
with blob.reader_context() as reader:
|
||||
self.assertEqual(self.blob_bytes, reader.read())
|
||||
self.assertIsNone(blob._verified_bytes)
|
||||
with self.assertRaises(OSError):
|
||||
with blob.reader_context() as reader:
|
||||
self.assertEqual(self.blob_bytes, reader.read())
|
||||
self.assertIsNone(blob._verified_bytes)
|
||||
|
||||
async def test_close_writers_on_finished(self):
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
|
||||
await self._test_close_writers_on_finished(BlobBuffer)
|
||||
await self._test_close_writers_on_finished(BlobFile, tmp_dir)
|
||||
|
||||
async def test_concurrency_and_premature_closes(self):
|
||||
blob_directory = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(blob_directory))
|
||||
blob = self._get_blob(BlobBuffer, blob_directory=blob_directory)
|
||||
writer = blob.get_blob_writer('1.1.1.1', 1337)
|
||||
self.assertEqual(1, len(blob.writers))
|
||||
with self.assertRaises(OSError):
|
||||
blob.get_blob_writer('1.1.1.1', 1337)
|
||||
writer.close_handle()
|
||||
self.assertTrue(blob.writers[('1.1.1.1', 1337)].closed())
|
||||
writer = blob.get_blob_writer('1.1.1.1', 1337)
|
||||
self.assertEqual(blob.writers[('1.1.1.1', 1337)], writer)
|
||||
writer.close_handle()
|
||||
await asyncio.sleep(0.000000001) # flush callbacks
|
||||
self.assertEqual(0, len(blob.writers))
|
||||
|
||||
async def test_delete(self):
|
||||
blob_buffer = await self._test_create_blob(BlobBuffer)
|
||||
self.assertIsInstance(blob_buffer, BlobBuffer)
|
||||
self.assertIsNotNone(blob_buffer._verified_bytes)
|
||||
self.assertTrue(blob_buffer.get_is_verified())
|
||||
blob_buffer.delete()
|
||||
self.assertIsNone(blob_buffer._verified_bytes)
|
||||
self.assertFalse(blob_buffer.get_is_verified())
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
|
||||
|
||||
storage = SQLiteStorage(Config(), os.path.join(tmp_dir, "lbrynet.sqlite"))
|
||||
blob_manager = BlobFileManager(loop, tmp_dir, storage)
|
||||
blob_file = await self._test_create_blob(BlobFile, tmp_dir)
|
||||
self.assertIsInstance(blob_file, BlobFile)
|
||||
self.assertTrue(os.path.isfile(blob_file.file_path))
|
||||
self.assertTrue(blob_file.get_is_verified())
|
||||
blob_file.delete()
|
||||
self.assertFalse(os.path.isfile(blob_file.file_path))
|
||||
self.assertFalse(blob_file.get_is_verified())
|
||||
|
||||
await storage.open()
|
||||
await blob_manager.setup()
|
||||
async def test_delete_corrupt(self):
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
|
||||
blob = BlobFile(
|
||||
self.loop, self.blob_hash, len(self.blob_bytes), blob_completed_callback=self.blob_manager.blob_completed,
|
||||
blob_directory=tmp_dir
|
||||
)
|
||||
writer = blob.get_blob_writer()
|
||||
writer.write(self.blob_bytes)
|
||||
await blob.verified.wait()
|
||||
blob.close()
|
||||
blob = BlobFile(
|
||||
self.loop, self.blob_hash, len(self.blob_bytes), blob_completed_callback=self.blob_manager.blob_completed,
|
||||
blob_directory=tmp_dir
|
||||
)
|
||||
self.assertTrue(blob.get_is_verified())
|
||||
|
||||
# add the blob on the server
|
||||
blob = blob_manager.get_blob(blob_hash, len(blob_bytes))
|
||||
self.assertEqual(blob.get_is_verified(), False)
|
||||
self.assertNotIn(blob_hash, blob_manager.completed_blob_hashes)
|
||||
with open(blob.file_path, 'wb+') as f:
|
||||
f.write(b'\x00')
|
||||
blob = BlobFile(
|
||||
self.loop, self.blob_hash, len(self.blob_bytes), blob_completed_callback=self.blob_manager.blob_completed,
|
||||
blob_directory=tmp_dir
|
||||
)
|
||||
self.assertFalse(blob.get_is_verified())
|
||||
self.assertFalse(os.path.isfile(blob.file_path))
|
||||
|
||||
writer = blob.open_for_writing()
|
||||
writer.write(blob_bytes)
|
||||
await blob.finished_writing.wait()
|
||||
self.assertTrue(os.path.isfile(blob.file_path), True)
|
||||
self.assertEqual(blob.get_is_verified(), True)
|
||||
self.assertIn(blob_hash, blob_manager.completed_blob_hashes)
|
||||
def test_invalid_blob_hash(self):
|
||||
self.assertRaises(InvalidBlobHashError, BlobBuffer, self.loop, '', len(self.blob_bytes))
|
||||
self.assertRaises(InvalidBlobHashError, BlobBuffer, self.loop, 'x' * 96, len(self.blob_bytes))
|
||||
self.assertRaises(InvalidBlobHashError, BlobBuffer, self.loop, 'a' * 97, len(self.blob_bytes))
|
||||
|
||||
async def _test_close_reader(self, blob_class=AbstractBlob, blob_directory=None):
|
||||
blob = await self._test_create_blob(blob_class, blob_directory)
|
||||
reader = blob.reader_context()
|
||||
self.assertEqual(0, len(blob.readers))
|
||||
|
||||
async def read_blob_buffer():
|
||||
with reader as read_handle:
|
||||
self.assertEqual(1, len(blob.readers))
|
||||
await asyncio.sleep(2, loop=self.loop)
|
||||
self.assertEqual(0, len(blob.readers))
|
||||
return read_handle.read()
|
||||
|
||||
self.loop.call_later(1, blob.close)
|
||||
with self.assertRaises(ValueError) as err:
|
||||
read_task = self.loop.create_task(read_blob_buffer())
|
||||
await read_task
|
||||
self.assertEqual(err.exception, ValueError("I/O operation on closed file"))
|
||||
|
||||
async def test_close_reader(self):
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
|
||||
await self._test_close_reader(BlobBuffer)
|
||||
await self._test_close_reader(BlobFile, tmp_dir)
|
||||
@@ -5,50 +5,53 @@ import os
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
|
||||
|
||||
class TestBlobManager(AsyncioTestCase):
|
||||
async def test_sync_blob_manager_on_startup(self):
|
||||
loop = asyncio.get_event_loop()
|
||||
async def setup_blob_manager(self, save_blobs=True):
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
|
||||
self.config = Config(save_blobs=save_blobs)
|
||||
self.storage = SQLiteStorage(self.config, os.path.join(tmp_dir, "lbrynet.sqlite"))
|
||||
self.blob_manager = BlobManager(self.loop, tmp_dir, self.storage, self.config)
|
||||
await self.storage.open()
|
||||
|
||||
storage = SQLiteStorage(Config(), os.path.join(tmp_dir, "lbrynet.sqlite"))
|
||||
blob_manager = BlobFileManager(loop, tmp_dir, storage)
|
||||
async def test_sync_blob_file_manager_on_startup(self):
|
||||
await self.setup_blob_manager(save_blobs=True)
|
||||
|
||||
# add a blob file
|
||||
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
|
||||
blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
|
||||
with open(os.path.join(blob_manager.blob_dir, blob_hash), 'wb') as f:
|
||||
with open(os.path.join(self.blob_manager.blob_dir, blob_hash), 'wb') as f:
|
||||
f.write(blob_bytes)
|
||||
|
||||
# it should not have been added automatically on startup
|
||||
await storage.open()
|
||||
await blob_manager.setup()
|
||||
self.assertSetEqual(blob_manager.completed_blob_hashes, set())
|
||||
|
||||
await self.blob_manager.setup()
|
||||
self.assertSetEqual(self.blob_manager.completed_blob_hashes, set())
|
||||
|
||||
# make sure we can add the blob
|
||||
await blob_manager.blob_completed(blob_manager.get_blob(blob_hash, len(blob_bytes)))
|
||||
self.assertSetEqual(blob_manager.completed_blob_hashes, {blob_hash})
|
||||
await self.blob_manager.blob_completed(self.blob_manager.get_blob(blob_hash, len(blob_bytes)))
|
||||
self.assertSetEqual(self.blob_manager.completed_blob_hashes, {blob_hash})
|
||||
|
||||
# stop the blob manager and restart it, make sure the blob is there
|
||||
blob_manager.stop()
|
||||
self.assertSetEqual(blob_manager.completed_blob_hashes, set())
|
||||
await blob_manager.setup()
|
||||
self.assertSetEqual(blob_manager.completed_blob_hashes, {blob_hash})
|
||||
self.blob_manager.stop()
|
||||
self.assertSetEqual(self.blob_manager.completed_blob_hashes, set())
|
||||
await self.blob_manager.setup()
|
||||
self.assertSetEqual(self.blob_manager.completed_blob_hashes, {blob_hash})
|
||||
|
||||
# test that the blob is removed upon the next startup after the file being manually deleted
|
||||
blob_manager.stop()
|
||||
self.blob_manager.stop()
|
||||
|
||||
# manually delete the blob file and restart the blob manager
|
||||
os.remove(os.path.join(blob_manager.blob_dir, blob_hash))
|
||||
await blob_manager.setup()
|
||||
self.assertSetEqual(blob_manager.completed_blob_hashes, set())
|
||||
os.remove(os.path.join(self.blob_manager.blob_dir, blob_hash))
|
||||
await self.blob_manager.setup()
|
||||
self.assertSetEqual(self.blob_manager.completed_blob_hashes, set())
|
||||
|
||||
# check that the deleted blob was updated in the database
|
||||
self.assertEqual(
|
||||
'pending', (
|
||||
await storage.run_and_return_one_or_none('select status from blob where blob_hash=?', blob_hash)
|
||||
await self.storage.run_and_return_one_or_none('select status from blob where blob_hash=?', blob_hash)
|
||||
)
|
||||
)
|
||||
@@ -9,9 +9,9 @@ from lbrynet.blob_exchange.serialization import BlobRequest
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
from lbrynet.blob_exchange.server import BlobServer, BlobServerProtocol
|
||||
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
|
||||
from lbrynet.blob_exchange.client import request_blob
|
||||
from lbrynet.dht.peer import KademliaPeer, PeerManager
|
||||
|
||||
# import logging
|
||||
@@ -35,13 +35,13 @@ class BlobExchangeTestBase(AsyncioTestCase):
|
||||
self.server_config = Config(data_dir=self.server_dir, download_dir=self.server_dir, wallet=self.server_dir,
|
||||
reflector_servers=[])
|
||||
self.server_storage = SQLiteStorage(self.server_config, os.path.join(self.server_dir, "lbrynet.sqlite"))
|
||||
self.server_blob_manager = BlobFileManager(self.loop, self.server_dir, self.server_storage)
|
||||
self.server_blob_manager = BlobManager(self.loop, self.server_dir, self.server_storage, self.server_config)
|
||||
self.server = BlobServer(self.loop, self.server_blob_manager, 'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP')
|
||||
|
||||
self.client_config = Config(data_dir=self.client_dir, download_dir=self.client_dir, wallet=self.client_dir,
|
||||
reflector_servers=[])
|
||||
self.client_storage = SQLiteStorage(self.client_config, os.path.join(self.client_dir, "lbrynet.sqlite"))
|
||||
self.client_blob_manager = BlobFileManager(self.loop, self.client_dir, self.client_storage)
|
||||
self.client_blob_manager = BlobManager(self.loop, self.client_dir, self.client_storage, self.client_config)
|
||||
self.client_peer_manager = PeerManager(self.loop)
|
||||
self.server_from_client = KademliaPeer(self.loop, "127.0.0.1", b'1' * 48, tcp_port=33333)
|
||||
|
||||
@@ -51,6 +51,7 @@ class BlobExchangeTestBase(AsyncioTestCase):
|
||||
await self.server_blob_manager.setup()
|
||||
|
||||
self.server.start_server(33333, '127.0.0.1')
|
||||
self.addCleanup(self.server.stop_server)
|
||||
await self.server.started_listening.wait()
|
||||
|
||||
|
||||
@@ -58,21 +59,25 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
async def _add_blob_to_server(self, blob_hash: str, blob_bytes: bytes):
|
||||
# add the blob on the server
|
||||
server_blob = self.server_blob_manager.get_blob(blob_hash, len(blob_bytes))
|
||||
writer = server_blob.open_for_writing()
|
||||
writer = server_blob.get_blob_writer()
|
||||
writer.write(blob_bytes)
|
||||
await server_blob.finished_writing.wait()
|
||||
await server_blob.verified.wait()
|
||||
self.assertTrue(os.path.isfile(server_blob.file_path))
|
||||
self.assertEqual(server_blob.get_is_verified(), True)
|
||||
self.assertTrue(writer.closed())
|
||||
|
||||
async def _test_transfer_blob(self, blob_hash: str):
|
||||
client_blob = self.client_blob_manager.get_blob(blob_hash)
|
||||
|
||||
# download the blob
|
||||
downloaded = await request_blob(self.loop, client_blob, self.server_from_client.address,
|
||||
self.server_from_client.tcp_port, 2, 3)
|
||||
await client_blob.finished_writing.wait()
|
||||
downloaded, transport = await request_blob(self.loop, client_blob, self.server_from_client.address,
|
||||
self.server_from_client.tcp_port, 2, 3)
|
||||
self.assertIsNotNone(transport)
|
||||
self.addCleanup(transport.close)
|
||||
await client_blob.verified.wait()
|
||||
self.assertEqual(client_blob.get_is_verified(), True)
|
||||
self.assertTrue(downloaded)
|
||||
client_blob.close()
|
||||
|
||||
async def test_transfer_sd_blob(self):
|
||||
sd_hash = "3e2706157a59aaa47ef52bc264fce488078b4026c0b9bab649a8f2fe1ecc5e5cad7182a2bb7722460f856831a1ac0f02"
|
||||
@@ -92,9 +97,11 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
|
||||
second_client_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, second_client_dir)
|
||||
|
||||
second_client_storage = SQLiteStorage(Config(), os.path.join(second_client_dir, "lbrynet.sqlite"))
|
||||
second_client_blob_manager = BlobFileManager(self.loop, second_client_dir, second_client_storage)
|
||||
second_client_conf = Config()
|
||||
second_client_storage = SQLiteStorage(second_client_conf, os.path.join(second_client_dir, "lbrynet.sqlite"))
|
||||
second_client_blob_manager = BlobManager(
|
||||
self.loop, second_client_dir, second_client_storage, second_client_conf
|
||||
)
|
||||
server_from_second_client = KademliaPeer(self.loop, "127.0.0.1", b'1' * 48, tcp_port=33333)
|
||||
|
||||
await second_client_storage.open()
|
||||
@@ -102,7 +109,7 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
|
||||
await self._add_blob_to_server(blob_hash, mock_blob_bytes)
|
||||
|
||||
second_client_blob = self.client_blob_manager.get_blob(blob_hash)
|
||||
second_client_blob = second_client_blob_manager.get_blob(blob_hash)
|
||||
|
||||
# download the blob
|
||||
await asyncio.gather(
|
||||
@@ -112,9 +119,65 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
),
|
||||
self._test_transfer_blob(blob_hash)
|
||||
)
|
||||
await second_client_blob.finished_writing.wait()
|
||||
await second_client_blob.verified.wait()
|
||||
self.assertEqual(second_client_blob.get_is_verified(), True)
|
||||
|
||||
async def test_blob_writers_concurrency(self):
|
||||
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
|
||||
mock_blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
|
||||
blob = self.server_blob_manager.get_blob(blob_hash)
|
||||
write_blob = blob._write_blob
|
||||
write_called_count = 0
|
||||
|
||||
def wrap_write_blob(blob_bytes):
|
||||
nonlocal write_called_count
|
||||
write_called_count += 1
|
||||
write_blob(blob_bytes)
|
||||
blob._write_blob = wrap_write_blob
|
||||
|
||||
writer1 = blob.get_blob_writer(peer_port=1)
|
||||
writer2 = blob.get_blob_writer(peer_port=2)
|
||||
reader1_ctx_before_write = blob.reader_context()
|
||||
|
||||
with self.assertRaises(OSError):
|
||||
blob.get_blob_writer(peer_port=2)
|
||||
with self.assertRaises(OSError):
|
||||
with blob.reader_context():
|
||||
pass
|
||||
|
||||
blob.set_length(len(mock_blob_bytes))
|
||||
results = {}
|
||||
|
||||
def check_finished_callback(writer, num):
|
||||
def inner(writer_future: asyncio.Future):
|
||||
results[num] = writer_future.result()
|
||||
writer.finished.add_done_callback(inner)
|
||||
|
||||
check_finished_callback(writer1, 1)
|
||||
check_finished_callback(writer2, 2)
|
||||
|
||||
def write_task(writer):
|
||||
async def _inner():
|
||||
writer.write(mock_blob_bytes)
|
||||
return self.loop.create_task(_inner())
|
||||
|
||||
await asyncio.gather(write_task(writer1), write_task(writer2), loop=self.loop)
|
||||
|
||||
self.assertDictEqual({1: mock_blob_bytes, 2: mock_blob_bytes}, results)
|
||||
self.assertEqual(1, write_called_count)
|
||||
self.assertTrue(blob.get_is_verified())
|
||||
self.assertDictEqual({}, blob.writers)
|
||||
|
||||
with reader1_ctx_before_write as f:
|
||||
self.assertEqual(mock_blob_bytes, f.read())
|
||||
with blob.reader_context() as f:
|
||||
self.assertEqual(mock_blob_bytes, f.read())
|
||||
with blob.reader_context() as f:
|
||||
blob.close()
|
||||
with self.assertRaises(ValueError):
|
||||
f.read()
|
||||
self.assertListEqual([], blob.readers)
|
||||
|
||||
async def test_host_different_blobs_to_multiple_peers_at_once(self):
|
||||
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
|
||||
mock_blob_bytes = b'1' * ((2 * 2 ** 20) - 1)
|
||||
@@ -124,9 +187,12 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
|
||||
second_client_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, second_client_dir)
|
||||
second_client_conf = Config()
|
||||
|
||||
second_client_storage = SQLiteStorage(Config(), os.path.join(second_client_dir, "lbrynet.sqlite"))
|
||||
second_client_blob_manager = BlobFileManager(self.loop, second_client_dir, second_client_storage)
|
||||
second_client_storage = SQLiteStorage(second_client_conf, os.path.join(second_client_dir, "lbrynet.sqlite"))
|
||||
second_client_blob_manager = BlobManager(
|
||||
self.loop, second_client_dir, second_client_storage, second_client_conf
|
||||
)
|
||||
server_from_second_client = KademliaPeer(self.loop, "127.0.0.1", b'1' * 48, tcp_port=33333)
|
||||
|
||||
await second_client_storage.open()
|
||||
@@ -143,7 +209,7 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
server_from_second_client.tcp_port, 2, 3
|
||||
),
|
||||
self._test_transfer_blob(sd_hash),
|
||||
second_client_blob.finished_writing.wait()
|
||||
second_client_blob.verified.wait()
|
||||
)
|
||||
self.assertEqual(second_client_blob.get_is_verified(), True)
|
||||
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from lbrynet import utils
|
||||
|
||||
import unittest
|
||||
import asyncio
|
||||
from lbrynet import utils
|
||||
from torba.testcase import AsyncioTestCase
|
||||
|
||||
|
||||
class CompareVersionTest(unittest.TestCase):
|
||||
@@ -61,3 +62,76 @@ class SdHashTests(unittest.TestCase):
|
||||
}
|
||||
}
|
||||
self.assertIsNone(utils.get_sd_hash(claim))
|
||||
|
||||
|
||||
class CacheConcurrentDecoratorTests(AsyncioTestCase):
|
||||
def setUp(self):
|
||||
self.called = []
|
||||
self.finished = []
|
||||
self.counter = 0
|
||||
|
||||
@utils.cache_concurrent
|
||||
async def foo(self, arg1, arg2=None, delay=1):
|
||||
self.called.append((arg1, arg2, delay))
|
||||
await asyncio.sleep(delay, loop=self.loop)
|
||||
self.counter += 1
|
||||
self.finished.append((arg1, arg2, delay))
|
||||
return object()
|
||||
|
||||
async def test_gather_duplicates(self):
|
||||
result = await asyncio.gather(
|
||||
self.loop.create_task(self.foo(1)), self.loop.create_task(self.foo(1)), loop=self.loop
|
||||
)
|
||||
self.assertEqual(1, len(self.called))
|
||||
self.assertEqual(1, len(self.finished))
|
||||
self.assertEqual(1, self.counter)
|
||||
self.assertIs(result[0], result[1])
|
||||
self.assertEqual(2, len(result))
|
||||
|
||||
async def test_one_cancelled_all_cancel(self):
|
||||
t1 = self.loop.create_task(self.foo(1))
|
||||
self.loop.call_later(0.1, t1.cancel)
|
||||
|
||||
with self.assertRaises(asyncio.CancelledError):
|
||||
await asyncio.gather(
|
||||
t1, self.loop.create_task(self.foo(1)), loop=self.loop
|
||||
)
|
||||
self.assertEqual(1, len(self.called))
|
||||
self.assertEqual(0, len(self.finished))
|
||||
self.assertEqual(0, self.counter)
|
||||
|
||||
async def test_error_after_success(self):
|
||||
def cause_type_error():
|
||||
self.counter = ""
|
||||
|
||||
self.loop.call_later(0.1, cause_type_error)
|
||||
|
||||
t1 = self.loop.create_task(self.foo(1))
|
||||
t2 = self.loop.create_task(self.foo(1))
|
||||
|
||||
with self.assertRaises(TypeError):
|
||||
await t2
|
||||
self.assertEqual(1, len(self.called))
|
||||
self.assertEqual(0, len(self.finished))
|
||||
self.assertTrue(t1.done())
|
||||
self.assertEqual("", self.counter)
|
||||
|
||||
# test that the task is run fresh, it should not error
|
||||
self.counter = 0
|
||||
t3 = self.loop.create_task(self.foo(1))
|
||||
self.assertTrue((await t3))
|
||||
self.assertEqual(1, self.counter)
|
||||
|
||||
# the previously failed call should still raise if awaited
|
||||
with self.assertRaises(TypeError):
|
||||
await t1
|
||||
|
||||
self.assertEqual(1, self.counter)
|
||||
|
||||
async def test_break_it(self):
|
||||
t1 = self.loop.create_task(self.foo(1))
|
||||
t2 = self.loop.create_task(self.foo(1))
|
||||
t3 = self.loop.create_task(self.foo(2, delay=0))
|
||||
t3.add_done_callback(lambda _: t2.cancel())
|
||||
with self.assertRaises(asyncio.CancelledError):
|
||||
await asyncio.gather(t1, t2, t3)
|
||||
@@ -7,7 +7,7 @@ from torba.testcase import AsyncioTestCase
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_info import BlobInfo
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from tests.test_utils import random_lbry_hash
|
||||
|
||||
@@ -68,17 +68,18 @@ fake_claim_info = {
|
||||
|
||||
class StorageTest(AsyncioTestCase):
|
||||
async def asyncSetUp(self):
|
||||
self.storage = SQLiteStorage(Config(), ':memory:')
|
||||
self.conf = Config()
|
||||
self.storage = SQLiteStorage(self.conf, ':memory:')
|
||||
self.blob_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, self.blob_dir)
|
||||
self.blob_manager = BlobFileManager(asyncio.get_event_loop(), self.blob_dir, self.storage)
|
||||
self.blob_manager = BlobManager(asyncio.get_event_loop(), self.blob_dir, self.storage, self.conf)
|
||||
await self.storage.open()
|
||||
|
||||
async def asyncTearDown(self):
|
||||
await self.storage.close()
|
||||
|
||||
async def store_fake_blob(self, blob_hash, length=100):
|
||||
await self.storage.add_completed_blob(blob_hash, length)
|
||||
await self.storage.add_blobs((blob_hash, length), finished=True)
|
||||
|
||||
async def store_fake_stream(self, stream_hash, blobs=None, file_name="fake_file", key="DEADBEEF"):
|
||||
blobs = blobs or [BlobInfo(1, 100, "DEADBEEF", random_lbry_hash())]
|
||||
@@ -130,7 +131,7 @@ class TestSQLiteStorage(StorageTest):
|
||||
} for i in range(20)]
|
||||
expected_supports = {}
|
||||
for idx, claim_id in enumerate(claim_ids):
|
||||
await self.storage.save_supports(claim_id, random_supports[idx*2:idx*2+2])
|
||||
await self.storage.save_supports({claim_id: random_supports[idx*2:idx*2+2]})
|
||||
for random_support in random_supports[idx*2:idx*2+2]:
|
||||
random_support['claim_id'] = claim_id
|
||||
expected_supports.setdefault(claim_id, []).append(random_support)
|
||||
|
||||
@@ -17,10 +17,10 @@ class MockAsyncGen:
|
||||
return self
|
||||
|
||||
async def __anext__(self):
|
||||
await asyncio.sleep(self.delay, loop=self.loop)
|
||||
if self.count > self.stop_cnt - 1:
|
||||
raise StopAsyncIteration()
|
||||
self.count += 1
|
||||
await asyncio.sleep(self.delay, loop=self.loop)
|
||||
return self.result
|
||||
|
||||
async def aclose(self):
|
||||
@@ -48,6 +48,18 @@ class TestAsyncGeneratorJunction(AsyncioTestCase):
|
||||
self.assertEqual(fast_gen.called_close, True)
|
||||
self.assertEqual(slow_gen.called_close, True)
|
||||
|
||||
async def test_nothing_to_yield(self):
|
||||
async def __nothing():
|
||||
for _ in []:
|
||||
yield self.fail("nada")
|
||||
await self._test_junction([], __nothing())
|
||||
|
||||
async def test_fast_iteratiors(self):
|
||||
async def __gotta_go_fast():
|
||||
for _ in range(10):
|
||||
yield 0
|
||||
await self._test_junction([0]*40, __gotta_go_fast(), __gotta_go_fast(), __gotta_go_fast(), __gotta_go_fast())
|
||||
|
||||
@unittest.SkipTest
|
||||
async def test_one_stopped_first(self):
|
||||
expected_order = [1, 2, 1, 1, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2]
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
import asyncio
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from unittest import mock, TestCase
|
||||
from lbrynet.dht.protocol.data_store import DictDataStore
|
||||
from lbrynet.dht.peer import PeerManager
|
||||
|
||||
|
||||
class DataStoreTests(AsyncioTestCase):
|
||||
class DataStoreTests(TestCase):
|
||||
def setUp(self):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.loop = mock.Mock(spec=asyncio.BaseEventLoop)
|
||||
self.loop.time = lambda: 0.0
|
||||
self.peer_manager = PeerManager(self.loop)
|
||||
self.data_store = DictDataStore(self.loop, self.peer_manager)
|
||||
|
||||
|
||||
@@ -78,15 +78,14 @@ class TestBlobAnnouncer(AsyncioTestCase):
|
||||
blob2 = binascii.hexlify(b'2' * 48).decode()
|
||||
|
||||
async with self._test_network_context():
|
||||
await self.storage.add_completed_blob(blob1, 1024)
|
||||
await self.storage.add_completed_blob(blob2, 1024)
|
||||
await self.storage.add_blobs((blob1, 1024), (blob2, 1024), finished=True)
|
||||
await self.storage.db.execute(
|
||||
"update blob set next_announce_time=0, should_announce=1 where blob_hash in (?, ?)",
|
||||
(blob1, blob2)
|
||||
)
|
||||
to_announce = await self.storage.get_blobs_to_announce()
|
||||
self.assertEqual(2, len(to_announce))
|
||||
self.blob_announcer.start()
|
||||
self.blob_announcer.start(batch_size=1) # so it covers batching logic
|
||||
await self.advance(61.0)
|
||||
to_announce = await self.storage.get_blobs_to_announce()
|
||||
self.assertEqual(0, len(to_announce))
|
||||
@@ -106,7 +105,7 @@ class TestBlobAnnouncer(AsyncioTestCase):
|
||||
|
||||
_, task = last.accumulate_peers(search_q, peer_q)
|
||||
found_peers = await peer_q.get()
|
||||
task.cancel()
|
||||
await task
|
||||
|
||||
self.assertEqual(1, len(found_peers))
|
||||
self.assertEqual(self.node.protocol.node_id, found_peers[0].node_id)
|
||||
|
||||
@@ -4,13 +4,13 @@ from lbrynet.schema import mime_types
|
||||
|
||||
class TestMimeTypes(unittest.TestCase):
|
||||
def test_mp4_video(self):
|
||||
self.assertEqual("video/mp4", mime_types.guess_media_type("test.mp4"))
|
||||
self.assertEqual("video/mp4", mime_types.guess_media_type("test.MP4"))
|
||||
self.assertEqual("video/mp4", mime_types.guess_media_type("test.mp4")[0])
|
||||
self.assertEqual("video/mp4", mime_types.guess_media_type("test.MP4")[0])
|
||||
|
||||
def test_x_ext_(self):
|
||||
self.assertEqual("application/x-ext-lbry", mime_types.guess_media_type("test.lbry"))
|
||||
self.assertEqual("application/x-ext-lbry", mime_types.guess_media_type("test.LBRY"))
|
||||
self.assertEqual("application/x-ext-lbry", mime_types.guess_media_type("test.lbry")[0])
|
||||
self.assertEqual("application/x-ext-lbry", mime_types.guess_media_type("test.LBRY")[0])
|
||||
|
||||
def test_octet_stream(self):
|
||||
self.assertEqual("application/octet-stream", mime_types.guess_media_type("test."))
|
||||
self.assertEqual("application/octet-stream", mime_types.guess_media_type("test"))
|
||||
self.assertEqual("application/octet-stream", mime_types.guess_media_type("test.")[0])
|
||||
self.assertEqual("application/octet-stream", mime_types.guess_media_type("test")[0])
|
||||
@@ -20,11 +20,11 @@ class TestOldJSONSchemaCompatibility(TestCase):
|
||||
self.assertEqual(stream.description, '10MB test file to measure download speed on Lbry p2p-network.')
|
||||
self.assertEqual(stream.license, 'None')
|
||||
self.assertEqual(stream.author, 'root')
|
||||
self.assertEqual(stream.language, 'en')
|
||||
self.assertEqual(stream.media_type, 'application/octet-stream')
|
||||
self.assertEqual(stream.thumbnail_url, '/home/robert/lbry/speed.jpg')
|
||||
self.assertEqual(stream.langtags, ['en'])
|
||||
self.assertEqual(stream.source.media_type, 'application/octet-stream')
|
||||
self.assertEqual(stream.thumbnail.url, '/home/robert/lbry/speed.jpg')
|
||||
self.assertEqual(
|
||||
stream.sd_hash,
|
||||
stream.source.sd_hash,
|
||||
'bbd1f68374ff9a1044a90d7dd578ce41979211c386caf19e'
|
||||
'6f496536db5f2c96b58fe2c7a6677b331419a117873b539f'
|
||||
)
|
||||
@@ -51,10 +51,10 @@ class TestOldJSONSchemaCompatibility(TestCase):
|
||||
self.assertEqual(stream.license, 'Creative Commons Attribution 3.0 United States')
|
||||
self.assertEqual(stream.license_url, 'https://creativecommons.org/licenses/by/3.0/us/legalcode')
|
||||
self.assertEqual(stream.author, 'Mii')
|
||||
self.assertEqual(stream.language, 'en')
|
||||
self.assertEqual(stream.media_type, 'application/x-msdownload')
|
||||
self.assertEqual(stream.langtags, ['en'])
|
||||
self.assertEqual(stream.source.media_type, 'application/x-msdownload')
|
||||
self.assertEqual(
|
||||
stream.sd_hash,
|
||||
stream.source.sd_hash,
|
||||
'c5ffee0fa5168e16681b519d9d85446e8d1d818a616bd555'
|
||||
'40aa7374d2321b51abf2ac3dae1443a03dadcc8f7affaa62'
|
||||
)
|
||||
@@ -77,10 +77,10 @@ class TestOldJSONSchemaCompatibility(TestCase):
|
||||
self.assertEqual(stream.description, 'asd')
|
||||
self.assertEqual(stream.license, 'Creative Commons Attribution 4.0 International')
|
||||
self.assertEqual(stream.author, 'sgb')
|
||||
self.assertEqual(stream.language, 'en')
|
||||
self.assertEqual(stream.media_type, 'video/mp4')
|
||||
self.assertEqual(stream.langtags, ['en'])
|
||||
self.assertEqual(stream.source.media_type, 'video/mp4')
|
||||
self.assertEqual(
|
||||
stream.sd_hash,
|
||||
stream.source.sd_hash,
|
||||
'd83db664c6d7d570aa824300f4869e0bfb560e765efa477a'
|
||||
'ebf566467f8d3a57f4f8c704cab1308eb75ff8b7e84e3caf'
|
||||
)
|
||||
@@ -120,11 +120,11 @@ class TestTypesV1Compatibility(TestCase):
|
||||
)
|
||||
self.assertEqual(stream.license, 'Copyrighted (contact author)')
|
||||
self.assertEqual(stream.author, 'The Linux Gamer')
|
||||
self.assertEqual(stream.language, 'en')
|
||||
self.assertEqual(stream.media_type, 'video/mp4')
|
||||
self.assertEqual(stream.thumbnail_url, 'https://berk.ninja/thumbnails/FrTdBCOS_fc')
|
||||
self.assertEqual(stream.langtags, ['en'])
|
||||
self.assertEqual(stream.source.media_type, 'video/mp4')
|
||||
self.assertEqual(stream.thumbnail.url, 'https://berk.ninja/thumbnails/FrTdBCOS_fc')
|
||||
self.assertEqual(
|
||||
stream.sd_hash,
|
||||
stream.source.sd_hash,
|
||||
'040e8ac6e89c061f982528c23ad33829fd7146435bf7a4cc'
|
||||
'22f0bff70c4fe0b91fd36da9a375e3e1c171db825bf5d1f3'
|
||||
)
|
||||
@@ -157,10 +157,10 @@ class TestTypesV1Compatibility(TestCase):
|
||||
self.assertEqual(stream.description, 'midi')
|
||||
self.assertEqual(stream.license, 'Creative Commons Attribution 4.0 International')
|
||||
self.assertEqual(stream.author, 'rpg midi')
|
||||
self.assertEqual(stream.language, 'en')
|
||||
self.assertEqual(stream.media_type, 'application/x-zip-compressed')
|
||||
self.assertEqual(stream.langtags, ['en'])
|
||||
self.assertEqual(stream.source.media_type, 'application/x-zip-compressed')
|
||||
self.assertEqual(
|
||||
stream.sd_hash,
|
||||
stream.source.sd_hash,
|
||||
'1f41eb0312aa7e8a5ce49349bc77d811da975833719d7515'
|
||||
'23b19f123fc3d528d6a94e3446ccddb7b9329f27a9cad7e3'
|
||||
)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from unittest import TestCase
|
||||
from decimal import Decimal
|
||||
|
||||
from lbrynet.schema.claim import Claim, Channel, Stream
|
||||
from lbrynet.schema.claim import Claim, Stream
|
||||
|
||||
|
||||
class TestClaimContainerAwareness(TestCase):
|
||||
@@ -9,27 +9,13 @@ class TestClaimContainerAwareness(TestCase):
|
||||
def test_stream_claim(self):
|
||||
stream = Stream()
|
||||
claim = stream.claim
|
||||
self.assertTrue(claim.is_stream)
|
||||
self.assertFalse(claim.is_channel)
|
||||
self.assertEqual(claim.claim_type, Claim.STREAM)
|
||||
claim = Claim.from_bytes(claim.to_bytes())
|
||||
self.assertTrue(claim.is_stream)
|
||||
self.assertFalse(claim.is_channel)
|
||||
self.assertEqual(claim.claim_type, Claim.STREAM)
|
||||
self.assertIsNotNone(claim.stream)
|
||||
with self.assertRaisesRegex(ValueError, 'Claim is not a channel.'):
|
||||
print(claim.channel)
|
||||
|
||||
def test_channel_claim(self):
|
||||
channel = Channel()
|
||||
claim = channel.claim
|
||||
self.assertFalse(claim.is_stream)
|
||||
self.assertTrue(claim.is_channel)
|
||||
claim = Claim.from_bytes(claim.to_bytes())
|
||||
self.assertFalse(claim.is_stream)
|
||||
self.assertTrue(claim.is_channel)
|
||||
self.assertIsNotNone(claim.channel)
|
||||
with self.assertRaisesRegex(ValueError, 'Claim is not a stream.'):
|
||||
print(claim.stream)
|
||||
|
||||
|
||||
class TestFee(TestCase):
|
||||
|
||||
@@ -61,3 +47,103 @@ class TestFee(TestCase):
|
||||
print(stream.fee.lbc)
|
||||
with self.assertRaisesRegex(ValueError, 'Dewies can only be returned for LBC fees.'):
|
||||
print(stream.fee.dewies)
|
||||
|
||||
|
||||
class TestLanguages(TestCase):
|
||||
|
||||
def test_language_successful_parsing(self):
|
||||
stream = Stream()
|
||||
|
||||
stream.languages.append('en')
|
||||
self.assertEqual(stream.languages[0].langtag, 'en')
|
||||
self.assertEqual(stream.languages[0].language, 'en')
|
||||
self.assertEqual(stream.langtags, ['en'])
|
||||
|
||||
stream.languages.append('en-US')
|
||||
self.assertEqual(stream.languages[1].langtag, 'en-US')
|
||||
self.assertEqual(stream.languages[1].language, 'en')
|
||||
self.assertEqual(stream.languages[1].region, 'US')
|
||||
self.assertEqual(stream.langtags, ['en', 'en-US'])
|
||||
|
||||
stream.languages.append('en-Latn-US')
|
||||
self.assertEqual(stream.languages[2].langtag, 'en-Latn-US')
|
||||
self.assertEqual(stream.languages[2].language, 'en')
|
||||
self.assertEqual(stream.languages[2].script, 'Latn')
|
||||
self.assertEqual(stream.languages[2].region, 'US')
|
||||
self.assertEqual(stream.langtags, ['en', 'en-US', 'en-Latn-US'])
|
||||
|
||||
stream = Stream()
|
||||
stream.languages.extend(['en-Latn-US', 'es-ES', 'de-DE'])
|
||||
self.assertEqual(stream.languages[0].language, 'en')
|
||||
self.assertEqual(stream.languages[1].language, 'es')
|
||||
self.assertEqual(stream.languages[2].language, 'de')
|
||||
|
||||
def test_language_error_parsing(self):
|
||||
stream = Stream()
|
||||
with self.assertRaisesRegex(ValueError, 'Language has no value defined for name zz'):
|
||||
stream.languages.append('zz')
|
||||
with self.assertRaisesRegex(ValueError, 'Script has no value defined for name Zabc'):
|
||||
stream.languages.append('en-Zabc')
|
||||
with self.assertRaisesRegex(ValueError, 'Country has no value defined for name ZZ'):
|
||||
stream.languages.append('en-Zzzz-ZZ')
|
||||
with self.assertRaisesRegex(AssertionError, 'Failed to parse language tag: en-Zzz-US'):
|
||||
stream.languages.append('en-Zzz-US')
|
||||
|
||||
|
||||
class TestLocations(TestCase):
|
||||
|
||||
def test_location_successful_parsing(self):
|
||||
# from simple string
|
||||
stream = Stream()
|
||||
stream.locations.append('US')
|
||||
self.assertEqual(stream.locations[0].country, 'US')
|
||||
|
||||
# from full string
|
||||
stream = Stream()
|
||||
stream.locations.append('US:NH:Manchester:03101:42.990605:-71.460989')
|
||||
self.assertEqual(stream.locations[0].country, 'US')
|
||||
self.assertEqual(stream.locations[0].state, 'NH')
|
||||
self.assertEqual(stream.locations[0].city, 'Manchester')
|
||||
self.assertEqual(stream.locations[0].code, '03101')
|
||||
self.assertEqual(stream.locations[0].latitude, '42.990605')
|
||||
self.assertEqual(stream.locations[0].longitude, '-71.460989')
|
||||
|
||||
# from partial string
|
||||
stream = Stream()
|
||||
stream.locations.append('::Manchester:03101:')
|
||||
self.assertEqual(stream.locations[0].country, None)
|
||||
self.assertEqual(stream.locations[0].state, '')
|
||||
self.assertEqual(stream.locations[0].city, 'Manchester')
|
||||
self.assertEqual(stream.locations[0].code, '03101')
|
||||
self.assertEqual(stream.locations[0].latitude, None)
|
||||
self.assertEqual(stream.locations[0].longitude, None)
|
||||
|
||||
# from partial string lat/long
|
||||
stream = Stream()
|
||||
stream.locations.append('::::42.990605:-71.460989')
|
||||
self.assertEqual(stream.locations[0].country, None)
|
||||
self.assertEqual(stream.locations[0].state, '')
|
||||
self.assertEqual(stream.locations[0].city, '')
|
||||
self.assertEqual(stream.locations[0].code, '')
|
||||
self.assertEqual(stream.locations[0].latitude, '42.990605')
|
||||
self.assertEqual(stream.locations[0].longitude, '-71.460989')
|
||||
|
||||
# from short circuit lat/long
|
||||
stream = Stream()
|
||||
stream.locations.append('42.990605:-71.460989')
|
||||
self.assertEqual(stream.locations[0].country, None)
|
||||
self.assertEqual(stream.locations[0].state, '')
|
||||
self.assertEqual(stream.locations[0].city, '')
|
||||
self.assertEqual(stream.locations[0].code, '')
|
||||
self.assertEqual(stream.locations[0].latitude, '42.990605')
|
||||
self.assertEqual(stream.locations[0].longitude, '-71.460989')
|
||||
|
||||
# from json string
|
||||
stream = Stream()
|
||||
stream.locations.append('{"country": "ES"}')
|
||||
self.assertEqual(stream.locations[0].country, 'ES')
|
||||
|
||||
# from dict
|
||||
stream = Stream()
|
||||
stream.locations.append({"country": "UA"})
|
||||
self.assertEqual(stream.locations[0].country, 'UA')
|
||||
@@ -1,117 +0,0 @@
|
||||
import os
|
||||
import asyncio
|
||||
import tempfile
|
||||
import shutil
|
||||
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.stream.assembler import StreamAssembler
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.stream.stream_manager import StreamManager
|
||||
|
||||
|
||||
class TestStreamAssembler(AsyncioTestCase):
|
||||
def setUp(self):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.key = b'deadbeef' * 4
|
||||
self.cleartext = b'test'
|
||||
|
||||
async def test_create_and_decrypt_one_blob_stream(self, corrupt=False):
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
|
||||
self.storage = SQLiteStorage(Config(), ":memory:")
|
||||
await self.storage.open()
|
||||
self.blob_manager = BlobFileManager(self.loop, tmp_dir, self.storage)
|
||||
|
||||
download_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(download_dir))
|
||||
|
||||
# create the stream
|
||||
file_path = os.path.join(tmp_dir, "test_file")
|
||||
with open(file_path, 'wb') as f:
|
||||
f.write(self.cleartext)
|
||||
|
||||
sd = await StreamDescriptor.create_stream(self.loop, tmp_dir, file_path, key=self.key)
|
||||
|
||||
# copy blob files
|
||||
sd_hash = sd.calculate_sd_hash()
|
||||
shutil.copy(os.path.join(tmp_dir, sd_hash), os.path.join(download_dir, sd_hash))
|
||||
for blob_info in sd.blobs:
|
||||
if blob_info.blob_hash:
|
||||
shutil.copy(os.path.join(tmp_dir, blob_info.blob_hash), os.path.join(download_dir, blob_info.blob_hash))
|
||||
if corrupt and blob_info.length == MAX_BLOB_SIZE:
|
||||
with open(os.path.join(download_dir, blob_info.blob_hash), "rb+") as handle:
|
||||
handle.truncate()
|
||||
handle.flush()
|
||||
|
||||
downloader_storage = SQLiteStorage(Config(), os.path.join(download_dir, "lbrynet.sqlite"))
|
||||
await downloader_storage.open()
|
||||
|
||||
# add the blobs to the blob table (this would happen upon a blob download finishing)
|
||||
downloader_blob_manager = BlobFileManager(self.loop, download_dir, downloader_storage)
|
||||
descriptor = await downloader_blob_manager.get_stream_descriptor(sd_hash)
|
||||
|
||||
# assemble the decrypted file
|
||||
assembler = StreamAssembler(self.loop, downloader_blob_manager, descriptor.sd_hash)
|
||||
await assembler.assemble_decrypted_stream(download_dir)
|
||||
if corrupt:
|
||||
return self.assertFalse(os.path.isfile(os.path.join(download_dir, "test_file")))
|
||||
|
||||
with open(os.path.join(download_dir, "test_file"), "rb") as f:
|
||||
decrypted = f.read()
|
||||
self.assertEqual(decrypted, self.cleartext)
|
||||
self.assertEqual(True, self.blob_manager.get_blob(sd_hash).get_is_verified())
|
||||
self.assertEqual(True, self.blob_manager.get_blob(descriptor.blobs[0].blob_hash).get_is_verified())
|
||||
# its all blobs + sd blob - last blob, which is the same size as descriptor.blobs
|
||||
self.assertEqual(len(descriptor.blobs), len(await downloader_storage.get_all_finished_blobs()))
|
||||
self.assertEqual(
|
||||
[descriptor.sd_hash, descriptor.blobs[0].blob_hash], await downloader_storage.get_blobs_to_announce()
|
||||
)
|
||||
|
||||
await downloader_storage.close()
|
||||
await self.storage.close()
|
||||
|
||||
async def test_create_and_decrypt_multi_blob_stream(self):
|
||||
self.cleartext = b'test\n' * 20000000
|
||||
await self.test_create_and_decrypt_one_blob_stream()
|
||||
|
||||
async def test_create_and_decrypt_padding(self):
|
||||
for i in range(16):
|
||||
self.cleartext = os.urandom((MAX_BLOB_SIZE*2) + i)
|
||||
await self.test_create_and_decrypt_one_blob_stream()
|
||||
|
||||
for i in range(16):
|
||||
self.cleartext = os.urandom((MAX_BLOB_SIZE*2) - i)
|
||||
await self.test_create_and_decrypt_one_blob_stream()
|
||||
|
||||
async def test_create_and_decrypt_random(self):
|
||||
self.cleartext = os.urandom(20000000)
|
||||
await self.test_create_and_decrypt_one_blob_stream()
|
||||
|
||||
async def test_create_managed_stream_announces(self):
|
||||
# setup a blob manager
|
||||
storage = SQLiteStorage(Config(), ":memory:")
|
||||
await storage.open()
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
|
||||
blob_manager = BlobFileManager(self.loop, tmp_dir, storage)
|
||||
stream_manager = StreamManager(self.loop, Config(), blob_manager, None, storage, None)
|
||||
# create the stream
|
||||
download_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(download_dir))
|
||||
file_path = os.path.join(download_dir, "test_file")
|
||||
with open(file_path, 'wb') as f:
|
||||
f.write(b'testtest')
|
||||
|
||||
stream = await stream_manager.create_stream(file_path)
|
||||
self.assertEqual(
|
||||
[stream.sd_hash, stream.descriptor.blobs[0].blob_hash],
|
||||
await storage.get_blobs_to_announce())
|
||||
|
||||
async def test_create_truncate_and_handle_stream(self):
|
||||
self.cleartext = b'potato' * 1337 * 5279
|
||||
# The purpose of this test is just to make sure it can finish even if a blob is corrupt/truncated
|
||||
await asyncio.wait_for(self.test_create_and_decrypt_one_blob_stream(corrupt=True), timeout=5)
|
||||
@@ -1,102 +0,0 @@
|
||||
import os
|
||||
import time
|
||||
import unittest
|
||||
from unittest import mock
|
||||
import asyncio
|
||||
|
||||
from lbrynet.blob_exchange.serialization import BlobResponse
|
||||
from lbrynet.blob_exchange.server import BlobServerProtocol
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.stream.downloader import StreamDownloader
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
|
||||
from tests.unit.blob_exchange.test_transfer_blob import BlobExchangeTestBase
|
||||
|
||||
|
||||
class TestStreamDownloader(BlobExchangeTestBase):
|
||||
async def setup_stream(self, blob_count: int = 10):
|
||||
self.stream_bytes = b''
|
||||
for _ in range(blob_count):
|
||||
self.stream_bytes += os.urandom((MAX_BLOB_SIZE - 1))
|
||||
# create the stream
|
||||
file_path = os.path.join(self.server_dir, "test_file")
|
||||
with open(file_path, 'wb') as f:
|
||||
f.write(self.stream_bytes)
|
||||
descriptor = await StreamDescriptor.create_stream(self.loop, self.server_blob_manager.blob_dir, file_path)
|
||||
self.sd_hash = descriptor.calculate_sd_hash()
|
||||
conf = Config(data_dir=self.server_dir, wallet_dir=self.server_dir, download_dir=self.server_dir,
|
||||
reflector_servers=[])
|
||||
self.downloader = StreamDownloader(self.loop, conf, self.client_blob_manager, self.sd_hash)
|
||||
|
||||
async def _test_transfer_stream(self, blob_count: int, mock_accumulate_peers=None):
|
||||
await self.setup_stream(blob_count)
|
||||
mock_node = mock.Mock(spec=Node)
|
||||
|
||||
def _mock_accumulate_peers(q1, q2):
|
||||
async def _task():
|
||||
pass
|
||||
q2.put_nowait([self.server_from_client])
|
||||
return q2, self.loop.create_task(_task())
|
||||
|
||||
mock_node.accumulate_peers = mock_accumulate_peers or _mock_accumulate_peers
|
||||
self.downloader.download(mock_node)
|
||||
await self.downloader.stream_finished_event.wait()
|
||||
self.assertTrue(self.downloader.stream_handle.closed)
|
||||
self.assertTrue(os.path.isfile(self.downloader.output_path))
|
||||
self.downloader.stop()
|
||||
self.assertIs(self.downloader.stream_handle, None)
|
||||
self.assertTrue(os.path.isfile(self.downloader.output_path))
|
||||
with open(self.downloader.output_path, 'rb') as f:
|
||||
self.assertEqual(f.read(), self.stream_bytes)
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
async def test_transfer_stream(self):
|
||||
await self._test_transfer_stream(10)
|
||||
|
||||
@unittest.SkipTest
|
||||
async def test_transfer_hundred_blob_stream(self):
|
||||
await self._test_transfer_stream(100)
|
||||
|
||||
async def test_transfer_stream_bad_first_peer_good_second(self):
|
||||
await self.setup_stream(2)
|
||||
|
||||
mock_node = mock.Mock(spec=Node)
|
||||
q = asyncio.Queue()
|
||||
|
||||
bad_peer = KademliaPeer(self.loop, "127.0.0.1", b'2' * 48, tcp_port=3334)
|
||||
|
||||
def _mock_accumulate_peers(q1, q2):
|
||||
async def _task():
|
||||
pass
|
||||
|
||||
q2.put_nowait([bad_peer])
|
||||
self.loop.call_later(1, q2.put_nowait, [self.server_from_client])
|
||||
return q2, self.loop.create_task(_task())
|
||||
|
||||
mock_node.accumulate_peers = _mock_accumulate_peers
|
||||
|
||||
self.downloader.download(mock_node)
|
||||
await self.downloader.stream_finished_event.wait()
|
||||
self.assertTrue(os.path.isfile(self.downloader.output_path))
|
||||
with open(self.downloader.output_path, 'rb') as f:
|
||||
self.assertEqual(f.read(), self.stream_bytes)
|
||||
# self.assertIs(self.server_from_client.tcp_last_down, None)
|
||||
# self.assertIsNot(bad_peer.tcp_last_down, None)
|
||||
|
||||
async def test_client_chunked_response(self):
|
||||
self.server.stop_server()
|
||||
class ChunkedServerProtocol(BlobServerProtocol):
|
||||
|
||||
def send_response(self, responses):
|
||||
to_send = []
|
||||
while responses:
|
||||
to_send.append(responses.pop())
|
||||
for byte in BlobResponse(to_send).serialize():
|
||||
self.transport.write(bytes([byte]))
|
||||
self.server.server_protocol_class = ChunkedServerProtocol
|
||||
self.server.start_server(33333, '127.0.0.1')
|
||||
self.assertEqual(0, len(self.client_blob_manager.completed_blob_hashes))
|
||||
await asyncio.wait_for(self._test_transfer_stream(10), timeout=2)
|
||||
self.assertEqual(11, len(self.client_blob_manager.completed_blob_hashes))
|
||||
@@ -0,0 +1,170 @@
|
||||
import os
|
||||
import shutil
|
||||
import unittest
|
||||
from unittest import mock
|
||||
import asyncio
|
||||
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
|
||||
from lbrynet.blob_exchange.serialization import BlobResponse
|
||||
from lbrynet.blob_exchange.server import BlobServerProtocol
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.extras.daemon.storage import StoredStreamClaim
|
||||
from lbrynet.stream.managed_stream import ManagedStream
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from tests.unit.blob_exchange.test_transfer_blob import BlobExchangeTestBase
|
||||
|
||||
|
||||
def get_mock_node(loop):
|
||||
mock_node = mock.Mock(spec=Node)
|
||||
mock_node.joined = asyncio.Event(loop=loop)
|
||||
mock_node.joined.set()
|
||||
return mock_node
|
||||
|
||||
|
||||
class TestManagedStream(BlobExchangeTestBase):
|
||||
async def create_stream(self, blob_count: int = 10):
|
||||
self.stream_bytes = b''
|
||||
for _ in range(blob_count):
|
||||
self.stream_bytes += os.urandom((MAX_BLOB_SIZE - 1))
|
||||
# create the stream
|
||||
file_path = os.path.join(self.server_dir, "test_file")
|
||||
with open(file_path, 'wb') as f:
|
||||
f.write(self.stream_bytes)
|
||||
descriptor = await StreamDescriptor.create_stream(self.loop, self.server_blob_manager.blob_dir, file_path)
|
||||
self.sd_hash = descriptor.calculate_sd_hash()
|
||||
return descriptor
|
||||
|
||||
async def setup_stream(self, blob_count: int = 10):
|
||||
await self.create_stream(blob_count)
|
||||
self.stream = ManagedStream(
|
||||
self.loop, self.client_config, self.client_blob_manager, self.sd_hash, self.client_dir
|
||||
)
|
||||
|
||||
async def _test_transfer_stream(self, blob_count: int, mock_accumulate_peers=None, stop_when_done=True):
|
||||
await self.setup_stream(blob_count)
|
||||
mock_node = mock.Mock(spec=Node)
|
||||
|
||||
def _mock_accumulate_peers(q1, q2):
|
||||
async def _task():
|
||||
pass
|
||||
q2.put_nowait([self.server_from_client])
|
||||
return q2, self.loop.create_task(_task())
|
||||
|
||||
mock_node.accumulate_peers = mock_accumulate_peers or _mock_accumulate_peers
|
||||
await self.stream.save_file(node=mock_node)
|
||||
await self.stream.finished_writing.wait()
|
||||
self.assertTrue(os.path.isfile(self.stream.full_path))
|
||||
if stop_when_done:
|
||||
await self.stream.stop()
|
||||
self.assertTrue(os.path.isfile(self.stream.full_path))
|
||||
with open(self.stream.full_path, 'rb') as f:
|
||||
self.assertEqual(f.read(), self.stream_bytes)
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
async def test_transfer_stream(self):
|
||||
await self._test_transfer_stream(10)
|
||||
self.assertEqual(self.stream.status, "finished")
|
||||
self.assertFalse(self.stream._running.is_set())
|
||||
|
||||
async def test_delayed_stop(self):
|
||||
await self._test_transfer_stream(10, stop_when_done=False)
|
||||
self.assertEqual(self.stream.status, "finished")
|
||||
self.assertTrue(self.stream._running.is_set())
|
||||
await asyncio.sleep(0.5, loop=self.loop)
|
||||
self.assertTrue(self.stream._running.is_set())
|
||||
await asyncio.sleep(0.6, loop=self.loop)
|
||||
self.assertEqual(self.stream.status, "finished")
|
||||
self.assertFalse(self.stream._running.is_set())
|
||||
|
||||
@unittest.SkipTest
|
||||
async def test_transfer_hundred_blob_stream(self):
|
||||
await self._test_transfer_stream(100)
|
||||
|
||||
async def test_transfer_stream_bad_first_peer_good_second(self):
|
||||
await self.setup_stream(2)
|
||||
|
||||
mock_node = mock.Mock(spec=Node)
|
||||
q = asyncio.Queue()
|
||||
|
||||
bad_peer = KademliaPeer(self.loop, "127.0.0.1", b'2' * 48, tcp_port=3334)
|
||||
|
||||
def _mock_accumulate_peers(q1, q2):
|
||||
async def _task():
|
||||
pass
|
||||
|
||||
q2.put_nowait([bad_peer])
|
||||
self.loop.call_later(1, q2.put_nowait, [self.server_from_client])
|
||||
return q2, self.loop.create_task(_task())
|
||||
|
||||
mock_node.accumulate_peers = _mock_accumulate_peers
|
||||
|
||||
await self.stream.save_file(node=mock_node)
|
||||
await self.stream.finished_writing.wait()
|
||||
self.assertTrue(os.path.isfile(self.stream.full_path))
|
||||
with open(self.stream.full_path, 'rb') as f:
|
||||
self.assertEqual(f.read(), self.stream_bytes)
|
||||
await self.stream.stop()
|
||||
# self.assertIs(self.server_from_client.tcp_last_down, None)
|
||||
# self.assertIsNot(bad_peer.tcp_last_down, None)
|
||||
|
||||
async def test_client_chunked_response(self):
|
||||
self.server.stop_server()
|
||||
|
||||
class ChunkedServerProtocol(BlobServerProtocol):
|
||||
def send_response(self, responses):
|
||||
to_send = []
|
||||
while responses:
|
||||
to_send.append(responses.pop())
|
||||
for byte in BlobResponse(to_send).serialize():
|
||||
self.transport.write(bytes([byte]))
|
||||
self.server.server_protocol_class = ChunkedServerProtocol
|
||||
self.server.start_server(33333, '127.0.0.1')
|
||||
self.assertEqual(0, len(self.client_blob_manager.completed_blob_hashes))
|
||||
await asyncio.wait_for(self._test_transfer_stream(10), timeout=2)
|
||||
self.assertEqual(11, len(self.client_blob_manager.completed_blob_hashes))
|
||||
|
||||
async def test_create_and_decrypt_one_blob_stream(self, blobs=1, corrupt=False):
|
||||
descriptor = await self.create_stream(blobs)
|
||||
|
||||
# copy blob files
|
||||
shutil.copy(os.path.join(self.server_blob_manager.blob_dir, self.sd_hash),
|
||||
os.path.join(self.client_blob_manager.blob_dir, self.sd_hash))
|
||||
self.stream = ManagedStream(self.loop, self.client_config, self.client_blob_manager, self.sd_hash,
|
||||
self.client_dir)
|
||||
|
||||
for blob_info in descriptor.blobs[:-1]:
|
||||
shutil.copy(os.path.join(self.server_blob_manager.blob_dir, blob_info.blob_hash),
|
||||
os.path.join(self.client_blob_manager.blob_dir, blob_info.blob_hash))
|
||||
if corrupt and blob_info.length == MAX_BLOB_SIZE:
|
||||
with open(os.path.join(self.client_blob_manager.blob_dir, blob_info.blob_hash), "rb+") as handle:
|
||||
handle.truncate()
|
||||
handle.flush()
|
||||
await self.stream.save_file()
|
||||
await self.stream.finished_writing.wait()
|
||||
if corrupt:
|
||||
return self.assertFalse(os.path.isfile(os.path.join(self.client_dir, "test_file")))
|
||||
|
||||
with open(os.path.join(self.client_dir, "test_file"), "rb") as f:
|
||||
decrypted = f.read()
|
||||
self.assertEqual(decrypted, self.stream_bytes)
|
||||
|
||||
self.assertEqual(True, self.client_blob_manager.get_blob(self.sd_hash).get_is_verified())
|
||||
self.assertEqual(
|
||||
True, self.client_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified()
|
||||
)
|
||||
#
|
||||
# # its all blobs + sd blob - last blob, which is the same size as descriptor.blobs
|
||||
# self.assertEqual(len(descriptor.blobs), len(await downloader_storage.get_all_finished_blobs()))
|
||||
# self.assertEqual(
|
||||
# [descriptor.sd_hash, descriptor.blobs[0].blob_hash], await downloader_storage.get_blobs_to_announce()
|
||||
# )
|
||||
#
|
||||
# await downloader_storage.close()
|
||||
# await self.storage.close()
|
||||
|
||||
async def test_create_and_decrypt_multi_blob_stream(self):
|
||||
await self.test_create_and_decrypt_one_blob_stream(10)
|
||||
|
||||
# async def test_create_truncate_and_handle_stream(self):
|
||||
# # The purpose of this test is just to make sure it can finish even if a blob is corrupt/truncated
|
||||
# await asyncio.wait_for(self.test_create_and_decrypt_one_blob_stream(corrupt=True), timeout=5)
|
||||
@@ -5,7 +5,7 @@ import shutil
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
from lbrynet.stream.stream_manager import StreamManager
|
||||
from lbrynet.stream.reflector.server import ReflectorServer
|
||||
|
||||
@@ -18,16 +18,18 @@ class TestStreamAssembler(AsyncioTestCase):
|
||||
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
|
||||
self.storage = SQLiteStorage(Config(), os.path.join(tmp_dir, "lbrynet.sqlite"))
|
||||
self.conf = Config()
|
||||
self.storage = SQLiteStorage(self.conf, os.path.join(tmp_dir, "lbrynet.sqlite"))
|
||||
await self.storage.open()
|
||||
self.blob_manager = BlobFileManager(self.loop, tmp_dir, self.storage)
|
||||
self.blob_manager = BlobManager(self.loop, tmp_dir, self.storage, self.conf)
|
||||
self.stream_manager = StreamManager(self.loop, Config(), self.blob_manager, None, self.storage, None)
|
||||
|
||||
server_tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(server_tmp_dir))
|
||||
self.server_storage = SQLiteStorage(Config(), os.path.join(server_tmp_dir, "lbrynet.sqlite"))
|
||||
self.server_conf = Config()
|
||||
self.server_storage = SQLiteStorage(self.server_conf, os.path.join(server_tmp_dir, "lbrynet.sqlite"))
|
||||
await self.server_storage.open()
|
||||
self.server_blob_manager = BlobFileManager(self.loop, server_tmp_dir, self.server_storage)
|
||||
self.server_blob_manager = BlobManager(self.loop, server_tmp_dir, self.server_storage, self.server_conf)
|
||||
|
||||
download_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(download_dir))
|
||||
@@ -60,3 +62,8 @@ class TestStreamAssembler(AsyncioTestCase):
|
||||
|
||||
sent = await self.stream.upload_to_reflector('127.0.0.1', 5566)
|
||||
self.assertListEqual(sent, [])
|
||||
|
||||
async def test_announces(self):
|
||||
to_announce = await self.storage.get_blobs_to_announce()
|
||||
self.assertIn(self.stream.sd_hash, to_announce, "sd blob not set to announce")
|
||||
self.assertIn(self.stream.descriptor.blobs[0].blob_hash, to_announce, "head blob not set to announce")
|
||||
@@ -9,7 +9,7 @@ from torba.testcase import AsyncioTestCase
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.error import InvalidStreamDescriptorError
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_manager import BlobManager
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
|
||||
|
||||
@@ -20,9 +20,10 @@ class TestStreamDescriptor(AsyncioTestCase):
|
||||
self.cleartext = os.urandom(20000000)
|
||||
self.tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(self.tmp_dir))
|
||||
self.storage = SQLiteStorage(Config(), ":memory:")
|
||||
self.conf = Config()
|
||||
self.storage = SQLiteStorage(self.conf, ":memory:")
|
||||
await self.storage.open()
|
||||
self.blob_manager = BlobFileManager(self.loop, self.tmp_dir, self.storage)
|
||||
self.blob_manager = BlobManager(self.loop, self.tmp_dir, self.storage, self.conf)
|
||||
|
||||
self.file_path = os.path.join(self.tmp_dir, "test_file")
|
||||
with open(self.file_path, 'wb') as f:
|
||||
@@ -83,9 +84,10 @@ class TestRecoverOldStreamDescriptors(AsyncioTestCase):
|
||||
loop = asyncio.get_event_loop()
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(tmp_dir))
|
||||
storage = SQLiteStorage(Config(), ":memory:")
|
||||
self.conf = Config()
|
||||
storage = SQLiteStorage(self.conf, ":memory:")
|
||||
await storage.open()
|
||||
blob_manager = BlobFileManager(loop, tmp_dir, storage)
|
||||
blob_manager = BlobManager(loop, tmp_dir, storage, self.conf)
|
||||
|
||||
sd_bytes = b'{"stream_name": "4f62616d6120446f6e6b65792d322e73746c", "blobs": [{"length": 1153488, "blob_num' \
|
||||
b'": 0, "blob_hash": "9fa32a249ce3f2d4e46b78599800f368b72f2a7f22b81df443c7f6bdbef496bd61b4c0079c7' \
|
||||
@@ -99,7 +101,7 @@ class TestRecoverOldStreamDescriptors(AsyncioTestCase):
|
||||
|
||||
blob = blob_manager.get_blob(sd_hash)
|
||||
blob.set_length(len(sd_bytes))
|
||||
writer = blob.open_for_writing()
|
||||
writer = blob.get_blob_writer()
|
||||
writer.write(sd_bytes)
|
||||
await blob.verified.wait()
|
||||
descriptor = await StreamDescriptor.from_stream_descriptor_blob(
|
||||
@@ -116,7 +118,7 @@ class TestRecoverOldStreamDescriptors(AsyncioTestCase):
|
||||
sd_hash = '9313d1807551186126acc3662e74d9de29cede78d4f133349ace846273ef116b9bb86be86c54509eb84840e4b032f6b2'
|
||||
with open(os.path.join(tmp_dir, sd_hash), 'wb') as handle:
|
||||
handle.write(b'doesnt work')
|
||||
blob = BlobFile(loop, tmp_dir, sd_hash)
|
||||
blob = BlobFile(loop, sd_hash, blob_directory=tmp_dir)
|
||||
self.assertTrue(blob.file_exists)
|
||||
self.assertIsNotNone(blob.length)
|
||||
with self.assertRaises(InvalidStreamDescriptorError):
|
||||
|
||||
@@ -45,7 +45,7 @@ def get_mock_wallet(sd_hash, storage, balance=10.0, fee=None):
|
||||
"claim_id": "c49566d631226492317d06ad7fdbe1ed32925124",
|
||||
"claim_sequence": 1,
|
||||
"decoded_claim": True,
|
||||
"depth": 1057,
|
||||
"confirmations": 1057,
|
||||
"effective_amount": "0.1",
|
||||
"has_signature": False,
|
||||
"height": 514081,
|
||||
@@ -63,11 +63,11 @@ def get_mock_wallet(sd_hash, storage, balance=10.0, fee=None):
|
||||
elif fee['currency'] == 'USD':
|
||||
claim_obj.stream.fee.usd = Decimal(fee['amount'])
|
||||
claim_obj.stream.title = "33rpm"
|
||||
claim_obj.stream.language = "en"
|
||||
claim_obj.stream.sd_hash = sd_hash
|
||||
claim_obj.stream.media_type = "image/png"
|
||||
claim_obj.stream.languages.append("en")
|
||||
claim_obj.stream.source.sd_hash = sd_hash
|
||||
claim_obj.stream.source.media_type = "image/png"
|
||||
claim['value'] = claim_obj
|
||||
claim['hex'] = binascii.hexlify(claim_obj.to_bytes()).decode()
|
||||
claim['protobuf'] = binascii.hexlify(claim_obj.to_bytes()).decode()
|
||||
|
||||
async def mock_resolve(*args):
|
||||
await storage.save_claims([claim])
|
||||
@@ -192,8 +192,8 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
def check_post(event):
|
||||
self.assertEqual(event['event'], 'Time To First Bytes')
|
||||
self.assertEqual(event['properties']['error'], 'DownloadSDTimeout')
|
||||
self.assertEqual(event['properties']['tried_peers_count'], None)
|
||||
self.assertEqual(event['properties']['active_peer_count'], None)
|
||||
self.assertEqual(event['properties']['tried_peers_count'], 0)
|
||||
self.assertEqual(event['properties']['active_peer_count'], 0)
|
||||
self.assertEqual(event['properties']['use_fixed_peers'], False)
|
||||
self.assertEqual(event['properties']['added_fixed_peers'], False)
|
||||
self.assertEqual(event['properties']['fixed_peer_delay'], None)
|
||||
@@ -213,10 +213,10 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
|
||||
self.stream_manager.analytics_manager._post = check_post
|
||||
|
||||
self.assertSetEqual(self.stream_manager.streams, set())
|
||||
self.assertDictEqual(self.stream_manager.streams, {})
|
||||
stream = await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager)
|
||||
stream_hash = stream.stream_hash
|
||||
self.assertSetEqual(self.stream_manager.streams, {stream})
|
||||
self.assertDictEqual(self.stream_manager.streams, {stream.sd_hash: stream})
|
||||
self.assertTrue(stream.running)
|
||||
self.assertFalse(stream.finished)
|
||||
self.assertTrue(os.path.isfile(os.path.join(self.client_dir, "test_file")))
|
||||
@@ -225,7 +225,7 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
)
|
||||
self.assertEqual(stored_status, "running")
|
||||
|
||||
await self.stream_manager.stop_stream(stream)
|
||||
await stream.stop()
|
||||
|
||||
self.assertFalse(stream.finished)
|
||||
self.assertFalse(stream.running)
|
||||
@@ -235,8 +235,8 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
)
|
||||
self.assertEqual(stored_status, "stopped")
|
||||
|
||||
await self.stream_manager.start_stream(stream)
|
||||
await stream.downloader.stream_finished_event.wait()
|
||||
await stream.save_file(node=self.stream_manager.node)
|
||||
await stream.finished_writing.wait()
|
||||
await asyncio.sleep(0, loop=self.loop)
|
||||
self.assertTrue(stream.finished)
|
||||
self.assertFalse(stream.running)
|
||||
@@ -247,7 +247,7 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
self.assertEqual(stored_status, "finished")
|
||||
|
||||
await self.stream_manager.delete_stream(stream, True)
|
||||
self.assertSetEqual(self.stream_manager.streams, set())
|
||||
self.assertDictEqual(self.stream_manager.streams, {})
|
||||
self.assertFalse(os.path.isfile(os.path.join(self.client_dir, "test_file")))
|
||||
stored_status = await self.client_storage.run_and_return_one_or_none(
|
||||
"select status from file where stream_hash=?", stream_hash
|
||||
@@ -257,7 +257,7 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
|
||||
async def _test_download_error_on_start(self, expected_error, timeout=None):
|
||||
with self.assertRaises(expected_error):
|
||||
await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager, timeout=timeout)
|
||||
await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager, timeout)
|
||||
|
||||
async def _test_download_error_analytics_on_start(self, expected_error, timeout=None):
|
||||
received = []
|
||||
@@ -321,9 +321,9 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
await self.setup_stream_manager(old_sort=old_sort)
|
||||
self.stream_manager.analytics_manager._post = check_post
|
||||
|
||||
self.assertSetEqual(self.stream_manager.streams, set())
|
||||
self.assertDictEqual(self.stream_manager.streams, {})
|
||||
stream = await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager)
|
||||
await stream.downloader.stream_finished_event.wait()
|
||||
await stream.finished_writing.wait()
|
||||
await asyncio.sleep(0, loop=self.loop)
|
||||
self.stream_manager.stop()
|
||||
self.client_blob_manager.stop()
|
||||
@@ -333,8 +333,11 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
await self.client_blob_manager.setup()
|
||||
await self.stream_manager.start()
|
||||
self.assertEqual(1, len(self.stream_manager.streams))
|
||||
self.assertEqual(stream.sd_hash, list(self.stream_manager.streams)[0].sd_hash)
|
||||
self.assertEqual('stopped', list(self.stream_manager.streams)[0].status)
|
||||
self.assertListEqual([self.sd_hash], list(self.stream_manager.streams.keys()))
|
||||
for blob_hash in [stream.sd_hash] + [b.blob_hash for b in stream.descriptor.blobs[:-1]]:
|
||||
blob_status = await self.client_storage.get_blob_status(blob_hash)
|
||||
self.assertEqual('pending', blob_status)
|
||||
self.assertEqual('finished', self.stream_manager.streams[self.sd_hash].status)
|
||||
|
||||
sd_blob = self.client_blob_manager.get_blob(stream.sd_hash)
|
||||
self.assertTrue(sd_blob.file_exists)
|
||||
|
||||
@@ -61,7 +61,7 @@ class CLITest(AsyncioTestCase):
|
||||
# publish is ungrouped command, returns usage only implicitly
|
||||
self.assertIn('publish (<name> | --name=<name>)', self.shell(['publish']))
|
||||
# publish is ungrouped command, with explicit --help
|
||||
self.assertIn('Create or update a stream claim at a given name', self.shell(['publish', '--help']))
|
||||
self.assertIn('Create or replace a stream claim at a given name', self.shell(['publish', '--help']))
|
||||
# account is a group, returns help implicitly
|
||||
self.assertIn('Return the balance of an account', self.shell(['account']))
|
||||
# account is a group, with explicit --help
|
||||
|
||||
@@ -95,3 +95,20 @@ class TestAccount(AsyncioTestCase):
|
||||
account = Account.from_dict(self.ledger, Wallet(), account_data)
|
||||
account_data['ledger'] = 'lbc_mainnet'
|
||||
self.assertDictEqual(account_data, account.to_dict())
|
||||
|
||||
async def test_save_max_gap(self):
|
||||
account = Account.generate(
|
||||
self.ledger, Wallet(), 'lbryum', {
|
||||
'name': 'deterministic-chain',
|
||||
'receiving': {'gap': 3, 'maximum_uses_per_address': 2},
|
||||
'change': {'gap': 4, 'maximum_uses_per_address': 2}
|
||||
}
|
||||
)
|
||||
self.assertEqual(account.receiving.gap, 3)
|
||||
self.assertEqual(account.change.gap, 4)
|
||||
await account.save_max_gap()
|
||||
self.assertEqual(account.receiving.gap, 20)
|
||||
self.assertEqual(account.change.gap, 6)
|
||||
# doesn't fail for single-address account
|
||||
account2 = Account.generate(self.ledger, Wallet(), 'lbryum', {'name': 'single-address'})
|
||||
await account2.save_max_gap()
|
||||
Reference in new issue
Block a user