Compare commits
87
Commits
android
...
v0.31.0rc4
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
91e581422c | ||
|
|
b54d7185a1 | ||
|
|
f2968aab22 | ||
|
|
4e27eaf8e6 | ||
|
|
ccbe8aa0eb | ||
|
|
478f79353a | ||
|
|
c3b0151943 | ||
|
|
ba818354af | ||
|
|
e96b75a0d0 | ||
|
|
efe4afd09e | ||
|
|
c7cb6822cc | ||
|
|
c75665d3f0 | ||
|
|
744375b2c0 | ||
|
|
c1b4a012ec | ||
|
|
40da5b6cb6 | ||
|
|
b36cf3e36c | ||
|
|
4e3b529aec | ||
|
|
0a08f445a4 | ||
|
|
3eac99c509 | ||
|
|
f5bf1463af | ||
|
|
ca4a55ef28 | ||
|
|
832be0758b | ||
|
|
25050fdeeb | ||
|
|
df0635103e | ||
|
|
dabd9cbe1e | ||
|
|
42d7fbdbb3 | ||
|
|
2336015f9a | ||
|
|
5a28128b69 | ||
|
|
31445c7797 | ||
|
|
16d0ff8376 | ||
|
|
87f5d1bd4a | ||
|
|
be6e8744b0 | ||
|
|
2e9acd1baa | ||
|
|
607ccefe90 | ||
|
|
df86937f4e | ||
|
|
957d0290df | ||
|
|
40b90ce79b | ||
|
|
aa7f0845cc | ||
|
|
6e69d306de | ||
|
|
d85d3dfaa4 | ||
|
|
2b035009ef | ||
|
|
f9fd62c214 | ||
|
|
e8a1953815 | ||
|
|
34776e7eb9 | ||
|
|
a4419f562f | ||
|
|
c0160ab4e9 | ||
|
|
e60e5b3919 | ||
|
|
3589cc9977 | ||
|
|
2e978c00b2 | ||
|
|
2335243de9 | ||
|
|
6ad68eabd9 | ||
|
|
ca5c638124 | ||
|
|
262b9a624b | ||
|
|
05915c540a | ||
|
|
6aef6a80b7 | ||
|
|
16efe9ba95 | ||
|
|
fcda9500fe | ||
|
|
64dec821b7 | ||
|
|
d5f6e58aad | ||
|
|
087b65522e | ||
|
|
5ee4b2173a | ||
|
|
3e1c3d1e4b | ||
|
|
21f2b2dcd7 | ||
|
|
ffc7fc0793 | ||
|
|
15dbe5e54e | ||
|
|
2e633c0ee6 | ||
|
|
933d58d49b | ||
|
|
8f7cf0b38f | ||
|
|
adf378b6e9 | ||
|
|
1a4f083e76 | ||
|
|
7bde09dcf8 | ||
|
|
4630fd38aa | ||
|
|
418def56ac | ||
|
|
cefe3eb520 | ||
|
|
64dffa306f | ||
|
|
7d33b4f1f3 | ||
|
|
f507d95198 | ||
|
|
2a7d80f8b4 | ||
|
|
305034ffb2 | ||
|
|
bfaebf0085 | ||
|
|
a94cf04e87 | ||
|
|
71f9f8ae9c | ||
|
|
330862e487 | ||
|
|
778d3826ab | ||
|
|
4a11cf007f | ||
|
|
f6e7700c4c | ||
|
|
573285a5bb |
No files matched your search
@@ -1,12 +1,12 @@
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://travis-ci.org/lbryio/lbry) [](https://codecov.io/gh/lbryio/lbry)
|
||||
|
||||
LBRY is a decentralized peer-to-peer network providing distribution, discovery, and purchase of digital content (data). It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
|
||||
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
|
||||
|
||||
* Built on Python 3.7+ and `asyncio`.
|
||||
* DHT (Distributed Hash Table) implementation for finding peers ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
|
||||
* Blob exchange protocol for downloading content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
|
||||
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
|
||||
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
|
||||
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbrynet.schema](https://github.com/lbryio/lbry/tree/master/lbrynet/schema)).
|
||||
* Wallet implementation for the LBRY blockchain ([lbrynet.extras.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/wallet)).
|
||||
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbrynet.extras.daemon](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/daemon)).
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
__name__ = "lbrynet"
|
||||
__version__ = "0.31.0rc3"
|
||||
__version__ = "0.31.0rc4"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
@@ -77,7 +77,7 @@ class BlobFile:
|
||||
def writer_finished(self, writer: HashBlobWriter):
|
||||
def callback(finished: asyncio.Future):
|
||||
try:
|
||||
error = finished.result()
|
||||
error = finished.exception()
|
||||
except Exception as err:
|
||||
error = err
|
||||
if writer in self.writers: # remove this download attempt
|
||||
@@ -86,7 +86,7 @@ class BlobFile:
|
||||
while self.writers:
|
||||
other = self.writers.pop()
|
||||
other.finished.cancel()
|
||||
t = self.loop.create_task(self.save_verified_blob(writer))
|
||||
t = self.loop.create_task(self.save_verified_blob(writer, finished.result()))
|
||||
t.add_done_callback(lambda *_: self.finished_writing.set())
|
||||
return
|
||||
if isinstance(error, (InvalidBlobHashError, InvalidDataError)):
|
||||
@@ -96,24 +96,24 @@ class BlobFile:
|
||||
raise error
|
||||
return callback
|
||||
|
||||
async def save_verified_blob(self, writer):
|
||||
async def save_verified_blob(self, writer, verified_bytes: bytes):
|
||||
def _save_verified():
|
||||
# log.debug(f"write blob file {self.blob_hash[:8]} from {writer.peer.address}")
|
||||
if not self.saved_verified_blob and not os.path.isfile(self.file_path):
|
||||
if self.get_length() == len(writer.verified_bytes):
|
||||
if self.get_length() == len(verified_bytes):
|
||||
with open(self.file_path, 'wb') as write_handle:
|
||||
write_handle.write(writer.verified_bytes)
|
||||
write_handle.write(verified_bytes)
|
||||
self.saved_verified_blob = True
|
||||
else:
|
||||
raise Exception("length mismatch")
|
||||
|
||||
if self.verified.is_set():
|
||||
return
|
||||
async with self.blob_write_lock:
|
||||
if self.verified.is_set():
|
||||
return
|
||||
await self.loop.run_in_executor(None, _save_verified)
|
||||
self.verified.set()
|
||||
if self.blob_completed_callback:
|
||||
await self.blob_completed_callback(self)
|
||||
self.verified.set()
|
||||
|
||||
def open_for_writing(self) -> HashBlobWriter:
|
||||
if os.path.exists(self.file_path):
|
||||
|
||||
@@ -18,7 +18,6 @@ class HashBlobWriter:
|
||||
self.finished.add_done_callback(lambda *_: self.close_handle())
|
||||
self._hashsum = get_lbry_hash_obj()
|
||||
self.len_so_far = 0
|
||||
self.verified_bytes = b''
|
||||
|
||||
def __del__(self):
|
||||
if self.buffer is not None:
|
||||
@@ -46,7 +45,7 @@ class HashBlobWriter:
|
||||
self.len_so_far += len(data)
|
||||
if self.len_so_far > expected_length:
|
||||
self.close_handle()
|
||||
self.finished.set_result(InvalidDataError(
|
||||
self.finished.set_exception(InvalidDataError(
|
||||
f'Length so far is greater than the expected length. {self.len_so_far} to {expected_length}'
|
||||
))
|
||||
return
|
||||
@@ -55,15 +54,12 @@ class HashBlobWriter:
|
||||
blob_hash = self.calculate_blob_hash()
|
||||
if blob_hash != self.expected_blob_hash:
|
||||
self.close_handle()
|
||||
self.finished.set_result(InvalidBlobHashError(
|
||||
self.finished.set_exception(InvalidBlobHashError(
|
||||
f"blob hash is {blob_hash} vs expected {self.expected_blob_hash}"
|
||||
))
|
||||
return
|
||||
self.buffer.seek(0)
|
||||
self.verified_bytes = self.buffer.read()
|
||||
elif self.finished and not (self.finished.done() or self.finished.cancelled()):
|
||||
self.finished.set_result(self.buffer.getvalue())
|
||||
self.close_handle()
|
||||
if self.finished and not (self.finished.done() or self.finished.cancelled()):
|
||||
self.finished.set_result(None)
|
||||
|
||||
def close_handle(self):
|
||||
if self.buffer is not None:
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import typing
|
||||
import binascii
|
||||
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
@@ -19,15 +20,15 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
|
||||
self.writer: 'HashBlobWriter' = None
|
||||
self.blob: 'BlobFile' = None
|
||||
self.download_running = asyncio.Event(loop=self.loop)
|
||||
|
||||
self._blob_bytes_received = 0
|
||||
self._response_fut: asyncio.Future = None
|
||||
self._request_lock = asyncio.Lock(loop=self.loop)
|
||||
|
||||
def handle_data_received(self, data: bytes):
|
||||
if self.transport.is_closing():
|
||||
if self._response_fut and not (self._response_fut.done() or self._response_fut.cancelled()):
|
||||
def data_received(self, data: bytes):
|
||||
if not self.transport or self.transport.is_closing():
|
||||
log.warning("transport closing, but got more bytes from %s:%i\n%s", self.peer_address, self.peer_port,
|
||||
binascii.hexlify(data))
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return
|
||||
|
||||
@@ -36,60 +37,74 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
if response.responses and self.blob:
|
||||
blob_response = response.get_blob_response()
|
||||
if blob_response and not blob_response.error and blob_response.blob_hash == self.blob.blob_hash:
|
||||
# set the expected length for the incoming blob if we didn't know it
|
||||
self.blob.set_length(blob_response.length)
|
||||
elif blob_response and not blob_response.error and self.blob.blob_hash != blob_response.blob_hash:
|
||||
# the server started sending a blob we didn't request
|
||||
log.warning("mismatch with self.blob %s", self.blob.blob_hash)
|
||||
return
|
||||
if response.responses:
|
||||
log.debug("got response from %s:%i <- %s", self.peer_address, self.peer_port, response.to_dict())
|
||||
# fire the Future with the response to our request
|
||||
self._response_fut.set_result(response)
|
||||
if response.blob_data and self.writer and not self.writer.closed():
|
||||
log.debug("got %i blob bytes from %s:%i", len(response.blob_data), self.peer_address, self.peer_port)
|
||||
# write blob bytes if we're writing a blob and have blob bytes to write
|
||||
self._blob_bytes_received += len(response.blob_data)
|
||||
try:
|
||||
self.writer.write(response.blob_data)
|
||||
return
|
||||
except IOError as err:
|
||||
log.error("error downloading blob: %s", err)
|
||||
|
||||
def data_received(self, data):
|
||||
try:
|
||||
return self.handle_data_received(data)
|
||||
except (asyncio.CancelledError, asyncio.TimeoutError) as err:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.set_exception(err)
|
||||
log.error("error downloading blob from %s:%i: %s", self.peer_address, self.peer_port, err)
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.set_exception(err)
|
||||
except (asyncio.CancelledError, asyncio.TimeoutError) as err: # TODO: is this needed?
|
||||
log.error("%s downloading blob from %s:%i", str(err), self.peer_address, self.peer_port)
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.set_exception(err)
|
||||
|
||||
async def _download_blob(self) -> typing.Tuple[bool, bool]:
|
||||
"""
|
||||
:return: download success (bool), keep connection (bool)
|
||||
"""
|
||||
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
|
||||
try:
|
||||
self.transport.write(request.serialize())
|
||||
msg = request.serialize()
|
||||
log.debug("send request to %s:%i -> %s", self.peer_address, self.peer_port, msg.decode())
|
||||
self.transport.write(msg)
|
||||
response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop)
|
||||
availability_response = response.get_availability_response()
|
||||
price_response = response.get_price_response()
|
||||
blob_response = response.get_blob_response()
|
||||
if (not blob_response or blob_response.error) and\
|
||||
(not availability_response or not availability_response.available_blobs):
|
||||
log.warning("blob not in availability response")
|
||||
return False, True
|
||||
log.warning("%s not in availability response from %s:%i", self.blob.blob_hash, self.peer_address,
|
||||
self.peer_port)
|
||||
log.warning(response.to_dict())
|
||||
return False, False
|
||||
elif availability_response.available_blobs and \
|
||||
availability_response.available_blobs != [self.blob.blob_hash]:
|
||||
log.warning("blob availability response doesn't match our request")
|
||||
log.warning("blob availability response doesn't match our request from %s:%i",
|
||||
self.peer_address, self.peer_port)
|
||||
return False, False
|
||||
if not price_response or price_response.blob_data_payment_rate != 'RATE_ACCEPTED':
|
||||
log.warning("data rate rejected")
|
||||
log.warning("data rate rejected by %s:%i", self.peer_address, self.peer_port)
|
||||
return False, False
|
||||
if not blob_response or blob_response.error:
|
||||
log.warning("blob cant be downloaded from this peer")
|
||||
log.warning("blob cant be downloaded from %s:%i", self.peer_address, self.peer_port)
|
||||
return False, True
|
||||
if not blob_response.error and blob_response.blob_hash != self.blob.blob_hash:
|
||||
log.warning("incoming blob hash mismatch")
|
||||
log.warning("incoming blob hash mismatch from %s:%i", self.peer_address, self.peer_port)
|
||||
return False, False
|
||||
if self.blob.length is not None and self.blob.length != blob_response.length:
|
||||
log.warning("incoming blob unexpected length")
|
||||
log.warning("incoming blob unexpected length from %s:%i", self.peer_address, self.peer_port)
|
||||
return False, False
|
||||
msg = f"downloading {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}," \
|
||||
f" timeout in {self.peer_timeout}"
|
||||
log.info(msg)
|
||||
log.debug(msg)
|
||||
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
|
||||
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
|
||||
await self.blob.finished_writing.wait()
|
||||
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}"
|
||||
log.info(msg)
|
||||
return True, True
|
||||
except asyncio.CancelledError:
|
||||
@@ -104,9 +119,6 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
self._response_fut.cancel()
|
||||
if self.writer and not self.writer.closed():
|
||||
self.writer.close_handle()
|
||||
if self.blob:
|
||||
await self.blob.close()
|
||||
self.download_running.clear()
|
||||
self._response_fut = None
|
||||
self.writer = None
|
||||
self.blob = None
|
||||
@@ -117,52 +129,49 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
async def download_blob(self, blob: 'BlobFile') -> typing.Tuple[bool, bool]:
|
||||
if blob.get_is_verified():
|
||||
return False, True
|
||||
async with self._request_lock:
|
||||
try:
|
||||
if self.download_running.is_set():
|
||||
log.info("wait for download already running")
|
||||
await self.download_running.wait()
|
||||
self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0
|
||||
self.download_running.set()
|
||||
self._response_fut = asyncio.Future(loop=self.loop)
|
||||
return await self._download_blob()
|
||||
except OSError:
|
||||
log.error("race happened")
|
||||
# i'm not sure how to fix this race condition - jack
|
||||
return False, True
|
||||
except asyncio.TimeoutError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return False, False
|
||||
except asyncio.CancelledError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return False, True
|
||||
try:
|
||||
self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0
|
||||
self._response_fut = asyncio.Future(loop=self.loop)
|
||||
return await self._download_blob()
|
||||
except OSError:
|
||||
log.error("race happened downloading from %s:%i", self.peer_address, self.peer_port)
|
||||
# i'm not sure how to fix this race condition - jack
|
||||
return False, True
|
||||
except asyncio.TimeoutError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return False, False
|
||||
except asyncio.CancelledError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return False, True
|
||||
|
||||
def connection_made(self, transport: asyncio.Transport):
|
||||
self.transport = transport
|
||||
self.peer_address, self.peer_port = self.transport.get_extra_info('peername')
|
||||
# log.info("connection made to %s: %s", self.peer_address, transport)
|
||||
log.debug("connection made to %s:%i", self.peer_address, self.peer_port)
|
||||
|
||||
def connection_lost(self, reason):
|
||||
# log.info("connection lost to %s (reason: %s)", self.peer_address, reason)
|
||||
log.debug("connection lost to %s:%i (reason: %s, %s)", self.peer_address, self.peer_port, str(reason),
|
||||
str(type(reason)))
|
||||
self.transport = None
|
||||
self.loop.create_task(self.close())
|
||||
|
||||
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', protocol: 'BlobExchangeClientProtocol',
|
||||
address: str, tcp_port: int, peer_connect_timeout: float) -> typing.Tuple[bool, bool]:
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', address: str, tcp_port: int,
|
||||
peer_connect_timeout: float, blob_download_timeout: float) -> typing.Tuple[bool, bool]:
|
||||
"""
|
||||
Returns [<downloaded blob>, <keep connection>]
|
||||
"""
|
||||
|
||||
protocol = BlobExchangeClientProtocol(loop, blob_download_timeout)
|
||||
if blob.get_is_verified():
|
||||
return False, True
|
||||
if blob.get_is_verified():
|
||||
log.info("already verified")
|
||||
return False, True
|
||||
try:
|
||||
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
|
||||
peer_connect_timeout, loop=loop)
|
||||
return await protocol.download_blob(blob)
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError, ConnectionRefusedError, ConnectionAbortedError, OSError):
|
||||
return False, False
|
||||
finally:
|
||||
await protocol.close()
|
||||
@@ -1,10 +1,10 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet import conf
|
||||
from lbrynet.utils import drain_tasks
|
||||
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
|
||||
from lbrynet.blob_exchange.client import request_blob
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
@@ -18,115 +18,94 @@ def drain_into(a: list, b: list):
|
||||
b.append(a.pop())
|
||||
|
||||
|
||||
class BlobDownloader: # TODO: refactor to be the base class used by StreamDownloader
|
||||
"""A single blob downloader"""
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', config: conf.Config):
|
||||
class BlobDownloader:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager',
|
||||
peer_queue: asyncio.Queue):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
self.blob_manager = blob_manager
|
||||
self.new_peer_event = asyncio.Event(loop=self.loop)
|
||||
self.active_connections: typing.Dict['KademliaPeer', BlobExchangeClientProtocol] = {}
|
||||
self.running_download_requests: typing.List[asyncio.Task] = []
|
||||
self.requested_from: typing.Dict[str, typing.Dict['KademliaPeer', asyncio.Task]] = {}
|
||||
self.lock = asyncio.Lock(loop=self.loop)
|
||||
self.blob: 'BlobFile' = None
|
||||
self.blob_queue = asyncio.Queue(loop=self.loop)
|
||||
self.peer_queue = peer_queue
|
||||
self.active_connections: typing.Dict['KademliaPeer', asyncio.Task] = {} # active request_blob calls
|
||||
self.ignored: typing.Set['KademliaPeer'] = set()
|
||||
self.scores: typing.Dict['KademliaPeer', int] = {}
|
||||
|
||||
self.blob_download_timeout = config.get('blob_download_timeout')
|
||||
self.peer_connect_timeout = config.get('peer_connect_timeout')
|
||||
self.max_connections = config.get('max_connections_per_stream')
|
||||
def request_blob_from_peer(self, blob: 'BlobFile', peer: 'KademliaPeer'):
|
||||
async def _request_blob():
|
||||
if blob.get_is_verified():
|
||||
return
|
||||
try:
|
||||
success, keep_connection = await request_blob(
|
||||
self.loop, blob, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
|
||||
self.config.blob_download_timeout
|
||||
)
|
||||
finally:
|
||||
if peer in self.active_connections:
|
||||
self.active_connections.pop(peer)
|
||||
if not keep_connection and peer not in self.ignored:
|
||||
self.ignored.add(peer)
|
||||
log.debug("drop peer %s:%i", peer.address, peer.tcp_port)
|
||||
elif keep_connection:
|
||||
log.debug("keep peer %s:%i", peer.address, peer.tcp_port)
|
||||
if success:
|
||||
self.scores[peer] = self.scores.get(peer, 0) + 2
|
||||
else:
|
||||
self.scores[peer] = self.scores.get(peer, 0) - 1
|
||||
return self.loop.create_task(_request_blob())
|
||||
|
||||
async def _request_blob(self, peer: 'KademliaPeer'):
|
||||
if self.blob.get_is_verified():
|
||||
log.info("already verified")
|
||||
return
|
||||
if peer not in self.active_connections:
|
||||
log.warning("not active, adding: %s", str(peer))
|
||||
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.blob_download_timeout)
|
||||
protocol = self.active_connections[peer]
|
||||
success, keep_connection = await request_blob(self.loop, self.blob, protocol, peer.address, peer.tcp_port,
|
||||
self.peer_connect_timeout)
|
||||
await protocol.close()
|
||||
if not keep_connection:
|
||||
log.info("drop peer %s:%i", peer.address, peer.tcp_port)
|
||||
if peer in self.active_connections:
|
||||
async with self.lock:
|
||||
del self.active_connections[peer]
|
||||
return
|
||||
log.info("keep peer %s:%i", peer.address, peer.tcp_port)
|
||||
|
||||
def _update_requests(self):
|
||||
self.new_peer_event.clear()
|
||||
if self.blob.blob_hash not in self.requested_from:
|
||||
self.requested_from[self.blob.blob_hash] = {}
|
||||
to_add = []
|
||||
for peer in self.active_connections.keys():
|
||||
if peer not in self.requested_from[self.blob.blob_hash] and peer not in to_add:
|
||||
to_add.append(peer)
|
||||
if to_add or self.running_download_requests:
|
||||
log.info("adding download probes for %i peers to %i already active",
|
||||
min(len(to_add), 8 - len(self.running_download_requests)),
|
||||
len(self.running_download_requests))
|
||||
else:
|
||||
log.info("downloader idle...")
|
||||
for peer in to_add:
|
||||
if len(self.running_download_requests) >= 8:
|
||||
break
|
||||
task = self.loop.create_task(self._request_blob(peer))
|
||||
self.requested_from[self.blob.blob_hash][peer] = task
|
||||
self.running_download_requests.append(task)
|
||||
|
||||
def _add_peer_protocols(self, peers: typing.List['KademliaPeer']):
|
||||
added = 0
|
||||
for peer in peers:
|
||||
if peer not in self.active_connections:
|
||||
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.blob_download_timeout)
|
||||
added += 1
|
||||
if added:
|
||||
if not self.new_peer_event.is_set():
|
||||
log.info("added %i new peers", len(peers))
|
||||
self.new_peer_event.set()
|
||||
|
||||
async def _accumulate_connections(self, node: 'Node'):
|
||||
async def new_peer_or_finished(self, blob: 'BlobFile'):
|
||||
async def get_and_re_add_peers():
|
||||
new_peers = await self.peer_queue.get()
|
||||
self.peer_queue.put_nowait(new_peers)
|
||||
tasks = [self.loop.create_task(get_and_re_add_peers()), self.loop.create_task(blob.verified.wait())]
|
||||
try:
|
||||
async with node.stream_peer_search_junction(self.blob_queue) as search_junction:
|
||||
async for peers in search_junction:
|
||||
if not isinstance(peers, list): # TODO: what's up with this?
|
||||
log.error("not a list: %s %s", peers, str(type(peers)))
|
||||
else:
|
||||
self._add_peer_protocols(peers)
|
||||
return
|
||||
await asyncio.wait(tasks, loop=self.loop, return_when='FIRST_COMPLETED')
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
async def get_blob(self, blob_hash: str, node: 'Node') -> 'BlobFile':
|
||||
self.blob = self.blob_manager.get_blob(blob_hash)
|
||||
if self.blob.get_is_verified():
|
||||
return self.blob
|
||||
accumulator = self.loop.create_task(self._accumulate_connections(node))
|
||||
self.blob_queue.put_nowait(blob_hash)
|
||||
try:
|
||||
while not self.blob.get_is_verified():
|
||||
if len(self.running_download_requests) < self.max_connections:
|
||||
self._update_requests()
|
||||
|
||||
# drain the tasks into a temporary list
|
||||
download_tasks = []
|
||||
drain_into(self.running_download_requests, download_tasks)
|
||||
got_new_peer = self.loop.create_task(self.new_peer_event.wait())
|
||||
|
||||
# wait for a new peer to be added or for a download attempt to finish
|
||||
await asyncio.wait([got_new_peer] + download_tasks, return_when='FIRST_COMPLETED',
|
||||
loop=self.loop)
|
||||
if got_new_peer and not got_new_peer.done():
|
||||
got_new_peer.cancel()
|
||||
if self.blob.get_is_verified():
|
||||
if got_new_peer and not got_new_peer.done():
|
||||
got_new_peer.cancel()
|
||||
drain_tasks(download_tasks)
|
||||
return self.blob
|
||||
except asyncio.CancelledError:
|
||||
drain_tasks(self.running_download_requests)
|
||||
drain_tasks(tasks)
|
||||
raise
|
||||
finally:
|
||||
if accumulator and not accumulator.done():
|
||||
accumulator.cancel()
|
||||
|
||||
async def download_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
|
||||
blob = self.blob_manager.get_blob(blob_hash, length)
|
||||
if blob.get_is_verified():
|
||||
return blob
|
||||
try:
|
||||
while not blob.get_is_verified():
|
||||
batch: typing.List['KademliaPeer'] = []
|
||||
while not self.peer_queue.empty():
|
||||
batch.extend(await self.peer_queue.get())
|
||||
for peer in batch:
|
||||
if len(self.active_connections) >= self.config.max_connections_per_download:
|
||||
break
|
||||
if peer not in self.active_connections and peer not in self.ignored:
|
||||
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
|
||||
self.active_connections[peer] = self.request_blob_from_peer(blob, peer)
|
||||
await self.new_peer_or_finished(blob)
|
||||
to_re_add = list(set(filter(lambda peer: peer not in self.ignored, batch)))
|
||||
to_re_add.sort(key=lambda peer: self.scores.get(peer, 0), reverse=True)
|
||||
if to_re_add:
|
||||
self.peer_queue.put_nowait(to_re_add)
|
||||
while self.active_connections:
|
||||
peer, task = self.active_connections.popitem()
|
||||
if task and not task.done():
|
||||
task.cancel()
|
||||
await blob.close()
|
||||
log.info("downloaded %s", blob_hash[:8])
|
||||
return blob
|
||||
except asyncio.CancelledError:
|
||||
while self.active_connections:
|
||||
peer, task = self.active_connections.popitem()
|
||||
if task and not task.done():
|
||||
task.cancel()
|
||||
raise
|
||||
|
||||
|
||||
async def download_blob(loop, config: 'Config', blob_manager: 'BlobFileManager', node: 'Node',
|
||||
blob_hash: str) -> 'BlobFile':
|
||||
search_queue = asyncio.Queue(loop=loop, maxsize=config.max_connections_per_download)
|
||||
search_queue.put_nowait(blob_hash)
|
||||
peer_queue, accumulate_task = node.accumulate_peers(search_queue)
|
||||
downloader = BlobDownloader(loop, config, blob_manager, peer_queue)
|
||||
try:
|
||||
return await downloader.download_blob(blob_hash)
|
||||
finally:
|
||||
if accumulate_task and not accumulate_task.done():
|
||||
accumulate_task.cancel()
|
||||
@@ -146,32 +146,26 @@ def _parse_blob_response(response_msg: bytes) -> typing.Tuple[typing.Optional[ty
|
||||
# <blob bytes>
|
||||
# <json><blob bytes>
|
||||
|
||||
extra_data = b''
|
||||
response = None
|
||||
curr_pos = 0
|
||||
while True:
|
||||
next_close_paren = response_msg.find(b'}', curr_pos)
|
||||
if next_close_paren == -1:
|
||||
break
|
||||
return None, response_msg
|
||||
curr_pos = next_close_paren + 1
|
||||
try:
|
||||
response = json.loads(response_msg[:curr_pos])
|
||||
if not isinstance(response, dict):
|
||||
raise ValueError()
|
||||
for key in response.keys():
|
||||
if key not in [
|
||||
BlobPaymentAddressResponse.key,
|
||||
BlobAvailabilityResponse.key,
|
||||
BlobPriceResponse.key,
|
||||
BlobDownloadResponse.key]:
|
||||
raise ValueError()
|
||||
extra_data = response_msg[curr_pos:]
|
||||
break
|
||||
except ValueError:
|
||||
response = None
|
||||
if response is None:
|
||||
extra_data = response_msg
|
||||
return response, extra_data
|
||||
continue
|
||||
possible_response_keys = {
|
||||
BlobPaymentAddressResponse.key,
|
||||
BlobAvailabilityResponse.key,
|
||||
BlobPriceResponse.key,
|
||||
BlobDownloadResponse.key
|
||||
}
|
||||
if isinstance(response, dict) and response.keys():
|
||||
if set(response.keys()).issubset(possible_response_keys):
|
||||
return response, response_msg[curr_pos:]
|
||||
return None, response_msg
|
||||
|
||||
|
||||
class BlobRequest:
|
||||
|
||||
@@ -57,8 +57,13 @@ class BlobServerProtocol(asyncio.Protocol):
|
||||
incoming_blob = {'blob_hash': blob.blob_hash, 'length': blob.length}
|
||||
responses.append(BlobDownloadResponse(incoming_blob=incoming_blob))
|
||||
self.send_response(responses)
|
||||
log.info("send %s to %s:%i", blob.blob_hash[:8], peer_address, peer_port)
|
||||
sent = await blob.sendfile(self)
|
||||
log.debug("send %s to %s:%i", blob.blob_hash[:8], peer_address, peer_port)
|
||||
try:
|
||||
sent = await blob.sendfile(self)
|
||||
except (ConnectionResetError, BrokenPipeError, RuntimeError, OSError):
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
return
|
||||
log.info("sent %s (%i bytes) to %s:%i", blob.blob_hash[:8], sent, peer_address, peer_port)
|
||||
if responses:
|
||||
self.send_response(responses)
|
||||
@@ -72,7 +77,7 @@ class BlobServerProtocol(asyncio.Protocol):
|
||||
self.buf += data
|
||||
return
|
||||
try:
|
||||
request = BlobRequest.deserialize(data)
|
||||
request = BlobRequest.deserialize(self.buf + data)
|
||||
self.buf = remainder
|
||||
except JSONDecodeError:
|
||||
addr = self.transport.get_extra_info('peername')
|
||||
|
||||
+63
-64
@@ -16,29 +16,11 @@ log = logging.getLogger(__name__)
|
||||
NOT_SET = type(str('NOT_SET'), (object,), {})
|
||||
T = typing.TypeVar('T')
|
||||
|
||||
KB = 2 ** 10
|
||||
MB = 2 ** 20
|
||||
|
||||
ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
|
||||
ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
|
||||
API_ADDRESS = 'lbryapi'
|
||||
APP_NAME = 'LBRY'
|
||||
BLOBFILES_DIR = 'blobfiles'
|
||||
CRYPTSD_FILE_EXTENSION = '.cryptsd'
|
||||
CURRENCIES = {
|
||||
'BTC': {'type': 'crypto'},
|
||||
'LBC': {'type': 'crypto'},
|
||||
'USD': {'type': 'fiat'},
|
||||
}
|
||||
ICON_PATH = 'icons' if 'win' in sys.platform else 'app.icns'
|
||||
LOG_FILE_NAME = 'lbrynet.log'
|
||||
LOG_POST_URL = 'https://lbry.io/log-upload'
|
||||
MAX_BLOB_REQUEST_SIZE = 64 * KB
|
||||
MAX_HANDSHAKE_SIZE = 64 * KB
|
||||
MAX_REQUEST_SIZE = 64 * KB
|
||||
MAX_RESPONSE_INFO_SIZE = 64 * KB
|
||||
MAX_BLOB_INFOS_TO_REQUEST = 20
|
||||
PROTOCOL_PREFIX = 'lbry'
|
||||
SLACK_WEBHOOK = (
|
||||
'nUE0pUZ6Yl9bo29epl5moTSwnl5wo20ip2IlqzywMKZiIQSFZR5'
|
||||
'AHx4mY0VmF0WQZ1ESEP9kMHZlp1WzJwWOoKN3ImR1M2yUAaMyqGZ='
|
||||
@@ -471,7 +453,7 @@ class CLIConfig(BaseConfig):
|
||||
|
||||
|
||||
class Config(CLIConfig):
|
||||
|
||||
# directories
|
||||
data_dir = Path("Directory path to store blobs.", metavar='DIR')
|
||||
download_dir = Path(
|
||||
"Directory path to place assembled files downloaded from LBRY.",
|
||||
@@ -482,60 +464,77 @@ class Config(CLIConfig):
|
||||
previous_names=['lbryum_wallet_dir'], metavar='DIR'
|
||||
)
|
||||
|
||||
share_usage_data = Toggle(
|
||||
"Whether to share usage stats and diagnostic info with LBRY.", True,
|
||||
previous_names=['upload_log', 'upload_log', 'share_debug_info']
|
||||
# network
|
||||
use_upnp = Toggle(
|
||||
"Use UPnP to setup temporary port redirects for the DHT and the hosting of blobs. If you manually forward"
|
||||
"ports or have firewall rules you likely want to disable this.", True
|
||||
)
|
||||
udp_port = Integer("UDP port for communicating on the LBRY DHT", 4444, previous_names=['dht_node_port'])
|
||||
tcp_port = Integer("TCP port to listen for incoming blob requests", 3333, previous_names=['peer_port'])
|
||||
network_interface = String("Interface to use for the DHT and blob exchange", '0.0.0.0')
|
||||
|
||||
# protocol timeouts
|
||||
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
|
||||
blob_download_timeout = Float("Timeout to download a blob from a peer", 20.0)
|
||||
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0)
|
||||
node_rpc_timeout = Float("Timeout when making a DHT request", constants.rpc_timeout)
|
||||
|
||||
# blob announcement and download
|
||||
announce_head_and_sd_only = Toggle(
|
||||
"Announce only the descriptor and first (rather than all) data blob for a stream to the DHT", True,
|
||||
previous_names=['announce_head_blobs_only']
|
||||
)
|
||||
concurrent_blob_announcers = Integer(
|
||||
"Number of blobs to iteratively announce at once, set to 0 to disable", 10,
|
||||
previous_names=['concurrent_announcers']
|
||||
)
|
||||
max_connections_per_download = Integer(
|
||||
"Maximum number of peers to connect to while downloading a blob", 8,
|
||||
previous_names=['max_connections_per_stream']
|
||||
)
|
||||
fixed_peer_delay = Float(
|
||||
"Amount of seconds before adding the reflector servers as potential peers to download from in case dht"
|
||||
"peers are not found or are slow", 2.0
|
||||
)
|
||||
max_key_fee = MaxKeyFee(
|
||||
"Don't download streams with fees exceeding this amount", {'currency': 'USD', 'amount': 50.0}
|
||||
) # TODO: use this
|
||||
|
||||
# reflector settings
|
||||
reflect_streams = Toggle(
|
||||
"Upload completed streams (published and downloaded) reflector in order to re-host them", True,
|
||||
previous_names=['reflect_uploads']
|
||||
)
|
||||
concurrent_reflector_uploads = Integer(
|
||||
"Maximum number of streams to upload to a reflector server at a time", 10
|
||||
)
|
||||
|
||||
# claims set to expire within this many blocks will be
|
||||
# automatically renewed after startup (if set to 0, renews
|
||||
# will not be made automatically)
|
||||
auto_renew_claim_height_delta = Integer("", 0)
|
||||
cache_time = Integer("", 150)
|
||||
data_rate = Float("points/megabyte", .0001)
|
||||
delete_blobs_on_remove = Toggle("", True)
|
||||
dht_node_port = Integer("", 4444)
|
||||
download_timeout = Float("", 30.0)
|
||||
blob_download_timeout = Float("", 20.0)
|
||||
peer_connect_timeout = Float("", 3.0)
|
||||
node_rpc_timeout = Float("", constants.rpc_timeout)
|
||||
is_generous_host = Toggle("", True)
|
||||
announce_head_blobs_only = Toggle("", True)
|
||||
concurrent_announcers = Integer("", 10)
|
||||
known_dht_nodes = Servers("", [
|
||||
# servers
|
||||
reflector_servers = Servers("Reflector re-hosting servers", [
|
||||
('reflector.lbry.io', 5566)
|
||||
])
|
||||
lbryum_servers = Servers("SPV wallet servers", [
|
||||
('lbryumx1.lbry.io', 50001),
|
||||
('lbryumx2.lbry.io', 50001)
|
||||
])
|
||||
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
|
||||
('lbrynet1.lbry.io', 4444), # US EAST
|
||||
('lbrynet2.lbry.io', 4444), # US WEST
|
||||
('lbrynet3.lbry.io', 4444), # EU
|
||||
('lbrynet4.lbry.io', 4444) # ASIA
|
||||
])
|
||||
max_connections_per_stream = Integer("", 5)
|
||||
seek_head_blob_first = Toggle("", True)
|
||||
max_key_fee = MaxKeyFee("", {'currency': 'USD', 'amount': 50.0})
|
||||
min_info_rate = Float("points/1000 infos", .02)
|
||||
min_valuable_hash_rate = Float("points/1000 infos", .05)
|
||||
min_valuable_info_rate = Float("points/1000 infos", .05)
|
||||
peer_port = Integer("", 3333)
|
||||
pointtrader_server = String("", 'http://127.0.0.1:2424')
|
||||
reflector_port = Integer("", 5566)
|
||||
# if reflect_uploads is True, send files to reflector after publishing (as well as a periodic check in the
|
||||
# event the initial upload failed or was disconnected part way through, provided the auto_re_reflect_interval > 0)
|
||||
reflect_uploads = Toggle("", True)
|
||||
auto_re_reflect_interval = Integer("set to 0 to disable", 86400)
|
||||
reflector_servers = Servers("", [
|
||||
('reflector.lbry.io', 5566)
|
||||
])
|
||||
run_reflector_server = Toggle("adds reflector to components_to_skip unless True", False)
|
||||
sd_download_timeout = Integer("", 3)
|
||||
peer_search_timeout = Integer("", 60)
|
||||
use_upnp = Toggle("", True)
|
||||
use_keyring = Toggle("", False)
|
||||
blockchain_name = String("", 'lbrycrd_main')
|
||||
lbryum_servers = Servers("", [
|
||||
('lbryumx1.lbry.io', 50001),
|
||||
('lbryumx2.lbry.io', 50001)
|
||||
])
|
||||
|
||||
# blockchain
|
||||
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
|
||||
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
|
||||
cache_time = Integer("Time to cache resolved claims", 150) # TODO: use this
|
||||
|
||||
# daemon
|
||||
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
|
||||
share_usage_data = Toggle(
|
||||
"Whether to share usage stats and diagnostic info with LBRY.", True,
|
||||
previous_names=['upload_log', 'upload_log', 'share_debug_info']
|
||||
)
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
import time
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
@@ -9,7 +10,8 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobAnnouncer:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, node: 'Node', storage: 'SQLiteStorage'):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, node: 'Node', storage: 'SQLiteStorage',
|
||||
time_getter: typing.Callable[[], float] = time.time):
|
||||
self.loop = loop
|
||||
self.node = node
|
||||
self.storage = storage
|
||||
@@ -17,8 +19,11 @@ class BlobAnnouncer:
|
||||
self.announce_task: asyncio.Task = None
|
||||
self.running = False
|
||||
self.announce_queue: typing.List[str] = []
|
||||
self.time_getter = time_getter
|
||||
|
||||
async def _announce(self, batch_size: typing.Optional[int] = 10):
|
||||
if not batch_size:
|
||||
return
|
||||
if not self.node.joined.is_set():
|
||||
await self.node.joined.wait()
|
||||
blob_hashes = await self.storage.get_blobs_to_announce()
|
||||
@@ -39,7 +44,7 @@ class BlobAnnouncer:
|
||||
to_await.append(batch.pop())
|
||||
if to_await:
|
||||
await asyncio.gather(*tuple(to_await), loop=self.loop)
|
||||
await self.storage.update_last_announced_blobs(announced, self.loop.time())
|
||||
await self.storage.update_last_announced_blobs(announced, self.time_getter())
|
||||
log.info("announced %i blobs", len(announced))
|
||||
if self.running:
|
||||
self.pending_call = self.loop.call_later(60, self.announce, batch_size)
|
||||
|
||||
+55
-32
@@ -1,9 +1,9 @@
|
||||
import logging
|
||||
import asyncio
|
||||
import typing
|
||||
import socket
|
||||
import binascii
|
||||
import contextlib
|
||||
from lbrynet.utils import resolve_host
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.error import RemoteException
|
||||
from lbrynet.dht.protocol.async_generator_junction import AsyncGeneratorJunction
|
||||
@@ -20,10 +20,10 @@ log = logging.getLogger(__name__)
|
||||
|
||||
class Node:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int,
|
||||
internal_udp_port: int, peer_port: int, external_ip: str):
|
||||
internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: typing.Optional[float] = 5.0):
|
||||
self.loop = loop
|
||||
self.internal_udp_port = internal_udp_port
|
||||
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port)
|
||||
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port, rpc_timeout)
|
||||
self.listening_port: asyncio.DatagramTransport = None
|
||||
self.joined = asyncio.Event(loop=self.loop)
|
||||
self._join_task: asyncio.Task = None
|
||||
@@ -79,10 +79,9 @@ class Node:
|
||||
|
||||
if not self.protocol.external_ip:
|
||||
raise Exception("Cannot determine external IP")
|
||||
log.info("Store to %i peers", len(peers))
|
||||
log.info(peers)
|
||||
log.debug("Store to %i peers", len(peers))
|
||||
for peer in peers:
|
||||
log.info("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
|
||||
log.debug("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
|
||||
stored_to_tup = await asyncio.gather(
|
||||
*(self.protocol.store_to_peer(hash_value, peer) for peer in peers), loop=self.loop
|
||||
)
|
||||
@@ -116,39 +115,50 @@ class Node:
|
||||
log.warning("Already bound to port %s", self.listening_port)
|
||||
|
||||
async def join_network(self, interface: typing.Optional[str] = '',
|
||||
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
known_node_addresses: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
|
||||
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
|
||||
if not self.listening_port:
|
||||
await self.start_listening(interface)
|
||||
self.protocol.ping_queue.start()
|
||||
self._refresh_task = self.loop.create_task(self.refresh_node())
|
||||
|
||||
known_node_addresses = known_node_addresses or []
|
||||
# resolve the known node urls
|
||||
known_node_addresses = []
|
||||
url_to_addr = {}
|
||||
|
||||
if known_node_urls:
|
||||
for host, port in known_node_urls:
|
||||
info = await self.loop.getaddrinfo(
|
||||
host, 'https',
|
||||
proto=socket.IPPROTO_TCP,
|
||||
)
|
||||
if (info[0][4][0], port) not in known_node_addresses:
|
||||
known_node_addresses.append((info[0][4][0], port))
|
||||
futs = []
|
||||
for address, port in known_node_addresses:
|
||||
peer = self.protocol.get_rpc_peer(KademliaPeer(self.loop, address, udp_port=port))
|
||||
futs.append(peer.ping())
|
||||
if futs:
|
||||
await asyncio.wait(futs, loop=self.loop)
|
||||
address = await resolve_host(host)
|
||||
if (address, port) not in known_node_addresses:
|
||||
known_node_addresses.append((address, port))
|
||||
url_to_addr[address] = host
|
||||
|
||||
async with self.peer_search_junction(self.protocol.node_id, max_results=16) as junction:
|
||||
async for peers in junction:
|
||||
for peer in peers:
|
||||
if known_node_addresses:
|
||||
while not self.protocol.routing_table.get_peers():
|
||||
success = False
|
||||
# ping the seed nodes, this will set their node ids (since we don't know them ahead of time)
|
||||
for address, port in known_node_addresses:
|
||||
peer = self.protocol.get_rpc_peer(KademliaPeer(self.loop, address, udp_port=port))
|
||||
try:
|
||||
await self.protocol.get_rpc_peer(peer).ping()
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
pass
|
||||
self.joined.set()
|
||||
await peer.ping()
|
||||
success = True
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("seed node (%s:%i) timed out in %s", url_to_addr.get(address, address), port,
|
||||
round(self.protocol.rpc_timeout, 2))
|
||||
if success:
|
||||
break
|
||||
# now that we have the seed nodes in routing, to an iterative lookup of our own id to populate the buckets
|
||||
# in the routing table with good peers who are near us
|
||||
async with self.peer_search_junction(self.protocol.node_id, max_results=16) as junction:
|
||||
async for peers in junction:
|
||||
for peer in peers:
|
||||
try:
|
||||
await self.protocol.get_rpc_peer(peer).ping()
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
pass
|
||||
|
||||
log.info("Joined DHT, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
|
||||
self.protocol.routing_table.buckets_with_contacts())
|
||||
self.joined.set()
|
||||
|
||||
def start(self, interface: str, known_node_urls: typing.List[typing.Tuple[str, int]]):
|
||||
self._join_task = self.loop.create_task(
|
||||
@@ -214,13 +224,26 @@ class Node:
|
||||
async def peer_search(self, node_id: bytes, count=constants.k, max_results=constants.k*2,
|
||||
bottom_out_limit=20) -> typing.List['KademliaPeer']:
|
||||
accumulated: typing.List['KademliaPeer'] = []
|
||||
async with self.peer_search_junction(self.protocol.node_id, max_results=max_results,
|
||||
async with self.peer_search_junction(node_id, max_results=max_results,
|
||||
bottom_out_limit=bottom_out_limit) as junction:
|
||||
async for peers in junction:
|
||||
log.info("peer search: %s", peers)
|
||||
accumulated.extend(peers)
|
||||
log.info("junction done")
|
||||
log.info("context done")
|
||||
distance = Distance(node_id)
|
||||
accumulated.sort(key=lambda peer: distance(peer.node_id))
|
||||
return accumulated[:count]
|
||||
|
||||
async def _accumulate_search_junction(self, search_queue: asyncio.Queue,
|
||||
result_queue: asyncio.Queue):
|
||||
try:
|
||||
async with self.stream_peer_search_junction(search_queue) as search_junction: # pylint: disable=E1701
|
||||
async for peers in search_junction:
|
||||
if peers:
|
||||
result_queue.put_nowait(peers)
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
|
||||
def accumulate_peers(self, search_queue: asyncio.Queue,
|
||||
peer_queue: typing.Optional[asyncio.Queue] = None) -> typing.Tuple[
|
||||
asyncio.Queue, asyncio.Task]:
|
||||
q = peer_queue or asyncio.Queue()
|
||||
return q, asyncio.create_task(self._accumulate_search_junction(search_queue, q))
|
||||
+33
-54
@@ -30,37 +30,30 @@ class PeerManager:
|
||||
self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = {}
|
||||
self._node_tokens: typing.Dict[bytes, (float, bytes)] = {}
|
||||
self._kademlia_peers: typing.Dict[typing.Tuple[bytes, str, int], 'KademliaPeer']
|
||||
self._lock = asyncio.Lock(loop=loop)
|
||||
|
||||
async def report_failure(self, address: str, udp_port: int):
|
||||
def report_failure(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
async with self._lock:
|
||||
_, previous = self._rpc_failures.pop((address, udp_port), (None, None))
|
||||
self._rpc_failures[(address, udp_port)] = (previous, now)
|
||||
_, previous = self._rpc_failures.pop((address, udp_port), (None, None))
|
||||
self._rpc_failures[(address, udp_port)] = (previous, now)
|
||||
|
||||
async def report_last_sent(self, address: str, udp_port: int):
|
||||
def report_last_sent(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
async with self._lock:
|
||||
self._last_sent[(address, udp_port)] = now
|
||||
self._last_sent[(address, udp_port)] = now
|
||||
|
||||
async def report_last_replied(self, address: str, udp_port: int):
|
||||
def report_last_replied(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
async with self._lock:
|
||||
self._last_replied[(address, udp_port)] = now
|
||||
self._last_replied[(address, udp_port)] = now
|
||||
|
||||
async def report_last_requested(self, address: str, udp_port: int):
|
||||
def report_last_requested(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
async with self._lock:
|
||||
self._last_requested[(address, udp_port)] = now
|
||||
self._last_requested[(address, udp_port)] = now
|
||||
|
||||
async def clear_token(self, node_id: bytes):
|
||||
async with self._lock:
|
||||
self._node_tokens.pop(node_id, None)
|
||||
def clear_token(self, node_id: bytes):
|
||||
self._node_tokens.pop(node_id, None)
|
||||
|
||||
async def update_token(self, node_id: bytes, token: bytes):
|
||||
def update_token(self, node_id: bytes, token: bytes):
|
||||
now = self._loop.time()
|
||||
async with self._lock:
|
||||
self._node_tokens[node_id] = (now, token)
|
||||
self._node_tokens[node_id] = (now, token)
|
||||
|
||||
def get_node_token(self, node_id: bytes) -> typing.Optional[bytes]:
|
||||
ts, token = self._node_tokens.get(node_id, (None, None))
|
||||
@@ -70,50 +63,36 @@ class PeerManager:
|
||||
def get_last_replied(self, address: str, udp_port: int) -> typing.Optional[float]:
|
||||
return self._last_replied.get((address, udp_port))
|
||||
|
||||
def get_node_id(self, address: str, udp_port: int) -> typing.Optional[bytes]:
|
||||
return self._node_id_mapping.get((address, udp_port))
|
||||
|
||||
def get_node_address(self, node_id: bytes) -> typing.Optional[typing.Tuple[str, int]]:
|
||||
return self._node_id_reverse_mapping.get(node_id)
|
||||
|
||||
async def get_node_address_and_port(self, node_id: bytes) -> typing.Optional[typing.Tuple[str, int]]:
|
||||
async with self._lock:
|
||||
addr_tuple = self._node_id_reverse_mapping.get(node_id)
|
||||
if addr_tuple and addr_tuple in self._node_id_mapping:
|
||||
return addr_tuple
|
||||
|
||||
async def update_contact_triple(self, node_id: bytes, address: str, udp_port: int):
|
||||
def update_contact_triple(self, node_id: bytes, address: str, udp_port: int):
|
||||
"""
|
||||
Update the mapping of node_id -> address tuple and that of address tuple -> node_id
|
||||
This is to handle peers changing addresses and ids while assuring that the we only ever have
|
||||
one node id / address tuple mapped to each other
|
||||
"""
|
||||
async with self._lock:
|
||||
if (address, udp_port) in self._node_id_mapping:
|
||||
self._node_id_reverse_mapping.pop(self._node_id_mapping.pop((address, udp_port)))
|
||||
if node_id in self._node_id_reverse_mapping:
|
||||
self._node_id_mapping.pop(self._node_id_reverse_mapping.pop(node_id))
|
||||
self._node_id_mapping[(address, udp_port)] = node_id
|
||||
self._node_id_reverse_mapping[node_id] = (address, udp_port)
|
||||
if (address, udp_port) in self._node_id_mapping:
|
||||
self._node_id_reverse_mapping.pop(self._node_id_mapping.pop((address, udp_port)))
|
||||
if node_id in self._node_id_reverse_mapping:
|
||||
self._node_id_mapping.pop(self._node_id_reverse_mapping.pop(node_id))
|
||||
self._node_id_mapping[(address, udp_port)] = node_id
|
||||
self._node_id_reverse_mapping[node_id] = (address, udp_port)
|
||||
|
||||
def get_kademlia_peer(self, node_id: bytes, address: str, udp_port: int) -> 'KademliaPeer':
|
||||
return KademliaPeer(self._loop, address, node_id, udp_port)
|
||||
|
||||
async def prune(self):
|
||||
def prune(self): # TODO: periodically call this
|
||||
now = self._loop.time()
|
||||
async with self._lock:
|
||||
to_pop = []
|
||||
for (address, udp_port), (_, last_failure) in self._rpc_failures.items():
|
||||
if last_failure and last_failure < now - constants.rpc_attempts_pruning_window:
|
||||
to_pop.append((address, udp_port))
|
||||
while to_pop:
|
||||
del self._rpc_failures[to_pop.pop()]
|
||||
to_pop = []
|
||||
for node_id, (age, token) in self._node_tokens.items():
|
||||
if age < now - constants.token_secret_refresh_interval:
|
||||
to_pop.append(node_id)
|
||||
while to_pop:
|
||||
del self._node_tokens[to_pop.pop()]
|
||||
to_pop = []
|
||||
for (address, udp_port), (_, last_failure) in self._rpc_failures.items():
|
||||
if last_failure and last_failure < now - constants.rpc_attempts_pruning_window:
|
||||
to_pop.append((address, udp_port))
|
||||
while to_pop:
|
||||
del self._rpc_failures[to_pop.pop()]
|
||||
to_pop = []
|
||||
for node_id, (age, token) in self._node_tokens.items():
|
||||
if age < now - constants.token_secret_refresh_interval:
|
||||
to_pop.append(node_id)
|
||||
while to_pop:
|
||||
del self._node_tokens[to_pop.pop()]
|
||||
|
||||
def contact_triple_is_good(self, node_id: bytes, address: str, udp_port: int):
|
||||
"""
|
||||
|
||||
@@ -100,6 +100,6 @@ class AsyncGeneratorJunction:
|
||||
pass
|
||||
finally:
|
||||
if exc_type:
|
||||
if exc_type not in (asyncio.CancelledError, asyncio.TimeoutError, StopAsyncIteration):
|
||||
if exc_type not in (asyncio.CancelledError, asyncio.TimeoutError, StopAsyncIteration, GeneratorExit):
|
||||
err = traceback.format_exception(exc_type, exc, tb)
|
||||
log.error(err)
|
||||
@@ -1,7 +1,9 @@
|
||||
import asyncio
|
||||
from binascii import hexlify
|
||||
from itertools import chain
|
||||
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet.utils import drain_tasks
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.error import RemoteException
|
||||
from lbrynet.dht.protocol.distance import Distance
|
||||
@@ -90,8 +92,8 @@ class IterativeFinder:
|
||||
self.exclude = exclude or []
|
||||
|
||||
self.shortlist: typing.List['KademliaPeer'] = get_shortlist(routing_table, key, shortlist)
|
||||
self.active: typing.List['KademliaPeer'] = []
|
||||
self.contacted: typing.List[typing.Tuple[str, int]] = []
|
||||
self.active: typing.Set['KademliaPeer'] = set()
|
||||
self.contacted: typing.Set[typing.Tuple[str, int]] = set()
|
||||
self.distance = Distance(key)
|
||||
|
||||
self.closest_peer: typing.Optional['KademliaPeer'] = None if not self.shortlist else self.shortlist[0]
|
||||
@@ -99,14 +101,12 @@ class IterativeFinder:
|
||||
|
||||
self.iteration_queue = asyncio.Queue(loop=self.loop)
|
||||
|
||||
self.running_probes: typing.List[asyncio.Task] = []
|
||||
self.lock = asyncio.Lock(loop=self.loop)
|
||||
self.running_probes: typing.Set[asyncio.Task] = set()
|
||||
self.iteration_count = 0
|
||||
self.bottom_out_count = 0
|
||||
self.running = False
|
||||
self.tasks: typing.List[asyncio.Task] = []
|
||||
self.delayed_calls: typing.List[asyncio.Handle] = []
|
||||
self.finished = asyncio.Event(loop=self.loop)
|
||||
|
||||
async def send_probe(self, peer: 'KademliaPeer') -> FindResponse:
|
||||
"""
|
||||
@@ -114,9 +114,16 @@ class IterativeFinder:
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def search_exhausted(self):
|
||||
"""
|
||||
This method ends the iterator due no more peers to contact.
|
||||
Override to provide last time results.
|
||||
"""
|
||||
self.iteration_queue.put_nowait(None)
|
||||
|
||||
def check_result_ready(self, response: FindResponse):
|
||||
"""
|
||||
Called with a lock after adding peers from an rpc result to the shortlist.
|
||||
Called after adding peers from an rpc result to the shortlist.
|
||||
This method is responsible for putting a result for the generator into the Queue
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
@@ -129,9 +136,7 @@ class IterativeFinder:
|
||||
return []
|
||||
|
||||
def _is_closer(self, peer: 'KademliaPeer') -> bool:
|
||||
if not self.closest_peer:
|
||||
return True
|
||||
return self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
|
||||
return not self.closest_peer or self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
|
||||
|
||||
def _update_closest(self):
|
||||
self.shortlist.sort(key=lambda peer: self.distance(peer.node_id), reverse=True)
|
||||
@@ -141,21 +146,18 @@ class IterativeFinder:
|
||||
self.closest_peer = self.shortlist[-1]
|
||||
|
||||
async def _handle_probe_result(self, peer: 'KademliaPeer', response: FindResponse):
|
||||
async with self.lock:
|
||||
if peer not in self.shortlist:
|
||||
self.shortlist.append(peer)
|
||||
if peer not in self.active:
|
||||
self.active.append(peer)
|
||||
for contact_triple in response.get_close_triples():
|
||||
addr_tuple = (contact_triple[1], contact_triple[2])
|
||||
if addr_tuple not in self.contacted: # and not self.peer_manager.is_ignored(addr_tuple)
|
||||
found_peer = self.peer_manager.get_kademlia_peer(
|
||||
contact_triple[0], contact_triple[1], contact_triple[2]
|
||||
)
|
||||
if found_peer not in self.shortlist and self.peer_manager.peer_is_good(peer) is not False:
|
||||
self.shortlist.append(found_peer)
|
||||
self._update_closest()
|
||||
self.check_result_ready(response)
|
||||
if peer not in self.shortlist:
|
||||
self.shortlist.append(peer)
|
||||
if peer not in self.active:
|
||||
self.active.add(peer)
|
||||
for contact_triple in response.get_close_triples():
|
||||
node_id, address, udp_port = contact_triple
|
||||
if (address, udp_port) not in self.contacted: # and not self.peer_manager.is_ignored(addr_tuple)
|
||||
found_peer = self.peer_manager.get_kademlia_peer(node_id, address, udp_port)
|
||||
if found_peer not in self.shortlist and self.peer_manager.peer_is_good(peer) is not False:
|
||||
self.shortlist.append(found_peer)
|
||||
self._update_closest()
|
||||
self.check_result_ready(response)
|
||||
|
||||
async def _send_probe(self, peer: 'KademliaPeer'):
|
||||
try:
|
||||
@@ -163,13 +165,11 @@ class IterativeFinder:
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
except asyncio.TimeoutError:
|
||||
if peer in self.active:
|
||||
self.active.remove(peer)
|
||||
self.active.discard(peer)
|
||||
return
|
||||
except ValueError as err:
|
||||
log.warning(str(err))
|
||||
if peer in self.active:
|
||||
self.active.remove(peer)
|
||||
self.active.discard(peer)
|
||||
return
|
||||
except RemoteException:
|
||||
return
|
||||
@@ -181,31 +181,35 @@ class IterativeFinder:
|
||||
"""
|
||||
|
||||
added = 0
|
||||
async with self.lock:
|
||||
self.shortlist.sort(key=lambda p: self.distance(p.node_id), reverse=True)
|
||||
while self.running and len(self.shortlist) and added < constants.alpha:
|
||||
peer = self.shortlist.pop()
|
||||
origin_address = (peer.address, peer.udp_port)
|
||||
if origin_address in self.exclude or self.peer_manager.peer_is_good(peer) is False:
|
||||
continue
|
||||
if peer.node_id == self.protocol.node_id:
|
||||
continue
|
||||
if (peer.address, peer.udp_port) == (self.protocol.external_ip, self.protocol.udp_port):
|
||||
continue
|
||||
if (peer.address, peer.udp_port) not in self.contacted:
|
||||
self.contacted.append((peer.address, peer.udp_port))
|
||||
self.shortlist.sort(key=lambda p: self.distance(p.node_id), reverse=True)
|
||||
while self.running and len(self.shortlist) and added < constants.alpha:
|
||||
peer = self.shortlist.pop()
|
||||
origin_address = (peer.address, peer.udp_port)
|
||||
if origin_address in self.exclude or self.peer_manager.peer_is_good(peer) is False:
|
||||
continue
|
||||
if peer.node_id == self.protocol.node_id:
|
||||
continue
|
||||
if (peer.address, peer.udp_port) == (self.protocol.external_ip, self.protocol.udp_port):
|
||||
continue
|
||||
if (peer.address, peer.udp_port) not in self.contacted:
|
||||
self.contacted.add((peer.address, peer.udp_port))
|
||||
|
||||
t = self.loop.create_task(self._send_probe(peer))
|
||||
t = self.loop.create_task(self._send_probe(peer))
|
||||
|
||||
def callback(_):
|
||||
if t and t in self.running_probes:
|
||||
self.running_probes.remove(t)
|
||||
if not self.running_probes and self.shortlist:
|
||||
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
|
||||
def callback(_):
|
||||
self.running_probes.difference_update({
|
||||
probe for probe in self.running_probes if probe.done() or probe == t
|
||||
})
|
||||
if not self.running_probes and self.shortlist:
|
||||
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
|
||||
|
||||
t.add_done_callback(callback)
|
||||
self.running_probes.append(t)
|
||||
added += 1
|
||||
t.add_done_callback(callback)
|
||||
self.running_probes.add(t)
|
||||
added += 1
|
||||
log.debug("running %d probes", len(self.running_probes))
|
||||
if not added and not self.running_probes:
|
||||
log.debug("search for %s exhausted", hexlify(self.key)[:8])
|
||||
self.search_exhausted()
|
||||
|
||||
async def _search_task(self, delay: typing.Optional[float] = constants.iterative_lookup_delay):
|
||||
try:
|
||||
@@ -215,70 +219,41 @@ class IterativeFinder:
|
||||
self.delayed_calls.append(self.loop.call_later(delay, self._search))
|
||||
except (asyncio.CancelledError, StopAsyncIteration):
|
||||
if self.running:
|
||||
drain_tasks(self.running_probes)
|
||||
self.running = False
|
||||
self.loop.call_soon(self.aclose)
|
||||
|
||||
def _search(self):
|
||||
self.tasks.append(self.loop.create_task(self._search_task()))
|
||||
|
||||
def search(self):
|
||||
def __aiter__(self):
|
||||
if self.running:
|
||||
raise Exception("already running")
|
||||
self.running = True
|
||||
self._search()
|
||||
|
||||
async def next_queue_or_finished(self) -> typing.List['KademliaPeer']:
|
||||
peers = self.loop.create_task(self.iteration_queue.get())
|
||||
finished = self.loop.create_task(self.finished.wait())
|
||||
err = None
|
||||
try:
|
||||
await asyncio.wait([peers, finished], loop=self.loop, return_when='FIRST_COMPLETED')
|
||||
if peers.done():
|
||||
return peers.result()
|
||||
raise StopAsyncIteration()
|
||||
except asyncio.CancelledError as error:
|
||||
err = error
|
||||
finally:
|
||||
if not finished.done() and not finished.cancelled():
|
||||
finished.cancel()
|
||||
if not peers.done() and not peers.cancelled():
|
||||
peers.cancel()
|
||||
if err:
|
||||
raise err
|
||||
|
||||
def __aiter__(self):
|
||||
self.search()
|
||||
return self
|
||||
|
||||
async def __anext__(self) -> typing.List['KademliaPeer']:
|
||||
try:
|
||||
if self.iteration_count == 0:
|
||||
initial_results = self.get_initial_result()
|
||||
if initial_results:
|
||||
self.iteration_queue.put_nowait(initial_results)
|
||||
result = await self.next_queue_or_finished()
|
||||
result = self.get_initial_result() or await self.iteration_queue.get()
|
||||
else:
|
||||
result = await self.iteration_queue.get()
|
||||
if not result:
|
||||
raise StopAsyncIteration
|
||||
self.iteration_count += 1
|
||||
return result
|
||||
except (asyncio.CancelledError, StopAsyncIteration):
|
||||
await self.aclose()
|
||||
self.loop.call_soon(self.aclose)
|
||||
raise
|
||||
|
||||
def aclose(self):
|
||||
self.running = False
|
||||
self.iteration_queue.put_nowait(None)
|
||||
for task in chain(self.tasks, self.running_probes, self.delayed_calls):
|
||||
task.cancel()
|
||||
self.tasks.clear()
|
||||
self.running_probes.clear()
|
||||
self.delayed_calls.clear()
|
||||
|
||||
async def _aclose():
|
||||
async with self.lock:
|
||||
self.running = False
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
drain_tasks(self.tasks)
|
||||
drain_tasks(self.running_probes)
|
||||
while self.delayed_calls:
|
||||
timer = self.delayed_calls.pop()
|
||||
if timer:
|
||||
timer.cancel()
|
||||
|
||||
return asyncio.ensure_future(_aclose(), loop=self.loop)
|
||||
|
||||
|
||||
class IterativeNodeFinder(IterativeFinder):
|
||||
@@ -295,24 +270,26 @@ class IterativeNodeFinder(IterativeFinder):
|
||||
response = await self.protocol.get_rpc_peer(peer).find_node(self.key)
|
||||
return FindNodeResponse(self.key, response)
|
||||
|
||||
def put_result(self, from_list: typing.List['KademliaPeer']):
|
||||
not_yet_yielded = [peer for peer in from_list if peer not in self.yielded_peers]
|
||||
def search_exhausted(self):
|
||||
self.put_result(self.active, finish=True)
|
||||
|
||||
def put_result(self, from_iter: typing.Iterable['KademliaPeer'], finish=False):
|
||||
not_yet_yielded = [peer for peer in from_iter if peer not in self.yielded_peers]
|
||||
not_yet_yielded.sort(key=lambda peer: self.distance(peer.node_id))
|
||||
to_yield = not_yet_yielded[:min(constants.k, len(not_yet_yielded))]
|
||||
if to_yield:
|
||||
for peer in to_yield:
|
||||
self.yielded_peers.add(peer)
|
||||
self.iteration_queue.put_nowait(to_yield)
|
||||
if finish:
|
||||
self.iteration_queue.put_nowait(None)
|
||||
|
||||
def check_result_ready(self, response: FindNodeResponse):
|
||||
found = response.found and self.key != self.protocol.node_id
|
||||
|
||||
if found:
|
||||
log.info("found")
|
||||
self.put_result(self.shortlist)
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return
|
||||
log.debug("found")
|
||||
return self.put_result(self.shortlist, finish=True)
|
||||
if self.prev_closest_peer and self.closest_peer and not self._is_closer(self.prev_closest_peer):
|
||||
# log.info("improving, %i %i %i %i %i", len(self.shortlist), len(self.active), len(self.contacted),
|
||||
# self.bottom_out_count, self.iteration_count)
|
||||
@@ -323,16 +300,10 @@ class IterativeNodeFinder(IterativeFinder):
|
||||
self.bottom_out_count)
|
||||
if self.bottom_out_count >= self.bottom_out_limit or self.iteration_count >= self.bottom_out_limit:
|
||||
log.info("limit hit")
|
||||
self.put_result(self.active)
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return
|
||||
if self.max_results and len(self.active) - len(self.yielded_peers) >= self.max_results:
|
||||
log.info("max results")
|
||||
self.put_result(self.active)
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return
|
||||
self.put_result(self.active, True)
|
||||
elif self.max_results and len(self.active) - len(self.yielded_peers) >= self.max_results:
|
||||
log.debug("max results")
|
||||
self.put_result(self.active, True)
|
||||
|
||||
|
||||
class IterativeValueFinder(IterativeFinder):
|
||||
@@ -366,14 +337,11 @@ class IterativeValueFinder(IterativeFinder):
|
||||
# log.info("enough blob peers found")
|
||||
# if not self.finished.is_set():
|
||||
# self.finished.set()
|
||||
return
|
||||
if self.prev_closest_peer and self.closest_peer:
|
||||
elif self.prev_closest_peer and self.closest_peer:
|
||||
self.bottom_out_count += 1
|
||||
if self.bottom_out_count >= self.bottom_out_limit:
|
||||
log.info("blob peer search bottomed out")
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return
|
||||
self.iteration_queue.put_nowait(None)
|
||||
|
||||
def get_initial_result(self) -> typing.List['KademliaPeer']:
|
||||
if self.protocol.data_store.has_peers_for_blob(self.key):
|
||||
|
||||
@@ -86,7 +86,7 @@ class KademliaRPC:
|
||||
has_other_peers = self.protocol.data_store.has_peers_for_blob(key)
|
||||
peers = []
|
||||
if has_other_peers:
|
||||
peers.extend(self.protocol.data_store.get_peers_for_blob(key))
|
||||
peers.extend([peer.compact_address_tcp() for peer in self.protocol.data_store.get_peers_for_blob(key)])
|
||||
|
||||
# if we don't have k storing peers to return and we have this hash locally, include our contact information
|
||||
if len(peers) < constants.k and binascii.hexlify(key).decode() in self.protocol.data_store.completed_blobs:
|
||||
@@ -180,7 +180,7 @@ class RemoteKademliaRPC:
|
||||
response = await self.protocol.send_request(
|
||||
self.peer, RequestDatagram.make_find_value(self.protocol.node_id, key)
|
||||
)
|
||||
await self.peer_tracker.update_token(self.peer.node_id, response.response[b'token'])
|
||||
self.peer_tracker.update_token(self.peer.node_id, response.response[b'token'])
|
||||
return response.response
|
||||
|
||||
|
||||
@@ -218,7 +218,7 @@ class PingQueue:
|
||||
await self._protocol.add_peer(p)
|
||||
return
|
||||
await self._protocol.get_rpc_peer(p).ping()
|
||||
except TimeoutError:
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
while True:
|
||||
@@ -415,8 +415,8 @@ class KademliaProtocol(DatagramProtocol):
|
||||
|
||||
async def handle_request_datagram(self, address, request_datagram: RequestDatagram):
|
||||
# This is an RPC method request
|
||||
await self.peer_manager.report_last_requested(address[0], address[1])
|
||||
await self.peer_manager.update_contact_triple(request_datagram.node_id, address[0], address[1])
|
||||
self.peer_manager.report_last_requested(address[0], address[1])
|
||||
self.peer_manager.update_contact_triple(request_datagram.node_id, address[0], address[1])
|
||||
# only add a requesting contact to the routing table if it has replied to one of our requests
|
||||
peer = self.peer_manager.get_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
try:
|
||||
@@ -457,8 +457,8 @@ class KademliaProtocol(DatagramProtocol):
|
||||
elif response_datagram.node_id == self.node_id:
|
||||
df.set_exception(RemoteException("incoming message is from our node id"))
|
||||
return
|
||||
await self.peer_manager.report_last_replied(address[0], address[1])
|
||||
await self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
|
||||
self.peer_manager.report_last_replied(address[0], address[1])
|
||||
self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
|
||||
if not df.cancelled():
|
||||
df.set_result(response_datagram)
|
||||
await self.add_peer(peer)
|
||||
@@ -505,7 +505,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
try:
|
||||
message = decode_datagram(datagram)
|
||||
except (ValueError, TypeError):
|
||||
self.loop.create_task(self.peer_manager.report_failure(address[0], address[1]))
|
||||
self.peer_manager.report_failure(address[0], address[1])
|
||||
log.warning("Couldn't decode dht datagram from %s: %s", address, binascii.hexlify(datagram).decode())
|
||||
return
|
||||
|
||||
@@ -522,10 +522,10 @@ class KademliaProtocol(DatagramProtocol):
|
||||
response_fut = self.sent_messages[request.rpc_id][1]
|
||||
try:
|
||||
response = await asyncio.wait_for(response_fut, self.rpc_timeout)
|
||||
await self.peer_manager.report_last_replied(peer.address, peer.udp_port)
|
||||
self.peer_manager.report_last_replied(peer.address, peer.udp_port)
|
||||
return response
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
await self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
if self.peer_manager.peer_is_good(peer) is False:
|
||||
self.routing_table.remove_peer(peer)
|
||||
raise
|
||||
@@ -575,9 +575,9 @@ class KademliaProtocol(DatagramProtocol):
|
||||
else:
|
||||
raise err
|
||||
if isinstance(message, RequestDatagram):
|
||||
await self.peer_manager.report_last_sent(peer.address, peer.udp_port)
|
||||
self.peer_manager.report_last_sent(peer.address, peer.udp_port)
|
||||
elif isinstance(message, ErrorDatagram):
|
||||
await self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
|
||||
def change_token(self):
|
||||
self.old_token_secret = self.token_secret
|
||||
@@ -601,7 +601,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
res = await self.get_rpc_peer(peer).store(hash_value)
|
||||
if res != b"OK":
|
||||
raise ValueError(res)
|
||||
log.info("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
log.debug("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
return peer.node_id, True
|
||||
except asyncio.TimeoutError:
|
||||
log.debug("Timeout while storing blob_hash %s at %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
@@ -609,7 +609,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
log.error("Unexpected response: %s" % err)
|
||||
except Exception as err:
|
||||
if 'Invalid token' in str(err):
|
||||
await self.peer_manager.clear_token(peer.node_id)
|
||||
self.peer_manager.clear_token(peer.node_id)
|
||||
else:
|
||||
log.exception("Unexpected error while storing blob_hash")
|
||||
return peer.node_id, False
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import os
|
||||
import sys
|
||||
import pathlib
|
||||
import json
|
||||
import asyncio
|
||||
import argparse
|
||||
@@ -216,12 +218,19 @@ def get_argument_parser():
|
||||
return main
|
||||
|
||||
|
||||
def ensure_directory_exists(path: str):
|
||||
if not os.path.isdir(path):
|
||||
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
argv = argv or sys.argv[1:]
|
||||
parser = get_argument_parser()
|
||||
args, command_args = parser.parse_known_args(argv)
|
||||
|
||||
conf = Config.create_from_arguments(args)
|
||||
for directory in (conf.data_dir, conf.download_dir, conf.wallet_dir):
|
||||
ensure_directory_exists(directory)
|
||||
|
||||
if args.cli_version:
|
||||
print(f"{lbrynet_name} {lbrynet_version}")
|
||||
@@ -243,6 +252,8 @@ def main(argv=None):
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(default_formatter)
|
||||
log.addHandler(handler)
|
||||
logging.getLogger('torba').addHandler(handler)
|
||||
logging.getLogger('torba').setLevel(logging.INFO)
|
||||
|
||||
logging.getLogger('aioupnp').setLevel(logging.WARNING)
|
||||
logging.getLogger('aiohttp').setLevel(logging.CRITICAL)
|
||||
|
||||
@@ -5,7 +5,6 @@ import logging
|
||||
import math
|
||||
import binascii
|
||||
import typing
|
||||
import socket
|
||||
from hashlib import sha256
|
||||
from types import SimpleNamespace
|
||||
import base58
|
||||
@@ -18,7 +17,6 @@ import lbrynet.schema
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import HEADERS_FILE_SHA256_CHECKSUM
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.dht.blob_announcer import BlobAnnouncer
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob_exchange.server import BlobServer
|
||||
@@ -65,14 +63,6 @@ async def get_external_ip(): # used if upnp is disabled or non-functioning
|
||||
pass
|
||||
|
||||
|
||||
async def resolve_host(loop: asyncio.BaseEventLoop, url: str):
|
||||
info = await loop.getaddrinfo(
|
||||
url, 'https',
|
||||
proto=socket.IPPROTO_TCP,
|
||||
)
|
||||
return info[0][4][0]
|
||||
|
||||
|
||||
class DatabaseComponent(Component):
|
||||
component_name = DATABASE_COMPONENT
|
||||
|
||||
@@ -359,7 +349,7 @@ class DHTComponent(Component):
|
||||
|
||||
async def get_status(self):
|
||||
return {
|
||||
'node_id': binascii.hexlify(self.dht_node.protocol.node_id),
|
||||
'node_id': None if not self.dht_node else binascii.hexlify(self.dht_node.protocol.node_id),
|
||||
'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.protocol.routing_table.get_peers())
|
||||
}
|
||||
|
||||
@@ -376,8 +366,8 @@ class DHTComponent(Component):
|
||||
async def start(self):
|
||||
log.info("start the dht")
|
||||
self.upnp_component = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.peer_port)
|
||||
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.dht_node_port)
|
||||
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.tcp_port)
|
||||
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.udp_port)
|
||||
external_ip = self.upnp_component.external_ip
|
||||
if not external_ip:
|
||||
log.warning("UPnP component failed to get external ip")
|
||||
@@ -389,13 +379,14 @@ class DHTComponent(Component):
|
||||
asyncio.get_event_loop(),
|
||||
self.component_manager.peer_manager,
|
||||
node_id=self.get_node_id(),
|
||||
internal_udp_port=self.conf.dht_node_port,
|
||||
internal_udp_port=self.conf.udp_port,
|
||||
udp_port=self.external_udp_port,
|
||||
external_ip=external_ip,
|
||||
peer_port=self.external_peer_port
|
||||
peer_port=self.external_peer_port,
|
||||
rpc_timeout=self.conf.node_rpc_timeout
|
||||
)
|
||||
self.dht_node.start(
|
||||
interface='0.0.0.0', known_node_urls=self.conf.known_dht_nodes
|
||||
interface=self.conf.network_interface, known_node_urls=self.conf.known_dht_nodes
|
||||
)
|
||||
log.info("Started the dht")
|
||||
|
||||
@@ -419,7 +410,7 @@ class HashAnnouncerComponent(Component):
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
self.hash_announcer = BlobAnnouncer(asyncio.get_event_loop(), dht_node, storage)
|
||||
self.hash_announcer.start(self.conf.concurrent_announcers)
|
||||
self.hash_announcer.start(self.conf.concurrent_blob_announcers)
|
||||
log.info("Started blob announcer")
|
||||
|
||||
async def stop(self):
|
||||
@@ -462,17 +453,13 @@ class StreamManagerComponent(Component):
|
||||
log.info('Starting the file manager')
|
||||
loop = asyncio.get_event_loop()
|
||||
self.stream_manager = StreamManager(
|
||||
loop, blob_manager, wallet, storage, node, self.conf.blob_download_timeout,
|
||||
self.conf.peer_connect_timeout, [
|
||||
KademliaPeer(loop, address=(await resolve_host(loop, url)), tcp_port=port + 1)
|
||||
for url, port in self.conf.reflector_servers
|
||||
], self.conf.reflector_servers
|
||||
loop, self.conf, blob_manager, wallet, storage, node,
|
||||
)
|
||||
await self.stream_manager.start()
|
||||
log.info('Done setting up file manager')
|
||||
|
||||
async def stop(self):
|
||||
await self.stream_manager.stop()
|
||||
self.stream_manager.stop()
|
||||
|
||||
|
||||
class PeerProtocolServerComponent(Component):
|
||||
@@ -492,10 +479,10 @@ class PeerProtocolServerComponent(Component):
|
||||
upnp = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
blob_manager: BlobFileManager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
wallet: LbryWalletManager = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
peer_port = upnp.upnp_redirects.get("TCP", self.conf.peer_port)
|
||||
peer_port = upnp.upnp_redirects.get("TCP", self.conf.tcp_port)
|
||||
address = await wallet.get_unused_address()
|
||||
self.blob_server = BlobServer(asyncio.get_event_loop(), blob_manager, address)
|
||||
self.blob_server.start_server(peer_port, interface='0.0.0.0')
|
||||
self.blob_server.start_server(peer_port, interface=self.conf.network_interface)
|
||||
await self.blob_server.started_listening.wait()
|
||||
|
||||
async def stop(self):
|
||||
@@ -508,8 +495,8 @@ class UPnPComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self._int_peer_port = self.conf.peer_port
|
||||
self._int_dht_node_port = self.conf.dht_node_port
|
||||
self._int_peer_port = self.conf.tcp_port
|
||||
self._int_dht_node_port = self.conf.udp_port
|
||||
self.use_upnp = self.conf.use_upnp
|
||||
self.upnp = None
|
||||
self.upnp_redirects = {}
|
||||
@@ -614,7 +601,9 @@ class UPnPComponent(Component):
|
||||
else:
|
||||
log.error("failed to setup upnp")
|
||||
if self.component_manager.analytics_manager:
|
||||
self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, await self.get_status())
|
||||
await self.component_manager.analytics_manager.send_upnp_setup_success_fail(
|
||||
success, await self.get_status()
|
||||
)
|
||||
self._maintain_redirects_task = asyncio.create_task(self._repeatedly_maintain_redirects(now=False))
|
||||
|
||||
async def stop(self):
|
||||
|
||||
@@ -6,10 +6,10 @@ import inspect
|
||||
import typing
|
||||
import aiohttp
|
||||
import base58
|
||||
import random
|
||||
from urllib.parse import urlencode, quote
|
||||
from typing import Callable, Optional, List
|
||||
from binascii import hexlify, unhexlify
|
||||
from copy import deepcopy
|
||||
from traceback import format_exc
|
||||
from aiohttp import web
|
||||
from functools import wraps
|
||||
@@ -18,9 +18,9 @@ from torba.client.baseaccount import SingleKey, HierarchicalDeterministic
|
||||
from lbrynet import __version__, utils
|
||||
from lbrynet.conf import Config, Setting, SLACK_WEBHOOK
|
||||
from lbrynet.blob.blob_file import is_valid_blobhash
|
||||
from lbrynet.blob_exchange.downloader import BlobDownloader
|
||||
from lbrynet.blob_exchange.downloader import download_blob
|
||||
from lbrynet.error import InsufficientFundsError, DownloadSDTimeout, ComponentsNotStarted
|
||||
from lbrynet.error import NullFundsError, NegativeFundsError, ResolveError, ComponentStartConditionNotMet
|
||||
from lbrynet.error import NullFundsError, NegativeFundsError, ComponentStartConditionNotMet
|
||||
from lbrynet.extras import system_info
|
||||
from lbrynet.extras.daemon import analytics
|
||||
from lbrynet.extras.daemon.Components import WALLET_COMPONENT, DATABASE_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT
|
||||
@@ -29,6 +29,7 @@ from lbrynet.extras.daemon.Components import EXCHANGE_RATE_MANAGER_COMPONENT, UP
|
||||
from lbrynet.extras.daemon.ComponentManager import RequiredCondition
|
||||
from lbrynet.extras.daemon.ComponentManager import ComponentManager
|
||||
from lbrynet.extras.daemon.json_response_encoder import JSONResponseEncoder
|
||||
from lbrynet.extras.daemon.mime_types import guess_media_type
|
||||
from lbrynet.extras.daemon.undecorated import undecorated
|
||||
from lbrynet.extras.wallet.account import Account as LBCAccount
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
||||
@@ -1307,8 +1308,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
file_list [--sd_hash=<sd_hash>] [--file_name=<file_name>] [--stream_hash=<stream_hash>]
|
||||
[--rowid=<rowid>] [--claim_id=<claim_id>] [--outpoint=<outpoint>] [--txid=<txid>] [--nout=<nout>]
|
||||
[--channel_claim_id=<channel_claim_id>] [--channel_name=<channel_name>]
|
||||
[--claim_name=<claim_name>] [--sort=<sort_by>] [--reverse] [--comparison=<comparison>]
|
||||
[--full_status=<full_status>]
|
||||
[--claim_name=<claim_name>] [--blobs_in_stream=<blobs_in_stream>]
|
||||
[--blobs_remaining=<blobs_remaining>] [--sort=<sort_by>]
|
||||
[--comparison=<comparison>] [--full_status=<full_status>] [--reverse]
|
||||
|
||||
Options:
|
||||
--sd_hash=<sd_hash> : (str) get file with matching sd hash
|
||||
@@ -1321,11 +1323,12 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--txid=<txid> : (str) get file with matching claim txid
|
||||
--nout=<nout> : (int) get file with matching claim nout
|
||||
--channel_claim_id=<channel_claim_id> : (str) get file with matching channel claim id
|
||||
--channel_name=<channel_name> : (str) get file with matching channel name
|
||||
--channel_name=<channel_name> : (str) get file with matching channel name
|
||||
--claim_name=<claim_name> : (str) get file with matching claim name
|
||||
--sort=<sort_method> : (str) sort by any property, like 'file_name'
|
||||
or 'metadata.author'; to specify direction
|
||||
append ',asc' or ',desc'
|
||||
--blobs_in_stream<blobs_in_stream> : (int) get file with matching blobs in stream
|
||||
--blobs_remaining=<blobs_remaining> : (int) amount of remaining blobs to download
|
||||
--sort=<sort_by> : (str) field to sort by (one of the above filter fields)
|
||||
--comparison=<comparison> : (str) logical comparision, (eq | ne | g | ge | l | le)
|
||||
|
||||
Returns:
|
||||
(list) List of files
|
||||
@@ -1344,21 +1347,24 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
'download_path': (str) download path of file,
|
||||
'mime_type': (str) mime type of file,
|
||||
'key': (str) key attached to file,
|
||||
'total_bytes': (int) file size in bytes,
|
||||
'total_bytes_lower_bound': (int) lower bound file size in bytes,
|
||||
'total_bytes': (int) file upper bound size in bytes,
|
||||
'written_bytes': (int) written size in bytes,
|
||||
'blobs_completed': (int) number of fully downloaded blobs,
|
||||
'blobs_in_stream': (int) total blobs on stream,
|
||||
'blobs_remaining': (int) total blobs remaining to download,
|
||||
'status': (str) downloader status
|
||||
'claim_id': (str) None if claim is not found else the claim id,
|
||||
'outpoint': (str) None if claim is not found else the tx and output,
|
||||
'txid': (str) None if claim is not found else the transaction id,
|
||||
'nout': (int) None if claim is not found else the transaction output index,
|
||||
'outpoint': (str) None if claim is not found else the tx and output,
|
||||
'metadata': (dict) None if claim is not found else the claim metadata,
|
||||
'channel_claim_id': (str) None if claim is not found or not signed,
|
||||
'channel_name': (str) None if claim is not found or not signed,
|
||||
'claim_name': (str) None if claim is not found else the claim name
|
||||
},
|
||||
]
|
||||
}
|
||||
"""
|
||||
sort = sort or 'status'
|
||||
comparison = comparison or 'eq'
|
||||
@@ -1551,44 +1557,12 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
}
|
||||
"""
|
||||
|
||||
timeout = timeout if timeout is not None else self.conf.download_timeout
|
||||
|
||||
parsed_uri = parse_lbry_uri(uri)
|
||||
if parsed_uri.is_channel:
|
||||
raise Exception("cannot download a channel claim, specify a /path")
|
||||
|
||||
resolved = (await self.wallet_manager.resolve(uri)).get(uri, {})
|
||||
resolved = resolved if 'value' in resolved else resolved.get('claim')
|
||||
|
||||
if not resolved:
|
||||
raise ResolveError(
|
||||
"Failed to resolve stream at lbry://{}".format(uri.replace("lbry://", ""))
|
||||
)
|
||||
if 'error' in resolved:
|
||||
raise ResolveError(f"error resolving stream: {resolved['error']}")
|
||||
|
||||
claim = ClaimDict.load_dict(resolved['value'])
|
||||
fee_amount, fee_address = None, None
|
||||
if claim.has_fee:
|
||||
fee_amount = round(self.exchange_rate_manager.convert_currency(
|
||||
claim.source_fee.currency, "LBC", claim.source_fee.amount
|
||||
), 5)
|
||||
fee_address = claim.source_fee.address
|
||||
outpoint = f"{resolved['txid']}:{resolved['nout']}"
|
||||
existing = self.stream_manager.get_filtered_streams(outpoint=outpoint)
|
||||
if not existing:
|
||||
existing.extend(self.stream_manager.get_filtered_streams(claim_id=resolved['claim_id'],
|
||||
sd_hash=claim.source_hash))
|
||||
if existing:
|
||||
log.info("already have matching stream for %s", uri)
|
||||
stream = existing[0]
|
||||
else:
|
||||
stream = await self.stream_manager.download_stream_from_claim(
|
||||
self.dht_node, self.conf.download_dir, resolved, file_name, timeout, fee_amount, fee_address
|
||||
)
|
||||
stream = await self.stream_manager.download_stream_from_uri(
|
||||
uri, self.exchange_rate_manager, file_name, timeout
|
||||
)
|
||||
if stream:
|
||||
return stream.as_dict()
|
||||
raise DownloadSDTimeout(resolved['value']['stream']['source']['source'])
|
||||
raise DownloadSDTimeout(uri)
|
||||
|
||||
@requires(STREAM_MANAGER_COMPONENT)
|
||||
async def jsonrpc_file_set_status(self, status, **kwargs):
|
||||
@@ -1618,11 +1592,11 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
if not streams:
|
||||
raise Exception(f'Unable to find a file for {kwargs}')
|
||||
stream = streams[0]
|
||||
if status == 'start' and not stream.running and not stream.finished:
|
||||
stream.downloader.download(self.dht_node)
|
||||
if status == 'start' and not stream.running:
|
||||
await self.stream_manager.start_stream(stream)
|
||||
msg = "Resumed download"
|
||||
elif status == 'stop' and stream.running:
|
||||
await stream.stop_download()
|
||||
stream.stop_download()
|
||||
msg = "Stopped download"
|
||||
else:
|
||||
msg = (
|
||||
@@ -1829,7 +1803,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
async def jsonrpc_publish(
|
||||
self, name, bid, metadata=None, file_path=None, fee=None, title=None,
|
||||
description=None, author=None, language=None, license=None,
|
||||
license_url=None, thumbnail=None, preview=None, nsfw=None, sources=None,
|
||||
license_url=None, thumbnail=None, preview=None, nsfw=None,
|
||||
channel_name=None, channel_id=None, channel_account_id=None, account_id=None,
|
||||
claim_address=None, change_address=None):
|
||||
"""
|
||||
@@ -1853,7 +1827,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
[--file_path=<file_path>] [--fee=<fee>] [--title=<title>]
|
||||
[--description=<description>] [--author=<author>] [--language=<language>]
|
||||
[--license=<license>] [--license_url=<license_url>] [--thumbnail=<thumbnail>]
|
||||
[--preview=<preview>] [--nsfw=<nsfw>] [--sources=<sources>]
|
||||
[--preview=<preview>] [--nsfw=<nsfw>]
|
||||
[--channel_name=<channel_name>] [--channel_id=<channel_id>]
|
||||
[--channel_account_id=<channel_account_id>...] [--account_id=<account_id>]
|
||||
[--claim_address=<claim_address>] [--change_address=<change_address>]
|
||||
@@ -1889,7 +1863,6 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--thumbnail=<thumbnail> : (str) thumbnail url
|
||||
--preview=<preview> : (str) preview url
|
||||
--nsfw=<nsfw> : (bool) whether the content is nsfw
|
||||
--sources=<sources> : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file
|
||||
--channel_name=<channel_name> : (str) name of the publisher channel name in the wallet
|
||||
--channel_id=<channel_id> : (str) claim id of the publisher channel, does not check
|
||||
for channel claim being in the wallet. This allows
|
||||
@@ -1913,7 +1886,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
"""
|
||||
|
||||
try:
|
||||
parse_lbry_uri(name)
|
||||
parsed = parse_lbry_uri(name)
|
||||
if parsed.name != name:
|
||||
raise Exception("Name given to publish has invalid characters")
|
||||
except (TypeError, URIParseError):
|
||||
raise Exception("Invalid name given to publish")
|
||||
|
||||
@@ -1929,9 +1904,10 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
account = self.get_account_or_default(account_id)
|
||||
|
||||
available = await account.get_balance()
|
||||
existing_claims = []
|
||||
existing_claims = await account.get_claims(claim_name=name)
|
||||
if len(existing_claims) > 1:
|
||||
log.warning("found %i claims for the name %s", len(existing_claims), name)
|
||||
if amount >= available:
|
||||
existing_claims = await account.get_claims(claim_name=name)
|
||||
if len(existing_claims) == 1:
|
||||
available += existing_claims[0].get_estimator(self.ledger).effective_amount
|
||||
if amount >= available:
|
||||
@@ -1990,39 +1966,64 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
}
|
||||
}
|
||||
|
||||
# this will be used to verify the format with lbrynet.schema
|
||||
claim_copy = deepcopy(claim_dict)
|
||||
if sources is not None:
|
||||
claim_dict['stream']['source'] = sources
|
||||
claim_copy['stream']['source'] = sources
|
||||
elif file_path is not None:
|
||||
sd_to_delete = None
|
||||
|
||||
if file_path:
|
||||
if not os.path.isfile(file_path):
|
||||
raise Exception("invalid file path to publish")
|
||||
if os.path.getsize(file_path) == 0:
|
||||
raise Exception(f"Cannot publish empty file {file_path}")
|
||||
if existing_claims:
|
||||
sd_to_delete = existing_claims[-1].claim_dict['stream']['source']['source']
|
||||
|
||||
# since the file hasn't yet been made into a stream, we don't have
|
||||
# a valid Source for the claim when validating the format, we'll use a fake one
|
||||
claim_copy['stream']['source'] = {
|
||||
claim_dict['stream']['source'] = {
|
||||
'version': '_0_0_1',
|
||||
'sourceType': 'lbry_sd_hash',
|
||||
'source': '0' * 96,
|
||||
'contentType': ''
|
||||
}
|
||||
elif not existing_claims:
|
||||
raise Exception("no previous stream to update")
|
||||
else:
|
||||
# there is no existing source to use, and a file was not provided to make a new one
|
||||
raise Exception("no source provided to publish")
|
||||
claim_dict['stream']['source'] = existing_claims[-1].claim_dict['stream']['source']
|
||||
try:
|
||||
ClaimDict.load_dict(claim_copy)
|
||||
claim_dict = ClaimDict.load_dict(claim_dict).claim_dict
|
||||
# the metadata to use in the claim can be serialized by lbrynet.schema
|
||||
except DecodeError as err:
|
||||
# there was a problem with a metadata field, raise an error here rather than
|
||||
# waiting to find out when we go to publish the claim (after having made the stream)
|
||||
raise Exception(f"invalid publish metadata: {err}")
|
||||
|
||||
certificate = None
|
||||
if channel_id or channel_name:
|
||||
certificate = await self.get_channel_or_error(
|
||||
self.get_accounts_or_all(channel_account_id), channel_id, channel_name
|
||||
)
|
||||
|
||||
if file_path:
|
||||
stream = await self.stream_manager.create_stream(file_path)
|
||||
await self.storage.save_published_file(stream.stream_hash, os.path.basename(file_path),
|
||||
os.path.dirname(file_path), 0)
|
||||
claim_dict['stream']['source']['source'] = stream.sd_hash
|
||||
claim_dict['stream']['source']['contentType'] = guess_media_type(file_path)
|
||||
|
||||
if sd_to_delete:
|
||||
stream_hash_to_delete = await self.storage.get_stream_hash_for_sd_hash(sd_to_delete)
|
||||
stream_to_delete = self.stream_manager.get_stream_by_stream_hash(stream_hash_to_delete)
|
||||
if stream_to_delete:
|
||||
await self.stream_manager.delete_stream(stream_to_delete, delete_file=False)
|
||||
log.info("updating claim to stream generated from %s, deleted previous stream %s",
|
||||
sd_to_delete[:8], file_path)
|
||||
else:
|
||||
log.info("previous stream %s from claim was not saved locally, nothing to delete",
|
||||
sd_to_delete[:8])
|
||||
else:
|
||||
log.info("generated stream from %s for claim", file_path)
|
||||
else:
|
||||
log.info("updating claim with stream %s from previous", claim_dict['stream']['source']['source'][:8])
|
||||
|
||||
sd_hash = claim_dict['stream']['source']['source']
|
||||
log.info("Publish: %s", {
|
||||
'name': name,
|
||||
'file_path': file_path,
|
||||
@@ -2033,35 +2034,14 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
'channel_id': channel_id,
|
||||
'channel_name': channel_name
|
||||
})
|
||||
|
||||
from lbrynet.extras.daemon.mime_types import guess_media_type
|
||||
|
||||
if file_path:
|
||||
if not os.path.isfile(file_path):
|
||||
raise Exception(f"File {file_path} not found")
|
||||
if os.path.getsize(file_path) == 0:
|
||||
raise Exception(f"Cannot publish empty file {file_path}")
|
||||
claim_dict['stream']['source'] = {}
|
||||
|
||||
stream = await self.stream_manager.create_stream(file_path)
|
||||
stream_hash = stream.stream_hash
|
||||
await self.storage.save_published_file(stream_hash, os.path.basename(file_path),
|
||||
os.path.dirname(file_path), 0)
|
||||
claim_dict['stream']['source']['source'] = stream.sd_hash
|
||||
claim_dict['stream']['source']['sourceType'] = 'lbry_sd_hash'
|
||||
claim_dict['stream']['source']['contentType'] = guess_media_type(file_path)
|
||||
claim_dict['stream']['source']['version'] = "_0_0_1" # need current version here
|
||||
else:
|
||||
if not ('source' not in claim_dict['stream'] and existing_claims):
|
||||
raise Exception("no previous stream to update")
|
||||
claim_dict['stream']['source'] = existing_claims[-1].claim_dict['stream']['source']
|
||||
stream_hash = await self.storage.get_stream_hash_for_sd_hash(claim_dict['stream']['source']['source'])
|
||||
tx = await self.wallet_manager.claim_name(
|
||||
account, name, amount, claim_dict, certificate, claim_address
|
||||
)
|
||||
await self.storage.save_content_claim(
|
||||
stream_hash, tx.outputs[0].id
|
||||
)
|
||||
stream_hash = await self.storage.get_stream_hash_for_sd_hash(sd_hash)
|
||||
if stream_hash:
|
||||
await self.storage.save_content_claim(
|
||||
stream_hash, tx.outputs[0].id
|
||||
)
|
||||
await self.analytics_manager.send_claim_action('publish')
|
||||
nout = 0
|
||||
txo = tx.outputs[nout]
|
||||
@@ -2289,7 +2269,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
}
|
||||
"""
|
||||
claims = await self.wallet_manager.get_claims_for_name(name) # type: dict
|
||||
sort_claim_results(claims['claims'])
|
||||
claims['claims'] = sort_claim_results(claims['claims'])
|
||||
return claims
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
@@ -2569,8 +2549,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
(str) Success/Fail message or (dict) decoded data
|
||||
"""
|
||||
|
||||
downloader = BlobDownloader(asyncio.get_event_loop(), self.blob_manager, self.conf)
|
||||
blob = await downloader.get_blob(blob_hash, self.dht_node)
|
||||
blob = await download_blob(asyncio.get_event_loop(), self.conf, self.blob_manager, self.dht_node, blob_hash)
|
||||
if read:
|
||||
with open(blob.file_path, 'rb') as handle:
|
||||
return handle.read().decode()
|
||||
@@ -2791,7 +2770,19 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
(list) list of blobs reflected
|
||||
"""
|
||||
|
||||
raise NotImplementedError()
|
||||
server, port = kwargs.get('server'), kwargs.get('port')
|
||||
if server and port:
|
||||
port = int(port)
|
||||
else:
|
||||
server, port = random.choice(self.conf.reflector_servers)
|
||||
reflected = await asyncio.gather(*[
|
||||
stream.upload_to_reflector(server, port)
|
||||
for stream in self.stream_manager.get_filtered_streams(**kwargs)
|
||||
])
|
||||
total = []
|
||||
for reflected_for_stream in reflected:
|
||||
total.extend(reflected_for_stream)
|
||||
return total
|
||||
|
||||
@requires(DHT_COMPONENT)
|
||||
async def jsonrpc_peer_ping(self, node_id, address, port):
|
||||
@@ -2817,7 +2808,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
try:
|
||||
result = await peer.ping()
|
||||
return result.decode()
|
||||
except TimeoutError:
|
||||
except asyncio.TimeoutError:
|
||||
return {'error': 'ping timeout'}
|
||||
|
||||
@requires(DHT_COMPONENT)
|
||||
|
||||
@@ -5,9 +5,12 @@ import logging
|
||||
import aiohttp
|
||||
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import Config, ANALYTICS_ENDPOINT, ANALYTICS_TOKEN
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras import system_info
|
||||
|
||||
ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
|
||||
ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
|
||||
|
||||
# Things We Track
|
||||
SERVER_STARTUP = 'Server Startup'
|
||||
SERVER_STARTUP_SUCCESS = 'Server Startup Success'
|
||||
|
||||
@@ -3,6 +3,7 @@ import sqlite3
|
||||
import typing
|
||||
import asyncio
|
||||
import binascii
|
||||
import time
|
||||
from torba.client.basedatabase import SQLiteMixin
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
||||
@@ -274,8 +275,8 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
def get_blobs_to_announce(self):
|
||||
def get_and_update(transaction):
|
||||
timestamp = self.loop.time()
|
||||
if self.conf.announce_head_blobs_only:
|
||||
timestamp = int(self.loop.time())
|
||||
if self.conf.announce_head_and_sd_only:
|
||||
r = transaction.execute(
|
||||
"select blob_hash from blob "
|
||||
"where blob_hash is not null and "
|
||||
@@ -399,16 +400,15 @@ class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
def save_downloaded_file(self, stream_hash, file_name, download_directory, data_payment_rate):
|
||||
return self.save_published_file(
|
||||
stream_hash, binascii.hexlify(file_name.encode()).decode(),
|
||||
binascii.hexlify(download_directory.encode()).decode(), data_payment_rate,
|
||||
status="running"
|
||||
stream_hash, file_name, download_directory, data_payment_rate, status="running"
|
||||
)
|
||||
|
||||
def save_published_file(self, stream_hash: str, file_name: str, download_directory: str, data_payment_rate: float,
|
||||
status="finished"):
|
||||
return self.db.execute(
|
||||
"insert into file values (?, ?, ?, ?, ?)",
|
||||
(stream_hash, file_name, download_directory, data_payment_rate, status)
|
||||
(stream_hash, binascii.hexlify(file_name.encode()).decode(),
|
||||
binascii.hexlify(download_directory.encode()).decode(), data_payment_rate, status)
|
||||
)
|
||||
|
||||
async def get_all_lbry_files(self) -> typing.List[typing.Dict]:
|
||||
@@ -447,6 +447,12 @@ class SQLiteStorage(SQLiteMixin):
|
||||
log.info("update file status %s -> %s", stream_hash, new_status)
|
||||
return self.db.execute("update file set status=? where stream_hash=?", (new_status, stream_hash))
|
||||
|
||||
def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: str, file_name: str):
|
||||
return self.db.execute("update file set download_directory=?, file_name=? where stream_hash=?", (
|
||||
binascii.hexlify(download_dir.encode()).decode(), binascii.hexlify(file_name.encode()).decode(),
|
||||
stream_hash
|
||||
))
|
||||
|
||||
def get_all_stream_hashes(self):
|
||||
return self.run_and_return_list("select stream_hash from stream")
|
||||
|
||||
@@ -682,7 +688,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
if success:
|
||||
return self.db.execute(
|
||||
"insert or replace into reflected_stream values (?, ?, ?)",
|
||||
(sd_hash, reflector_address, self.loop.time())
|
||||
(sd_hash, reflector_address, time.time())
|
||||
)
|
||||
return self.db.execute(
|
||||
"delete from reflected_stream where sd_hash=? and reflector_address=?",
|
||||
@@ -694,5 +700,5 @@ class SQLiteStorage(SQLiteMixin):
|
||||
"select s.sd_hash from stream s "
|
||||
"left outer join reflected_stream r on s.sd_hash=r.sd_hash "
|
||||
"where r.timestamp is null or r.timestamp < ?",
|
||||
self.loop.time() - self.conf.auto_re_reflect_interval
|
||||
self.loop.time() - 86400
|
||||
)
|
||||
+28
-19
@@ -43,11 +43,11 @@ class StreamAssembler:
|
||||
self.written_bytes: int = 0
|
||||
|
||||
async def _decrypt_blob(self, blob: 'BlobFile', blob_info: 'BlobInfo', key: str):
|
||||
offset = blob_info.blob_num * (MAX_BLOB_SIZE - 1)
|
||||
if not blob or self.stream_handle.closed:
|
||||
return False
|
||||
|
||||
def _decrypt_and_write():
|
||||
if self.stream_handle.closed:
|
||||
return False
|
||||
offset = blob_info.blob_num * (MAX_BLOB_SIZE - 1)
|
||||
self.stream_handle.seek(offset)
|
||||
_decrypted = blob.decrypt(
|
||||
binascii.unhexlify(key), binascii.unhexlify(blob_info.iv.encode())
|
||||
@@ -55,25 +55,33 @@ class StreamAssembler:
|
||||
self.stream_handle.write(_decrypted)
|
||||
self.stream_handle.flush()
|
||||
self.written_bytes += len(_decrypted)
|
||||
return True
|
||||
log.debug("decrypted %s", blob.blob_hash[:8])
|
||||
self.wrote_bytes_event.set()
|
||||
|
||||
decrypted = await self.loop.run_in_executor(None, _decrypt_and_write)
|
||||
if decrypted:
|
||||
log.info("decrypted %s", blob.blob_hash[:8])
|
||||
return
|
||||
await self.loop.run_in_executor(None, _decrypt_and_write)
|
||||
|
||||
async def setup(self):
|
||||
pass
|
||||
|
||||
async def after_got_descriptor(self):
|
||||
pass
|
||||
|
||||
async def after_finished(self):
|
||||
pass
|
||||
|
||||
async def assemble_decrypted_stream(self, output_dir: str, output_file_name: typing.Optional[str] = None):
|
||||
if not os.path.isdir(output_dir):
|
||||
raise OSError(f"output directory does not exist: '{output_dir}' '{output_file_name}'")
|
||||
await self.setup()
|
||||
self.sd_blob = await self.get_blob(self.sd_hash)
|
||||
await self.blob_manager.blob_completed(self.sd_blob)
|
||||
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(self.loop, self.blob_manager.blob_dir,
|
||||
self.sd_blob)
|
||||
if not self.got_descriptor.is_set():
|
||||
self.got_descriptor.set()
|
||||
self.output_path = await get_next_available_file_name(self.loop, output_dir,
|
||||
output_file_name or self.descriptor.suggested_file_name)
|
||||
|
||||
if not self.got_descriptor.is_set():
|
||||
self.got_descriptor.set()
|
||||
await self.after_got_descriptor()
|
||||
self.stream_handle = open(self.output_path, 'wb')
|
||||
await self.blob_manager.storage.store_stream(
|
||||
self.sd_blob, self.descriptor
|
||||
@@ -85,17 +93,18 @@ class StreamAssembler:
|
||||
blob = await self.get_blob(blob_info.blob_hash, blob_info.length)
|
||||
await self._decrypt_blob(blob, blob_info, self.descriptor.key)
|
||||
break
|
||||
except ValueError as err:
|
||||
log.error("failed to decrypt blob %s for stream %s - %s", blob_info.blob_hash,
|
||||
self.descriptor.sd_hash, str(err))
|
||||
except FileNotFoundError:
|
||||
log.debug("stream assembler stopped")
|
||||
return
|
||||
except (ValueError, IOError, OSError):
|
||||
log.warning("failed to decrypt blob %s for stream %s", blob_info.blob_hash,
|
||||
self.descriptor.sd_hash)
|
||||
continue
|
||||
if not self.wrote_bytes_event.is_set():
|
||||
self.wrote_bytes_event.set()
|
||||
|
||||
self.stream_finished_event.set()
|
||||
await self.after_finished()
|
||||
finally:
|
||||
self.stream_handle.close()
|
||||
|
||||
async def get_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
|
||||
f = asyncio.Future(loop=self.loop)
|
||||
f.set_result(self.blob_manager.get_blob(blob_hash, length))
|
||||
return await f
|
||||
return self.blob_manager.get_blob(blob_hash, length)
|
||||
+61
-207
@@ -1,13 +1,14 @@
|
||||
import os
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet.utils import drain_tasks, cancel_task
|
||||
from lbrynet.utils import resolve_host
|
||||
from lbrynet.stream.assembler import StreamAssembler
|
||||
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.blob_exchange.downloader import BlobDownloader
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
|
||||
@@ -19,217 +20,70 @@ def drain_into(a: list, b: list):
|
||||
b.append(a.pop())
|
||||
|
||||
|
||||
class StreamDownloader(StreamAssembler): # TODO: reduce duplication, refactor to inherit BlobDownloader
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', sd_hash: str,
|
||||
peer_timeout: float, peer_connect_timeout: float, output_dir: typing.Optional[str] = None,
|
||||
output_file_name: typing.Optional[str] = None,
|
||||
fixed_peers: typing.Optional[typing.List['KademliaPeer']] = None,
|
||||
max_connections_per_stream: typing.Optional[int] = 8):
|
||||
class StreamDownloader(StreamAssembler):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager', sd_hash: str,
|
||||
output_dir: typing.Optional[str] = None, output_file_name: typing.Optional[str] = None):
|
||||
super().__init__(loop, blob_manager, sd_hash)
|
||||
self.peer_timeout = peer_timeout
|
||||
self.peer_connect_timeout = peer_connect_timeout
|
||||
self.current_blob: 'BlobFile' = None
|
||||
|
||||
self.download_task: asyncio.Task = None
|
||||
self.accumulate_connections_task: asyncio.Task = None
|
||||
self.new_peer_event = asyncio.Event(loop=self.loop)
|
||||
self.active_connections: typing.Dict['KademliaPeer', BlobExchangeClientProtocol] = {}
|
||||
self.running_download_requests: typing.List[asyncio.Task] = []
|
||||
self.requested_from: typing.Dict[str, typing.Dict['KademliaPeer', asyncio.Task]] = {}
|
||||
self.output_dir = output_dir or os.getcwd()
|
||||
self.config = config
|
||||
self.output_dir = output_dir or self.config.download_dir
|
||||
self.output_file_name = output_file_name
|
||||
self._lock = asyncio.Lock(loop=self.loop)
|
||||
self.max_connections_per_stream = max_connections_per_stream
|
||||
self.fixed_peers = fixed_peers or []
|
||||
self.blob_downloader: typing.Optional[BlobDownloader] = None
|
||||
self.search_queue = asyncio.Queue(loop=loop)
|
||||
self.peer_queue = asyncio.Queue(loop=loop)
|
||||
self.accumulate_task: typing.Optional[asyncio.Task] = None
|
||||
self.descriptor: typing.Optional[StreamDescriptor]
|
||||
self.node: typing.Optional['Node'] = None
|
||||
self.assemble_task: typing.Optional[asyncio.Task] = None
|
||||
self.fixed_peers_handle: typing.Optional[asyncio.Handle] = None
|
||||
|
||||
async def _update_current_blob(self, blob: 'BlobFile'):
|
||||
async with self._lock:
|
||||
drain_tasks(self.running_download_requests)
|
||||
self.current_blob = blob
|
||||
if not blob.get_is_verified():
|
||||
self._update_requests()
|
||||
async def setup(self): # start the peer accumulator and initialize the downloader
|
||||
if self.blob_downloader:
|
||||
raise Exception("downloader is already set up")
|
||||
if self.node:
|
||||
_, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue)
|
||||
self.blob_downloader = BlobDownloader(self.loop, self.config, self.blob_manager, self.peer_queue)
|
||||
self.search_queue.put_nowait(self.sd_hash)
|
||||
|
||||
async def _request_blob(self, peer: 'KademliaPeer'):
|
||||
if self.current_blob.get_is_verified():
|
||||
log.info("already verified")
|
||||
return
|
||||
if peer not in self.active_connections:
|
||||
log.warning("not active, adding: %s", str(peer))
|
||||
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.peer_timeout)
|
||||
protocol = self.active_connections[peer]
|
||||
success, keep_connection = await request_blob(self.loop, self.current_blob, protocol,
|
||||
peer.address, peer.tcp_port, self.peer_connect_timeout)
|
||||
await protocol.close()
|
||||
if not keep_connection:
|
||||
log.info("drop peer %s:%i", peer.address, peer.tcp_port)
|
||||
if peer in self.active_connections:
|
||||
async with self._lock:
|
||||
del self.active_connections[peer]
|
||||
return
|
||||
log.info("keep peer %s:%i", peer.address, peer.tcp_port)
|
||||
async def after_got_descriptor(self):
|
||||
self.search_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
|
||||
log.info("added head blob to search")
|
||||
|
||||
def _update_requests(self):
|
||||
self.new_peer_event.clear()
|
||||
if self.current_blob.blob_hash not in self.requested_from:
|
||||
self.requested_from[self.current_blob.blob_hash] = {}
|
||||
to_add = []
|
||||
for peer in self.active_connections.keys():
|
||||
if peer not in self.requested_from[self.current_blob.blob_hash] and peer not in to_add:
|
||||
to_add.append(peer)
|
||||
if to_add or self.running_download_requests:
|
||||
log.info("adding download probes for %i peers to %i already active",
|
||||
min(len(to_add), 8 - len(self.running_download_requests)),
|
||||
len(self.running_download_requests))
|
||||
else:
|
||||
log.info("downloader idle...")
|
||||
for peer in to_add:
|
||||
if len(self.running_download_requests) >= 8:
|
||||
break
|
||||
task = self.loop.create_task(self._request_blob(peer))
|
||||
self.requested_from[self.current_blob.blob_hash][peer] = task
|
||||
self.running_download_requests.append(task)
|
||||
async def after_finished(self):
|
||||
log.info("downloaded stream %s -> %s", self.sd_hash, self.output_path)
|
||||
await self.blob_manager.storage.change_file_status(self.descriptor.stream_hash, 'finished')
|
||||
|
||||
async def wait_for_download_or_new_peer(self) -> typing.Optional['BlobFile']:
|
||||
async with self._lock:
|
||||
if len(self.running_download_requests) < self.max_connections_per_stream:
|
||||
# update the running download requests
|
||||
self._update_requests()
|
||||
|
||||
# drain the tasks into a temporary list
|
||||
download_tasks = []
|
||||
drain_into(self.running_download_requests, download_tasks)
|
||||
|
||||
got_new_peer = self.loop.create_task(self.new_peer_event.wait())
|
||||
|
||||
# wait for a new peer to be added or for a download attempt to finish
|
||||
await asyncio.wait([got_new_peer] + download_tasks, return_when='FIRST_COMPLETED',
|
||||
loop=self.loop)
|
||||
if got_new_peer and not got_new_peer.done():
|
||||
got_new_peer.cancel()
|
||||
async with self._lock:
|
||||
if self.current_blob.get_is_verified():
|
||||
if got_new_peer and not got_new_peer.done():
|
||||
got_new_peer.cancel()
|
||||
drain_tasks(download_tasks)
|
||||
return self.current_blob
|
||||
else:
|
||||
for task in download_tasks:
|
||||
if task and not task.done():
|
||||
self.running_download_requests.append(task)
|
||||
return
|
||||
def stop(self):
|
||||
if self.accumulate_task:
|
||||
self.accumulate_task.cancel()
|
||||
self.accumulate_task = None
|
||||
if self.assemble_task:
|
||||
self.assemble_task.cancel()
|
||||
self.assemble_task = None
|
||||
if self.fixed_peers_handle:
|
||||
self.fixed_peers_handle.cancel()
|
||||
self.fixed_peers_handle = None
|
||||
self.blob_downloader = None
|
||||
|
||||
async def get_blob(self, blob_hash: str, length: typing.Optional[int] = None) -> 'BlobFile':
|
||||
blob = self.blob_manager.get_blob(blob_hash, length)
|
||||
await self._update_current_blob(blob)
|
||||
if blob.get_is_verified():
|
||||
return blob
|
||||
return await self.blob_downloader.download_blob(blob_hash, length)
|
||||
|
||||
# the blob must be downloaded
|
||||
try:
|
||||
while not self.current_blob.get_is_verified():
|
||||
if not self.active_connections: # wait for a new connection
|
||||
await self.new_peer_event.wait()
|
||||
continue
|
||||
blob = await self.wait_for_download_or_new_peer()
|
||||
if blob:
|
||||
drain_tasks(self.running_download_requests)
|
||||
return blob
|
||||
return blob
|
||||
except asyncio.CancelledError:
|
||||
drain_tasks(self.running_download_requests)
|
||||
raise
|
||||
|
||||
def _add_peer_protocols(self, peers: typing.List['KademliaPeer']):
|
||||
added = 0
|
||||
for peer in peers:
|
||||
if peer not in self.active_connections:
|
||||
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.peer_timeout)
|
||||
added += 1
|
||||
if added:
|
||||
if not self.new_peer_event.is_set():
|
||||
log.info("added %i new peers", len(peers))
|
||||
self.new_peer_event.set()
|
||||
|
||||
async def _accumulate_connections(self, node: 'Node'):
|
||||
blob_queue = asyncio.Queue(loop=self.loop)
|
||||
blob_queue.put_nowait(self.sd_hash)
|
||||
task = asyncio.create_task(self.got_descriptor.wait())
|
||||
added_peers = asyncio.Event(loop=self.loop)
|
||||
add_fixed_peers_timer: typing.Optional[asyncio.Handle] = None
|
||||
|
||||
if self.fixed_peers:
|
||||
def check_added_peers():
|
||||
if not added_peers.is_set():
|
||||
self._add_peer_protocols(self.fixed_peers)
|
||||
log.info("no dht peers for download yet, adding fixed peer")
|
||||
added_peers.set()
|
||||
|
||||
add_fixed_peers_timer = self.loop.call_later(2, check_added_peers)
|
||||
|
||||
def got_descriptor(f):
|
||||
try:
|
||||
f.result()
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
log.info("add head blob hash to peer search")
|
||||
blob_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
|
||||
|
||||
task.add_done_callback(got_descriptor)
|
||||
try:
|
||||
async with node.stream_peer_search_junction(blob_queue) as search_junction:
|
||||
async for peers in search_junction:
|
||||
if not isinstance(peers, list): # TODO: what's up with this?
|
||||
log.error("not a list: %s %s", peers, str(type(peers)))
|
||||
else:
|
||||
self._add_peer_protocols(peers)
|
||||
if not added_peers.is_set():
|
||||
added_peers.set()
|
||||
return
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
finally:
|
||||
if task and not task.done():
|
||||
task.cancel()
|
||||
log.info("cancelled head blob task")
|
||||
if add_fixed_peers_timer and not add_fixed_peers_timer.cancelled():
|
||||
add_fixed_peers_timer.cancel()
|
||||
|
||||
async def stop(self):
|
||||
cancel_task(self.accumulate_connections_task)
|
||||
self.accumulate_connections_task = None
|
||||
drain_tasks(self.running_download_requests)
|
||||
|
||||
while self.requested_from:
|
||||
_, peer_task_dict = self.requested_from.popitem()
|
||||
while peer_task_dict:
|
||||
peer, task = peer_task_dict.popitem()
|
||||
try:
|
||||
cancel_task(task)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
while self.active_connections:
|
||||
_, client = self.active_connections.popitem()
|
||||
if client:
|
||||
await client.close()
|
||||
log.info("stopped downloader")
|
||||
|
||||
async def _download(self):
|
||||
try:
|
||||
|
||||
log.info("download and decrypt stream")
|
||||
await self.assemble_decrypted_stream(self.output_dir, self.output_file_name)
|
||||
log.info(
|
||||
"downloaded stream %s -> %s", self.sd_hash, self.output_path
|
||||
def add_fixed_peers(self):
|
||||
async def _add_fixed_peers():
|
||||
self.peer_queue.put_nowait([
|
||||
KademliaPeer(self.loop, address=(await resolve_host(url)), tcp_port=port + 1)
|
||||
for url, port in self.config.reflector_servers
|
||||
])
|
||||
if self.config.reflector_servers:
|
||||
self.fixed_peers_handle = self.loop.call_later(
|
||||
self.config.fixed_peer_delay if (
|
||||
'dht' not in self.config.components_to_skip
|
||||
and self.node
|
||||
and len(self.node.protocol.routing_table.get_peers())
|
||||
) else 0.0,
|
||||
lambda: self.loop.create_task(_add_fixed_peers())
|
||||
)
|
||||
await self.blob_manager.storage.change_file_status(
|
||||
self.descriptor.stream_hash, 'finished'
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
finally:
|
||||
await self.stop()
|
||||
|
||||
def download(self, node: 'Node'):
|
||||
self.accumulate_connections_task = self.loop.create_task(self._accumulate_connections(node))
|
||||
self.download_task = self.loop.create_task(self._download())
|
||||
def download(self, node: typing.Optional['Node'] = None):
|
||||
self.node = node
|
||||
self.assemble_task = self.loop.create_task(self.assemble_decrypted_stream(self.config.download_dir))
|
||||
self.add_fixed_peers()
|
||||
@@ -2,13 +2,16 @@ import os
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
import binascii
|
||||
from lbrynet.extras.daemon.mime_types import guess_media_type
|
||||
from lbrynet.stream.downloader import StreamDownloader
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.stream.reflector.client import StreamReflectorClient
|
||||
from lbrynet.extras.daemon.storage import StoredStreamClaim
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.extras.daemon.storage import StoredStreamClaim
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.dht.node import Node
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -20,7 +23,7 @@ class ManagedStream:
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', descriptor: 'StreamDescriptor',
|
||||
download_directory: str, file_name: str, downloader: typing.Optional[StreamDownloader] = None,
|
||||
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional['StoredStreamClaim'] = None):
|
||||
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional[StoredStreamClaim] = None):
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.download_directory = download_directory
|
||||
@@ -30,7 +33,6 @@ class ManagedStream:
|
||||
self.stream_hash = descriptor.stream_hash
|
||||
self.stream_claim_info = claim
|
||||
self._status = status
|
||||
self._store_after_finished: asyncio.Task = None
|
||||
self.fully_reflected = asyncio.Event(loop=self.loop)
|
||||
|
||||
@property
|
||||
@@ -98,13 +100,17 @@ class ManagedStream:
|
||||
def sd_hash(self):
|
||||
return self.descriptor.sd_hash
|
||||
|
||||
@property
|
||||
def blobs_remaining(self) -> int:
|
||||
return self.blobs_in_stream - self.blobs_completed
|
||||
|
||||
def as_dict(self) -> typing.Dict:
|
||||
full_path = os.path.join(self.download_directory, self.file_name)
|
||||
if not os.path.exists(full_path):
|
||||
if not os.path.isfile(full_path):
|
||||
full_path = None
|
||||
mime_type = guess_media_type(os.path.basename(self.file_name))
|
||||
|
||||
if self.downloader:
|
||||
if self.downloader and self.downloader.written_bytes:
|
||||
written_bytes = self.downloader.written_bytes
|
||||
elif full_path:
|
||||
written_bytes = os.stat(full_path).st_size
|
||||
@@ -128,6 +134,7 @@ class ManagedStream:
|
||||
'written_bytes': written_bytes,
|
||||
'blobs_completed': self.blobs_completed,
|
||||
'blobs_in_stream': self.blobs_in_stream,
|
||||
'blobs_remaining': self.blobs_remaining,
|
||||
'status': self.status,
|
||||
'claim_id': self.claim_id,
|
||||
'txid': self.txid,
|
||||
@@ -154,9 +161,13 @@ class ManagedStream:
|
||||
return cls(loop, blob_manager, descriptor, os.path.dirname(file_path), os.path.basename(file_path),
|
||||
status=cls.STATUS_FINISHED)
|
||||
|
||||
async def stop_download(self):
|
||||
def start_download(self, node: typing.Optional['Node']):
|
||||
self.downloader.download(node)
|
||||
self.update_status(self.STATUS_RUNNING)
|
||||
|
||||
def stop_download(self):
|
||||
if self.downloader:
|
||||
await self.downloader.stop()
|
||||
self.downloader.stop()
|
||||
if not self.finished:
|
||||
self.update_status(self.STATUS_STOPPED)
|
||||
|
||||
@@ -177,6 +188,11 @@ class ManagedStream:
|
||||
sent_sd, needed = await protocol.send_descriptor()
|
||||
if sent_sd:
|
||||
sent.append(self.sd_hash)
|
||||
if not sent_sd and not needed:
|
||||
if not self.fully_reflected.is_set():
|
||||
self.fully_reflected.set()
|
||||
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
|
||||
return []
|
||||
except (asyncio.CancelledError, asyncio.TimeoutError, ValueError):
|
||||
if protocol.transport:
|
||||
protocol.transport.close()
|
||||
@@ -193,4 +209,13 @@ class ManagedStream:
|
||||
protocol.transport.close()
|
||||
if not self.fully_reflected.is_set():
|
||||
self.fully_reflected.set()
|
||||
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
|
||||
return sent
|
||||
|
||||
def set_claim(self, claim_info: typing.Dict, claim: 'ClaimDict'):
|
||||
self.stream_claim_info = StoredStreamClaim(
|
||||
self.stream_hash, f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
|
||||
claim_info['name'], claim_info['amount'], claim_info['height'],
|
||||
binascii.hexlify(claim.serialized).decode(), claim.certificate_id, claim_info['address'],
|
||||
claim_info['claim_sequence'], claim_info.get('channel_name')
|
||||
)
|
||||
@@ -15,6 +15,7 @@ log = logging.getLogger(__name__)
|
||||
|
||||
class StreamReflectorClient(asyncio.Protocol):
|
||||
def __init__(self, blob_manager: 'BlobFileManager', descriptor: 'StreamDescriptor'):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.transport: asyncio.StreamWriter = None
|
||||
self.blob_manager = blob_manager
|
||||
self.descriptor = descriptor
|
||||
@@ -26,7 +27,7 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
|
||||
def connection_made(self, transport):
|
||||
self.transport = transport
|
||||
log.info("Connected to reflector")
|
||||
log.debug("Connected to reflector")
|
||||
self.connected.set()
|
||||
|
||||
def connection_lost(self, exc: typing.Optional[Exception]):
|
||||
@@ -45,7 +46,7 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
msg = json.dumps(request_dict)
|
||||
self.transport.write(msg.encode())
|
||||
try:
|
||||
self.pending_request = asyncio.get_event_loop().create_task(self.response_queue.get())
|
||||
self.pending_request = self.loop.create_task(self.response_queue.get())
|
||||
return await self.pending_request
|
||||
finally:
|
||||
self.pending_request = None
|
||||
|
||||
@@ -37,7 +37,8 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
try:
|
||||
self.writer.write(data)
|
||||
except IOError as err:
|
||||
log.error("error downloading blob: %s", err)
|
||||
log.error("error receiving blob: %s", err)
|
||||
self.transport.close()
|
||||
return
|
||||
try:
|
||||
request = json.loads(data.decode())
|
||||
@@ -67,28 +68,33 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
self.send_response({"send_sd_blob": True})
|
||||
try:
|
||||
await asyncio.wait_for(self.sd_blob.finished_writing.wait(), 30, loop=self.loop)
|
||||
self.send_response({"received_sd_blob": True})
|
||||
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
|
||||
self.loop, self.blob_manager.blob_dir, self.sd_blob
|
||||
)
|
||||
self.incoming.clear()
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
self.send_response({"received_sd_blob": True})
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||||
self.send_response({"received_sd_blob": False})
|
||||
self.incoming.clear()
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
self.transport.close()
|
||||
self.send_response({"received_sd_blob": False})
|
||||
return
|
||||
else:
|
||||
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
|
||||
self.loop, self.blob_manager.blob_dir, self.sd_blob
|
||||
)
|
||||
self.incoming.clear()
|
||||
if self.writer:
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
self.send_response({"send_sd_blob": False, 'needed': [
|
||||
blob.blob_hash for blob in self.descriptor.blobs[:-1]
|
||||
if not self.blob_manager.get_blob(blob.blob_hash).get_is_verified()
|
||||
]})
|
||||
return
|
||||
return
|
||||
elif self.descriptor:
|
||||
if 'blob_hash' not in request:
|
||||
|
||||
@@ -4,16 +4,20 @@ import typing
|
||||
import binascii
|
||||
import logging
|
||||
import random
|
||||
from lbrynet.error import ResolveError
|
||||
from lbrynet.stream.downloader import StreamDownloader
|
||||
from lbrynet.stream.managed_stream import ManagedStream
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.extras.daemon.storage import StoredStreamClaim, lbc_to_dewies
|
||||
from lbrynet.schema.uri import parse_lbry_uri
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
from lbrynet.extras.daemon.storage import lbc_to_dewies
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.extras.wallet import LbryWalletManager
|
||||
from lbrynet.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -31,7 +35,9 @@ filter_fields = [
|
||||
'nout',
|
||||
'channel_claim_id',
|
||||
'channel_name',
|
||||
'full_status'
|
||||
'full_status', # TODO: remove
|
||||
'blobs_remaining',
|
||||
'blobs_in_stream'
|
||||
]
|
||||
|
||||
comparison_operators = {
|
||||
@@ -45,42 +51,90 @@ comparison_operators = {
|
||||
|
||||
|
||||
class StreamManager:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', wallet: 'LbryWalletManager',
|
||||
storage: 'SQLiteStorage', node: typing.Optional['Node'], peer_timeout: float,
|
||||
peer_connect_timeout: float, fixed_peers: typing.Optional[typing.List['KademliaPeer']] = None,
|
||||
reflector_servers: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobFileManager',
|
||||
wallet: 'LbryWalletManager', storage: 'SQLiteStorage', node: typing.Optional['Node']):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
self.blob_manager = blob_manager
|
||||
self.wallet = wallet
|
||||
self.storage = storage
|
||||
self.node = node
|
||||
self.peer_timeout = peer_timeout
|
||||
self.peer_connect_timeout = peer_connect_timeout
|
||||
self.streams: typing.Set[ManagedStream] = set()
|
||||
self.starting_streams: typing.Dict[str, asyncio.Future] = {}
|
||||
self.resume_downloading_task: asyncio.Task = None
|
||||
self.re_reflect_task: asyncio.Task = None
|
||||
self.update_stream_finished_futs: typing.List[asyncio.Future] = []
|
||||
self.fixed_peers = fixed_peers
|
||||
self.reflector_servers = reflector_servers
|
||||
|
||||
async def _update_content_claim(self, stream: ManagedStream):
|
||||
claim_info = await self.storage.get_content_claim(stream.stream_hash)
|
||||
stream.set_claim(claim_info, smart_decode(claim_info['value']))
|
||||
|
||||
async def start_stream(self, stream: ManagedStream) -> bool:
|
||||
"""
|
||||
Resume or rebuild a partial or completed stream
|
||||
"""
|
||||
|
||||
path = os.path.join(stream.download_directory, stream.file_name)
|
||||
|
||||
if not stream.running and not os.path.isfile(path):
|
||||
if stream.downloader:
|
||||
stream.downloader.stop()
|
||||
stream.downloader = None
|
||||
|
||||
# the directory is gone, can happen when the folder that contains a published file is deleted
|
||||
# reset the download directory to the default and update the file name
|
||||
if not os.path.isdir(stream.download_directory):
|
||||
stream.download_directory = self.config.download_dir
|
||||
|
||||
stream.downloader = self.make_downloader(
|
||||
stream.sd_hash, stream.download_directory, stream.descriptor.suggested_file_name
|
||||
)
|
||||
if stream.status != ManagedStream.STATUS_FINISHED:
|
||||
await self.storage.change_file_status(stream.stream_hash, 'running')
|
||||
stream.update_status('running')
|
||||
stream.start_download(self.node)
|
||||
try:
|
||||
await asyncio.wait_for(self.loop.create_task(stream.downloader.got_descriptor.wait()),
|
||||
self.config.download_timeout)
|
||||
except asyncio.TimeoutError:
|
||||
stream.stop_download()
|
||||
stream.downloader = None
|
||||
return False
|
||||
file_name = os.path.basename(stream.downloader.output_path)
|
||||
await self.storage.change_file_download_dir_and_file_name(
|
||||
stream.stream_hash, self.config.download_dir, file_name
|
||||
)
|
||||
self.wait_for_stream_finished(stream)
|
||||
return True
|
||||
return True
|
||||
|
||||
def make_downloader(self, sd_hash: str, download_directory: str, file_name: str):
|
||||
return StreamDownloader(
|
||||
self.loop, self.config, self.blob_manager, sd_hash, download_directory, file_name
|
||||
)
|
||||
|
||||
async def add_stream(self, sd_hash: str, file_name: str, download_directory: str, status: str, claim):
|
||||
sd_blob = self.blob_manager.get_blob(sd_hash)
|
||||
if sd_blob.get_is_verified():
|
||||
descriptor = await self.blob_manager.get_stream_descriptor(sd_blob.blob_hash)
|
||||
downloader = self.make_downloader(descriptor.sd_hash, download_directory, file_name)
|
||||
stream = ManagedStream(
|
||||
self.loop, self.blob_manager, descriptor,
|
||||
download_directory,
|
||||
file_name,
|
||||
downloader, status, claim
|
||||
)
|
||||
self.streams.add(stream)
|
||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||
|
||||
async def load_streams_from_database(self):
|
||||
infos = await self.storage.get_all_lbry_files()
|
||||
for file_info in infos:
|
||||
sd_blob = self.blob_manager.get_blob(file_info['sd_hash'])
|
||||
if sd_blob.get_is_verified():
|
||||
descriptor = await self.blob_manager.get_stream_descriptor(sd_blob.blob_hash)
|
||||
downloader = StreamDownloader(
|
||||
self.loop, self.blob_manager, descriptor.sd_hash, self.peer_timeout,
|
||||
self.peer_connect_timeout, binascii.unhexlify(file_info['download_directory']).decode(),
|
||||
binascii.unhexlify(file_info['file_name']).decode(), self.fixed_peers
|
||||
)
|
||||
stream = ManagedStream(
|
||||
self.loop, self.blob_manager, descriptor,
|
||||
binascii.unhexlify(file_info['download_directory']).decode(),
|
||||
binascii.unhexlify(file_info['file_name']).decode(),
|
||||
downloader, file_info['status'], file_info['claim']
|
||||
)
|
||||
self.streams.add(stream)
|
||||
file_infos = await self.storage.get_all_lbry_files()
|
||||
await asyncio.gather(*[
|
||||
self.add_stream(
|
||||
file_info['sd_hash'], binascii.unhexlify(file_info['file_name']).decode(),
|
||||
binascii.unhexlify(file_info['download_directory']).decode(), file_info['status'], file_info['claim']
|
||||
) for file_info in file_infos
|
||||
])
|
||||
|
||||
async def resume(self):
|
||||
if not self.node:
|
||||
@@ -88,38 +142,45 @@ class StreamManager:
|
||||
return
|
||||
await self.node.joined.wait()
|
||||
resumed = 0
|
||||
for stream in self.streams:
|
||||
if stream.status == ManagedStream.STATUS_RUNNING:
|
||||
resumed += 1
|
||||
stream.downloader.download(self.node)
|
||||
self.wait_for_stream_finished(stream)
|
||||
t = [self.start_stream(stream) for stream in self.streams if stream.status == ManagedStream.STATUS_RUNNING]
|
||||
if resumed:
|
||||
log.info("resuming %i downloads", resumed)
|
||||
log.info("resuming %i downloads", t)
|
||||
await asyncio.gather(*t, loop=self.loop)
|
||||
|
||||
async def reflect_streams(self):
|
||||
streams = list(self.streams)
|
||||
batch = []
|
||||
while streams:
|
||||
stream = streams.pop()
|
||||
if not stream.fully_reflected.is_set():
|
||||
host, port = random.choice(self.reflector_servers)
|
||||
batch.append(stream.upload_to_reflector(host, port))
|
||||
if len(batch) >= 10:
|
||||
await asyncio.gather(*batch)
|
||||
while True:
|
||||
if self.config.reflector_servers:
|
||||
sd_hashes = await self.storage.get_streams_to_re_reflect()
|
||||
streams = list(filter(lambda s: s.sd_hash in sd_hashes, self.streams))
|
||||
batch = []
|
||||
if batch:
|
||||
await asyncio.gather(*batch)
|
||||
total = len(streams)
|
||||
while streams:
|
||||
stream = streams.pop()
|
||||
if not stream.fully_reflected.is_set():
|
||||
host, port = random.choice(self.config.reflector_servers)
|
||||
batch.append(stream.upload_to_reflector(host, port))
|
||||
if len(batch) >= self.config.concurrent_reflector_uploads:
|
||||
await asyncio.gather(*batch)
|
||||
batch = []
|
||||
if batch:
|
||||
await asyncio.gather(*batch)
|
||||
if total:
|
||||
log.info("uploaded %i streams to reflector", total)
|
||||
await asyncio.sleep(300, loop=self.loop)
|
||||
|
||||
async def start(self):
|
||||
await self.load_streams_from_database()
|
||||
self.resume_downloading_task = self.loop.create_task(self.resume())
|
||||
self.re_reflect_task = self.loop.create_task(self.reflect_streams())
|
||||
|
||||
async def stop(self):
|
||||
def stop(self):
|
||||
if self.resume_downloading_task and not self.resume_downloading_task.done():
|
||||
self.resume_downloading_task.cancel()
|
||||
if self.re_reflect_task and not self.re_reflect_task.done():
|
||||
self.re_reflect_task.cancel()
|
||||
while self.streams:
|
||||
stream = self.streams.pop()
|
||||
await stream.stop_download()
|
||||
stream.stop_download()
|
||||
while self.update_stream_finished_futs:
|
||||
self.update_stream_finished_futs.pop().cancel()
|
||||
|
||||
@@ -127,24 +188,21 @@ class StreamManager:
|
||||
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
|
||||
stream = await ManagedStream.create(self.loop, self.blob_manager, file_path, key, iv_generator)
|
||||
self.streams.add(stream)
|
||||
if self.reflector_servers:
|
||||
host, port = random.choice(self.reflector_servers)
|
||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||
if self.config.reflector_servers:
|
||||
host, port = random.choice(self.config.reflector_servers)
|
||||
self.loop.create_task(stream.upload_to_reflector(host, port))
|
||||
return stream
|
||||
|
||||
async def delete_stream(self, stream: ManagedStream, delete_file: typing.Optional[bool] = False):
|
||||
await stream.stop_download()
|
||||
stream.stop_download()
|
||||
self.streams.remove(stream)
|
||||
await self.storage.delete_stream(stream.descriptor)
|
||||
|
||||
blob_hashes = [stream.sd_hash]
|
||||
for blob_info in stream.descriptor.blobs[:-1]:
|
||||
blob_hashes.append(blob_info.blob_hash)
|
||||
for blob_hash in blob_hashes:
|
||||
blob = self.blob_manager.get_blob(blob_hash)
|
||||
if blob.get_is_verified():
|
||||
await blob.delete()
|
||||
|
||||
await self.blob_manager.delete_blobs(blob_hashes)
|
||||
if delete_file:
|
||||
path = os.path.join(stream.download_directory, stream.file_name)
|
||||
if os.path.isfile(path):
|
||||
@@ -166,76 +224,70 @@ class StreamManager:
|
||||
)
|
||||
|
||||
async def _download_stream_from_claim(self, node: 'Node', download_directory: str, claim_info: typing.Dict,
|
||||
file_name: typing.Optional[str] = None, data_rate: typing.Optional[int] = 0,
|
||||
sd_blob_timeout: typing.Optional[float] = 60
|
||||
) -> typing.Optional[ManagedStream]:
|
||||
file_name: typing.Optional[str] = None) -> typing.Optional[ManagedStream]:
|
||||
|
||||
claim = ClaimDict.load_dict(claim_info['value'])
|
||||
downloader = StreamDownloader(self.loop, self.blob_manager, claim.source_hash.decode(), self.peer_timeout,
|
||||
self.peer_connect_timeout, download_directory, file_name, self.fixed_peers)
|
||||
claim = smart_decode(claim_info['value'])
|
||||
downloader = StreamDownloader(self.loop, self.config, self.blob_manager, claim.source_hash.decode(),
|
||||
download_directory, file_name)
|
||||
try:
|
||||
downloader.download(node)
|
||||
await asyncio.wait_for(downloader.got_descriptor.wait(), sd_blob_timeout)
|
||||
await downloader.got_descriptor.wait()
|
||||
log.info("got descriptor %s for %s", claim.source_hash.decode(), claim_info['name'])
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||||
log.info("stream timeout")
|
||||
await downloader.stop()
|
||||
downloader.stop()
|
||||
log.info("stopped stream")
|
||||
return
|
||||
file_name = os.path.basename(downloader.output_path)
|
||||
download_directory = os.path.dirname(downloader.output_path)
|
||||
if not await self.blob_manager.storage.stream_exists(downloader.sd_hash):
|
||||
await self.blob_manager.storage.store_stream(downloader.sd_blob, downloader.descriptor)
|
||||
if not await self.blob_manager.storage.file_exists(downloader.sd_hash):
|
||||
await self.blob_manager.storage.save_downloaded_file(
|
||||
downloader.descriptor.stream_hash, os.path.basename(downloader.output_path), download_directory,
|
||||
data_rate
|
||||
downloader.descriptor.stream_hash, file_name, download_directory,
|
||||
0.0
|
||||
)
|
||||
await self.blob_manager.storage.save_content_claim(
|
||||
downloader.descriptor.stream_hash, f"{claim_info['txid']}:{claim_info['nout']}"
|
||||
)
|
||||
|
||||
stored_claim = StoredStreamClaim(
|
||||
downloader.descriptor.stream_hash, f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
|
||||
claim_info['name'], claim_info['amount'], claim_info['height'], claim_info['hex'],
|
||||
claim.certificate_id, claim_info['address'], claim_info['claim_sequence'],
|
||||
claim_info.get('channel_name')
|
||||
)
|
||||
stream = ManagedStream(self.loop, self.blob_manager, downloader.descriptor, download_directory,
|
||||
os.path.basename(downloader.output_path), downloader, ManagedStream.STATUS_RUNNING,
|
||||
stored_claim)
|
||||
file_name, downloader, ManagedStream.STATUS_RUNNING)
|
||||
stream.set_claim(claim_info, claim)
|
||||
self.streams.add(stream)
|
||||
try:
|
||||
await stream.downloader.wrote_bytes_event.wait()
|
||||
self.wait_for_stream_finished(stream)
|
||||
return stream
|
||||
except asyncio.CancelledError:
|
||||
await downloader.stop()
|
||||
downloader.stop()
|
||||
log.debug("stopped stream")
|
||||
|
||||
async def download_stream_from_claim(self, node: 'Node', download_directory: str, claim_info: typing.Dict,
|
||||
async def download_stream_from_claim(self, node: 'Node', claim_info: typing.Dict,
|
||||
file_name: typing.Optional[str] = None,
|
||||
sd_blob_timeout: typing.Optional[float] = 60,
|
||||
timeout: typing.Optional[float] = 60,
|
||||
fee_amount: typing.Optional[float] = 0.0,
|
||||
fee_address: typing.Optional[str] = None) -> typing.Optional[ManagedStream]:
|
||||
fee_address: typing.Optional[str] = None,
|
||||
should_pay: typing.Optional[bool] = True) -> typing.Optional[ManagedStream]:
|
||||
log.info("get lbry://%s#%s", claim_info['name'], claim_info['claim_id'])
|
||||
claim = ClaimDict.load_dict(claim_info['value'])
|
||||
if fee_address and fee_amount:
|
||||
if fee_amount > await self.wallet.default_account.get_balance():
|
||||
raise Exception("not enough funds")
|
||||
sd_hash = claim.source_hash.decode()
|
||||
if sd_hash in self.starting_streams:
|
||||
return await self.starting_streams[sd_hash]
|
||||
already_started = tuple(filter(lambda s: s.descriptor.sd_hash == sd_hash, self.streams))
|
||||
if already_started:
|
||||
return already_started[0]
|
||||
if should_pay and fee_address and fee_amount and fee_amount > await self.wallet.default_account.get_balance():
|
||||
raise Exception("not enough funds")
|
||||
|
||||
self.starting_streams[sd_hash] = asyncio.Future(loop=self.loop)
|
||||
stream_task = self.loop.create_task(
|
||||
self._download_stream_from_claim(node, download_directory, claim_info, file_name, 0, sd_blob_timeout)
|
||||
self._download_stream_from_claim(node, self.config.download_dir, claim_info, file_name)
|
||||
)
|
||||
try:
|
||||
await asyncio.wait_for(stream_task, sd_blob_timeout)
|
||||
await asyncio.wait_for(stream_task, timeout or self.config.download_timeout)
|
||||
stream = await stream_task
|
||||
self.starting_streams[sd_hash].set_result(stream)
|
||||
if fee_address and fee_amount:
|
||||
if should_pay and fee_address and fee_amount:
|
||||
await self.wallet.send_amount_to_address(lbc_to_dewies(str(fee_amount)), fee_address.encode('latin1'))
|
||||
return stream
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||||
@@ -245,6 +297,11 @@ class StreamManager:
|
||||
del self.starting_streams[sd_hash]
|
||||
log.info("returned from get lbry://%s#%s", claim_info['name'], claim_info['claim_id'])
|
||||
|
||||
def get_stream_by_stream_hash(self, stream_hash: str) -> typing.Optional[ManagedStream]:
|
||||
streams = tuple(filter(lambda stream: stream.stream_hash == stream_hash, self.streams))
|
||||
if streams:
|
||||
return streams[0]
|
||||
|
||||
def get_filtered_streams(self, sort_by: typing.Optional[str] = None, reverse: typing.Optional[bool] = False,
|
||||
comparison: typing.Optional[str] = None,
|
||||
**search_by) -> typing.List[ManagedStream]:
|
||||
@@ -280,3 +337,60 @@ class StreamManager:
|
||||
if reverse:
|
||||
streams.reverse()
|
||||
return streams
|
||||
|
||||
async def download_stream_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
|
||||
file_name: typing.Optional[str] = None,
|
||||
timeout: typing.Optional[float] = None) -> typing.Optional[ManagedStream]:
|
||||
timeout = timeout or self.config.download_timeout
|
||||
parsed_uri = parse_lbry_uri(uri)
|
||||
if parsed_uri.is_channel:
|
||||
raise Exception("cannot download a channel claim, specify a /path")
|
||||
|
||||
resolved = (await self.wallet.resolve(uri)).get(uri, {})
|
||||
resolved = resolved if 'value' in resolved else resolved.get('claim')
|
||||
|
||||
if not resolved:
|
||||
raise ResolveError(
|
||||
"Failed to resolve stream at lbry://{}".format(uri.replace("lbry://", ""))
|
||||
)
|
||||
if 'error' in resolved:
|
||||
raise ResolveError(f"error resolving stream: {resolved['error']}")
|
||||
|
||||
claim = ClaimDict.load_dict(resolved['value'])
|
||||
fee_amount, fee_address = None, None
|
||||
if claim.has_fee:
|
||||
fee_amount = round(exchange_rate_manager.convert_currency(
|
||||
claim.source_fee.currency, "LBC", claim.source_fee.amount
|
||||
), 5)
|
||||
fee_address = claim.source_fee.address.decode()
|
||||
outpoint = f"{resolved['txid']}:{resolved['nout']}"
|
||||
existing = self.get_filtered_streams(outpoint=outpoint)
|
||||
|
||||
if not existing:
|
||||
existing.extend(self.get_filtered_streams(sd_hash=claim.source_hash.decode()))
|
||||
if existing and existing[0].claim_id != resolved['claim_id']:
|
||||
raise Exception(f"stream for {existing[0].claim_id} collides with existing "
|
||||
f"download {resolved['claim_id']}")
|
||||
elif not existing:
|
||||
existing.extend(self.get_filtered_streams(claim_id=resolved['claim_id']))
|
||||
if existing and existing[0].sd_hash != claim.source_hash.decode():
|
||||
log.info("claim contains an update to a stream we have, downloading it")
|
||||
stream = await self.download_stream_from_claim(
|
||||
self.node, resolved, file_name, timeout, fee_amount, fee_address, False
|
||||
)
|
||||
log.info("started new stream, deleting old one")
|
||||
await self.delete_stream(existing[0])
|
||||
return stream
|
||||
elif existing:
|
||||
log.info("already have matching stream for %s", uri)
|
||||
stream = existing[0]
|
||||
await self.start_stream(stream)
|
||||
return stream
|
||||
else:
|
||||
stream = existing[0]
|
||||
await self.start_stream(stream)
|
||||
return stream
|
||||
log.info("download stream from %s", uri)
|
||||
return await self.download_stream_from_claim(
|
||||
self.node, resolved, file_name, timeout, fee_amount, fee_address
|
||||
)
|
||||
@@ -8,6 +8,7 @@ import json
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
import ipaddress
|
||||
import pkg_resources
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
@@ -136,3 +137,16 @@ def cancel_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
|
||||
def drain_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
|
||||
while tasks:
|
||||
cancel_task(tasks.pop())
|
||||
|
||||
|
||||
async def resolve_host(url: str) -> str:
|
||||
try:
|
||||
if ipaddress.ip_address(url):
|
||||
return url
|
||||
except ValueError:
|
||||
pass
|
||||
loop = asyncio.get_running_loop()
|
||||
return (await loop.getaddrinfo(
|
||||
url, 'https',
|
||||
proto=socket.IPPROTO_TCP,
|
||||
))[0][4][0]
|
||||
@@ -0,0 +1,53 @@
|
||||
import sys
|
||||
import os
|
||||
import asyncio
|
||||
import socket
|
||||
import ipaddress
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
|
||||
import logging
|
||||
|
||||
log = logging.getLogger("lbrynet")
|
||||
log.addHandler(logging.StreamHandler())
|
||||
log.setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
async def main(blob_hash: str, url: str):
|
||||
conf = Config()
|
||||
loop = asyncio.get_running_loop()
|
||||
host_url, port = url.split(":")
|
||||
try:
|
||||
host = None
|
||||
if ipaddress.ip_address(host_url):
|
||||
host = host_url
|
||||
except ValueError:
|
||||
host = None
|
||||
if not host:
|
||||
host_info = await loop.getaddrinfo(
|
||||
host_url, 'https',
|
||||
proto=socket.IPPROTO_TCP,
|
||||
)
|
||||
host = host_info[0][4][0]
|
||||
|
||||
storage = SQLiteStorage(conf, os.path.join(conf.data_dir, "lbrynet.sqlite"))
|
||||
blob_manager = BlobFileManager(loop, os.path.join(conf.data_dir, "blobfiles"), storage)
|
||||
await storage.open()
|
||||
await blob_manager.setup()
|
||||
|
||||
blob = blob_manager.get_blob(blob_hash)
|
||||
success, keep = await request_blob(loop, blob, host, int(port), conf.peer_connect_timeout,
|
||||
conf.blob_download_timeout)
|
||||
print(f"{'downloaded' if success else 'failed to download'} {blob_hash} from {host}:{port}\n"
|
||||
f"keep connection: {keep}")
|
||||
if blob.get_is_verified():
|
||||
await blob_manager.delete_blobs([blob.blob_hash])
|
||||
print(f"deleted {blob_hash}")
|
||||
|
||||
|
||||
if __name__ == "__main__": # usage: python download_blob_from_peer.py <blob_hash> [host url:port]
|
||||
url = 'reflector.lbry.io:5567'
|
||||
if len(sys.argv) > 2:
|
||||
url = sys.argv[2]
|
||||
asyncio.run(main(sys.argv[1], url))
|
||||
@@ -0,0 +1,33 @@
|
||||
import os
|
||||
import binascii
|
||||
import sqlite3
|
||||
from lbrynet.conf import Config
|
||||
|
||||
|
||||
def main():
|
||||
conf = Config()
|
||||
db = sqlite3.connect(os.path.join(conf.data_dir, 'lbrynet.sqlite'))
|
||||
cur = db.cursor()
|
||||
files = cur.execute("select stream_hash, file_name, download_directory from file").fetchall()
|
||||
update = {}
|
||||
for stream_hash, file_name, download_directory in files:
|
||||
try:
|
||||
binascii.unhexlify(file_name)
|
||||
except binascii.Error:
|
||||
try:
|
||||
binascii.unhexlify(download_directory)
|
||||
except binascii.Error:
|
||||
update[stream_hash] = (
|
||||
binascii.hexlify(file_name.encode()).decode(), binascii.hexlify(download_directory.encode()).decode()
|
||||
)
|
||||
if update:
|
||||
print(f"repair {len(update)} streams")
|
||||
for stream_hash, (file_name, download_directory) in update.items():
|
||||
cur.execute('update file set file_name=?, download_directory=? where stream_hash=?',
|
||||
(file_name, download_directory, stream_hash))
|
||||
db.commit()
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -57,31 +57,29 @@ def variance(times):
|
||||
return round(sum(((i - mean) ** 2.0 for i in times)) / (len(times) - 1), 3)
|
||||
|
||||
|
||||
async def wait_for_done(api, uri):
|
||||
async def wait_for_done(conf, uri):
|
||||
name = uri.split("#")[0]
|
||||
last_complete = 0
|
||||
hang_count = 0
|
||||
while True:
|
||||
files = await api.file_list(claim_name=name)
|
||||
files = await daemon_rpc(conf, "file_list", claim_name=name)
|
||||
file = files[0]
|
||||
if file['status'] in ['finished', 'stopped']:
|
||||
return True, f"{file['blobs_completed']}/{file['blobs_in_stream']}", int(file['blobs_completed'])
|
||||
if last_complete < int(file['blobs_completed']):
|
||||
print(f"{file['blobs_completed']}/{file['blobs_in_stream']}...")
|
||||
hang_count = 0
|
||||
last_complete = int(file['blobs_completed'])
|
||||
else:
|
||||
hang_count += 1
|
||||
await asyncio.sleep(1.0)
|
||||
if hang_count > 30:
|
||||
if hang_count > 10:
|
||||
return False, f"{file['blobs_completed']}/{file['blobs_in_stream']}", int(file['blobs_completed'])
|
||||
|
||||
|
||||
async def main(start_daemon=True, uris=None):
|
||||
async def main(uris=None):
|
||||
if not uris:
|
||||
uris = await get_frontpage_uris()
|
||||
conf = Config()
|
||||
daemon = None
|
||||
try:
|
||||
await daemon_rpc(conf, 'status')
|
||||
except (ClientConnectorError, ConnectionError):
|
||||
@@ -113,12 +111,12 @@ async def main(start_daemon=True, uris=None):
|
||||
first_byte = time.time()
|
||||
first_byte_times.append(first_byte - start)
|
||||
print(f"{i + 1}/{len(resolvable)} - {first_byte - start} {uri}")
|
||||
# downloaded, msg, blobs_in_stream = await wait_for_done(api, uri)
|
||||
# downloaded, msg, blobs_in_stream = await wait_for_done(conf, uri)
|
||||
# if downloaded:
|
||||
# downloaded_times.append((time.time()-start) / downloaded)
|
||||
# print(f"{i + 1}/{len(uris)} - downloaded @ {(time.time()-start) / blobs_in_stream}, {msg} {uri}")
|
||||
# print(f"\tdownloaded {uri} @ {(time.time()-start) / blobs_in_stream} seconds per blob")
|
||||
# else:
|
||||
# print(f"failed to downlload {uri}, got {msg}")
|
||||
# print(f"\tfailed to download {uri}, got {msg}")
|
||||
# download_failures.append(uri)
|
||||
except:
|
||||
print(f"{i + 1}/{len(uris)} - timeout in {time.time() - start} {uri}")
|
||||
@@ -136,8 +134,7 @@ async def main(start_daemon=True, uris=None):
|
||||
nt = '\n\t'
|
||||
result += f"\nFailures:\n\t{nt.join([f for f in failures])}"
|
||||
print(result)
|
||||
if daemon:
|
||||
await daemon.shutdown()
|
||||
|
||||
# webhook = os.environ.get('TTFB_SLACK_TOKEN', None)
|
||||
# if webhook:
|
||||
# await report_to_slack(result, webhook)
|
||||
|
||||
@@ -41,9 +41,10 @@ class CommandTestCase(IntegrationTestCase):
|
||||
conf.download_dir = self.wallet_node.data_path
|
||||
conf.share_usage_data = False
|
||||
conf.use_upnp = False
|
||||
conf.reflect_uploads = False
|
||||
conf.reflect_streams = False
|
||||
conf.blockchain_name = 'lbrycrd_regtest'
|
||||
conf.lbryum_servers = [('localhost', 50001)]
|
||||
conf.reflector_servers = []
|
||||
conf.known_dht_nodes = []
|
||||
|
||||
await self.account.ensure_address_gap()
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
import asyncio
|
||||
import tempfile
|
||||
from io import BytesIO
|
||||
|
||||
import shutil
|
||||
import os
|
||||
|
||||
from lbrynet.blob_exchange.serialization import BlobRequest
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob_exchange.server import BlobServer
|
||||
from lbrynet.blob_exchange.server import BlobServer, BlobServerProtocol
|
||||
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
|
||||
from lbrynet.dht.peer import KademliaPeer, PeerManager
|
||||
|
||||
@@ -14,6 +18,12 @@ from lbrynet.dht.peer import KademliaPeer, PeerManager
|
||||
# logging.getLogger("lbrynet").setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
def mock_config():
|
||||
config = Config()
|
||||
config.fixed_peer_delay = 10000
|
||||
return config
|
||||
|
||||
|
||||
class BlobExchangeTestBase(AsyncioTestCase):
|
||||
async def asyncSetUp(self):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
@@ -22,12 +32,15 @@ class BlobExchangeTestBase(AsyncioTestCase):
|
||||
self.server_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, self.client_dir)
|
||||
self.addCleanup(shutil.rmtree, self.server_dir)
|
||||
|
||||
self.server_storage = SQLiteStorage(Config(), os.path.join(self.server_dir, "lbrynet.sqlite"))
|
||||
self.server_config = Config(data_dir=self.server_dir, download_dir=self.server_dir, wallet=self.server_dir,
|
||||
reflector_servers=[])
|
||||
self.server_storage = SQLiteStorage(self.server_config, os.path.join(self.server_dir, "lbrynet.sqlite"))
|
||||
self.server_blob_manager = BlobFileManager(self.loop, self.server_dir, self.server_storage)
|
||||
self.server = BlobServer(self.loop, self.server_blob_manager, 'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP')
|
||||
|
||||
self.client_storage = SQLiteStorage(Config(), os.path.join(self.client_dir, "lbrynet.sqlite"))
|
||||
self.client_config = Config(data_dir=self.client_dir, download_dir=self.client_dir, wallet=self.client_dir,
|
||||
reflector_servers=[])
|
||||
self.client_storage = SQLiteStorage(self.client_config, os.path.join(self.client_dir, "lbrynet.sqlite"))
|
||||
self.client_blob_manager = BlobFileManager(self.loop, self.client_dir, self.client_storage)
|
||||
self.client_peer_manager = PeerManager(self.loop)
|
||||
self.server_from_client = KademliaPeer(self.loop, "127.0.0.1", b'1' * 48, tcp_port=33333)
|
||||
@@ -53,12 +66,10 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
|
||||
async def _test_transfer_blob(self, blob_hash: str):
|
||||
client_blob = self.client_blob_manager.get_blob(blob_hash)
|
||||
protocol = BlobExchangeClientProtocol(self.loop, 2)
|
||||
|
||||
# download the blob
|
||||
downloaded = await request_blob(self.loop, client_blob, protocol, self.server_from_client.address,
|
||||
self.server_from_client.tcp_port, 2)
|
||||
await protocol.close()
|
||||
downloaded = await request_blob(self.loop, client_blob, self.server_from_client.address,
|
||||
self.server_from_client.tcp_port, 2, 3)
|
||||
self.assertEqual(client_blob.get_is_verified(), True)
|
||||
self.assertTrue(downloaded)
|
||||
|
||||
@@ -91,17 +102,15 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
await self._add_blob_to_server(blob_hash, mock_blob_bytes)
|
||||
|
||||
second_client_blob = self.client_blob_manager.get_blob(blob_hash)
|
||||
protocol = BlobExchangeClientProtocol(self.loop, 2)
|
||||
|
||||
# download the blob
|
||||
await asyncio.gather(
|
||||
request_blob(
|
||||
self.loop, second_client_blob, protocol, server_from_second_client.address,
|
||||
server_from_second_client.tcp_port, 2
|
||||
self.loop, second_client_blob, server_from_second_client.address,
|
||||
server_from_second_client.tcp_port, 2, 3
|
||||
),
|
||||
self._test_transfer_blob(blob_hash)
|
||||
)
|
||||
await protocol.close()
|
||||
self.assertEqual(second_client_blob.get_is_verified(), True)
|
||||
|
||||
async def test_host_different_blobs_to_multiple_peers_at_once(self):
|
||||
@@ -125,14 +134,25 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
await self._add_blob_to_server(sd_hash, mock_sd_blob_bytes)
|
||||
|
||||
second_client_blob = self.client_blob_manager.get_blob(blob_hash)
|
||||
protocol = BlobExchangeClientProtocol(self.loop, 2)
|
||||
|
||||
await asyncio.gather(
|
||||
request_blob(
|
||||
self.loop, second_client_blob, protocol, server_from_second_client.address,
|
||||
server_from_second_client.tcp_port, 2
|
||||
self.loop, second_client_blob, server_from_second_client.address,
|
||||
server_from_second_client.tcp_port, 2, 3
|
||||
),
|
||||
self._test_transfer_blob(sd_hash)
|
||||
)
|
||||
await protocol.close()
|
||||
self.assertEqual(second_client_blob.get_is_verified(), True)
|
||||
|
||||
async def test_server_chunked_request(self):
|
||||
blob_hash = "7f5ab2def99f0ddd008da71db3a3772135f4002b19b7605840ed1034c8955431bd7079549e65e6b2a3b9c17c773073ed"
|
||||
server_protocol = BlobServerProtocol(self.loop, self.server_blob_manager, self.server.lbrycrd_address)
|
||||
transport = asyncio.Transport(extra={'peername': ('ip', 90)})
|
||||
received_data = BytesIO()
|
||||
transport.write = received_data.write
|
||||
server_protocol.connection_made(transport)
|
||||
blob_request = BlobRequest.make_request_for_blob_hash(blob_hash).serialize()
|
||||
for byte in blob_request:
|
||||
server_protocol.data_received(bytes([byte]))
|
||||
await asyncio.sleep(0.1) # yield execution
|
||||
self.assertTrue(len(received_data.getvalue()) > 0)
|
||||
@@ -1,6 +1,8 @@
|
||||
import asyncio
|
||||
import binascii
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from tests import dht_mocks
|
||||
from lbrynet.dht.serialization.bencoding import bencode, bdecode
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.protocol.protocol import KademliaProtocol
|
||||
from lbrynet.dht.peer import PeerManager
|
||||
@@ -67,6 +69,7 @@ class TestProtocol(AsyncioTestCase):
|
||||
peer2_from_peer1 = peer1.peer_manager.get_kademlia_peer(
|
||||
peer2.node_id, peer2.external_ip, udp_port=peer2.udp_port
|
||||
)
|
||||
peer2_from_peer1.update_tcp_port(3333)
|
||||
peer3 = peer1.peer_manager.get_kademlia_peer(
|
||||
constants.generate_id(), '1.2.3.6', udp_port=4444
|
||||
)
|
||||
@@ -76,14 +79,14 @@ class TestProtocol(AsyncioTestCase):
|
||||
self.assertEqual(True, peer1.data_store.has_peers_for_blob(b'2' * 48))
|
||||
self.assertEqual(False, peer1.data_store.has_peers_for_blob(b'3' * 48))
|
||||
self.assertListEqual([peer2_from_peer1], peer1.data_store.get_storing_contacts())
|
||||
|
||||
peer1.data_store.completed_blobs.add(binascii.hexlify(b'2' * 48).decode())
|
||||
find_value_response = peer1.node_rpc.find_value(peer3, b'2' * 48)
|
||||
self.assertSetEqual(
|
||||
{b'2' * 48, b'token', b'protocolVersion'}, set(find_value_response.keys())
|
||||
)
|
||||
self.assertEqual(1, len(find_value_response[b'2' * 48]))
|
||||
self.assertEqual(find_value_response[b'2' * 48][0], peer2_from_peer1)
|
||||
# self.assertEqual(peer2_from_peer1.tcp_port, 3333)
|
||||
self.assertEqual(2, len(find_value_response[b'2' * 48]))
|
||||
self.assertEqual(find_value_response[b'2' * 48][0], peer2_from_peer1.compact_address_tcp())
|
||||
self.assertDictEqual(bdecode(bencode(find_value_response)), find_value_response)
|
||||
|
||||
peer1.stop()
|
||||
peer2.stop()
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import unittest
|
||||
import asyncio
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from lbrynet.dht.protocol.async_generator_junction import AsyncGeneratorJunction
|
||||
@@ -41,16 +42,17 @@ class TestAsyncGeneratorJunction(AsyncioTestCase):
|
||||
|
||||
async def test_yield_order(self):
|
||||
expected_order = [1, 2, 1, 1, 2, 1, 1, 2, 1, 1, 2, 1, 1, 2, 1, 2, 2, 2, 2, 2]
|
||||
fast_gen = MockAsyncGen(self.loop, 1, 0.1)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.2)
|
||||
fast_gen = MockAsyncGen(self.loop, 1, 0.2)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.4)
|
||||
await self._test_junction(expected_order, fast_gen, slow_gen)
|
||||
self.assertEqual(fast_gen.called_close, True)
|
||||
self.assertEqual(slow_gen.called_close, True)
|
||||
|
||||
@unittest.SkipTest
|
||||
async def test_one_stopped_first(self):
|
||||
expected_order = [1, 2, 1, 1, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2]
|
||||
fast_gen = MockAsyncGen(self.loop, 1, 0.101, 5)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.201)
|
||||
fast_gen = MockAsyncGen(self.loop, 1, 0.2, 5)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.4)
|
||||
await self._test_junction(expected_order, fast_gen, slow_gen)
|
||||
self.assertEqual(fast_gen.called_close, True)
|
||||
self.assertEqual(slow_gen.called_close, True)
|
||||
@@ -62,16 +64,16 @@ class TestAsyncGeneratorJunction(AsyncioTestCase):
|
||||
for i in range(10):
|
||||
if i == 5:
|
||||
return
|
||||
await asyncio.sleep(0.101)
|
||||
await asyncio.sleep(0.2)
|
||||
yield 1
|
||||
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.201)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.4)
|
||||
await self._test_junction(expected_order, fast_gen(), slow_gen)
|
||||
self.assertEqual(slow_gen.called_close, True)
|
||||
|
||||
async def test_stop_when_encapsulating_task_cancelled(self):
|
||||
fast_gen = MockAsyncGen(self.loop, 1, 0.1)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.2)
|
||||
fast_gen = MockAsyncGen(self.loop, 1, 0.2)
|
||||
slow_gen = MockAsyncGen(self.loop, 2, 0.4)
|
||||
|
||||
async def _task():
|
||||
async with AsyncGeneratorJunction(self.loop) as junction:
|
||||
@@ -81,7 +83,7 @@ class TestAsyncGeneratorJunction(AsyncioTestCase):
|
||||
pass
|
||||
|
||||
task = self.loop.create_task(_task())
|
||||
self.loop.call_later(0.5, task.cancel)
|
||||
self.loop.call_later(1.0, task.cancel)
|
||||
with self.assertRaises(asyncio.CancelledError):
|
||||
await task
|
||||
self.assertEqual(fast_gen.called_close, True)
|
||||
|
||||
@@ -84,15 +84,6 @@ class TestCostEst(unittest.TestCase):
|
||||
result = yield f2d(daemon.get_est_cost("test", size))
|
||||
self.assertEqual(result, correct_result)
|
||||
|
||||
def test_fee_and_ungenerous_data(self):
|
||||
conf = Config(is_generous_host=False)
|
||||
size = 10000000
|
||||
fake_fee_amount = 4.5
|
||||
correct_result = size / 10 ** 6 * conf.data_rate + fake_fee_amount
|
||||
daemon = get_test_daemon(conf, with_fee=True)
|
||||
result = yield f2d(daemon.get_est_cost("test", size))
|
||||
self.assertEqual(result, round(correct_result, 1))
|
||||
|
||||
def test_generous_data_and_no_fee(self):
|
||||
size = 10000000
|
||||
correct_result = 0.0
|
||||
@@ -100,14 +91,6 @@ class TestCostEst(unittest.TestCase):
|
||||
result = yield f2d(daemon.get_est_cost("test", size))
|
||||
self.assertEqual(result, correct_result)
|
||||
|
||||
def test_ungenerous_data_and_no_fee(self):
|
||||
conf = Config(is_generous_host=False)
|
||||
size = 10000000
|
||||
correct_result = size / 10 ** 6 * conf.data_rate
|
||||
daemon = get_test_daemon(conf)
|
||||
result = yield f2d(daemon.get_est_cost("test", size))
|
||||
self.assertEqual(result, round(correct_result, 1))
|
||||
|
||||
|
||||
@unittest.SkipTest
|
||||
class TestJsonRpc(unittest.TestCase):
|
||||
|
||||
@@ -7,8 +7,8 @@ from lbrynet.conf import Config
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.stream.assembler import StreamAssembler
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
|
||||
|
||||
class TestStreamAssembler(AsyncioTestCase):
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import os
|
||||
import unittest
|
||||
from unittest import mock
|
||||
import asyncio
|
||||
import contextlib
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.stream.downloader import StreamDownloader
|
||||
from lbrynet.dht.node import Node
|
||||
@@ -21,59 +22,58 @@ class TestStreamDownloader(BlobExchangeTestBase):
|
||||
f.write(self.stream_bytes)
|
||||
descriptor = await StreamDescriptor.create_stream(self.loop, self.server_blob_manager.blob_dir, file_path)
|
||||
self.sd_hash = descriptor.calculate_sd_hash()
|
||||
self.downloader = StreamDownloader(self.loop, self.client_blob_manager, self.sd_hash, 3, 3, self.client_dir)
|
||||
conf = Config(data_dir=self.server_dir, wallet_dir=self.server_dir, download_dir=self.server_dir,
|
||||
reflector_servers=[])
|
||||
self.downloader = StreamDownloader(self.loop, conf, self.client_blob_manager, self.sd_hash)
|
||||
|
||||
async def _test_transfer_stream(self, blob_count: int, mock_peer_search=None):
|
||||
async def _test_transfer_stream(self, blob_count: int, mock_accumulate_peers=None):
|
||||
await self.setup_stream(blob_count)
|
||||
|
||||
mock_node = mock.Mock(spec=Node)
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _mock_peer_search(*_):
|
||||
async def _gen():
|
||||
yield [self.server_from_client]
|
||||
return
|
||||
def _mock_accumulate_peers(q1, q2):
|
||||
async def _task():
|
||||
pass
|
||||
q2.put_nowait([self.server_from_client])
|
||||
return q2, self.loop.create_task(_task())
|
||||
|
||||
yield _gen()
|
||||
|
||||
mock_node.stream_peer_search_junction = mock_peer_search or _mock_peer_search
|
||||
mock_node.accumulate_peers = mock_accumulate_peers or _mock_accumulate_peers
|
||||
|
||||
self.downloader.download(mock_node)
|
||||
await self.downloader.stream_finished_event.wait()
|
||||
await self.downloader.stop()
|
||||
self.downloader.stop()
|
||||
self.assertTrue(os.path.isfile(self.downloader.output_path))
|
||||
with open(self.downloader.output_path, 'rb') as f:
|
||||
self.assertEqual(f.read(), self.stream_bytes)
|
||||
await asyncio.sleep(0.01)
|
||||
self.assertTrue(self.downloader.stream_handle.closed)
|
||||
|
||||
async def test_transfer_stream(self):
|
||||
await self._test_transfer_stream(10)
|
||||
|
||||
# async def test_transfer_hundred_blob_stream(self):
|
||||
# await self._test_transfer_stream(100)
|
||||
@unittest.SkipTest
|
||||
async def test_transfer_hundred_blob_stream(self):
|
||||
await self._test_transfer_stream(100)
|
||||
|
||||
async def test_transfer_stream_bad_first_peer_good_second(self):
|
||||
await self.setup_stream(2)
|
||||
|
||||
mock_node = mock.Mock(spec=Node)
|
||||
q = asyncio.Queue()
|
||||
|
||||
bad_peer = KademliaPeer(self.loop, "127.0.0.1", b'2' * 48, tcp_port=3334)
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def mock_peer_search(*_):
|
||||
async def _gen():
|
||||
await asyncio.sleep(0.05, loop=self.loop)
|
||||
yield [bad_peer]
|
||||
await asyncio.sleep(0.1, loop=self.loop)
|
||||
yield [self.server_from_client]
|
||||
return
|
||||
def _mock_accumulate_peers(q1, q2):
|
||||
async def _task():
|
||||
pass
|
||||
|
||||
yield _gen()
|
||||
q2.put_nowait([bad_peer])
|
||||
self.loop.call_later(1, q2.put_nowait, [self.server_from_client])
|
||||
return q2, self.loop.create_task(_task())
|
||||
|
||||
mock_node.stream_peer_search_junction = mock_peer_search
|
||||
mock_node.accumulate_peers = _mock_accumulate_peers
|
||||
|
||||
self.downloader.download(mock_node)
|
||||
await self.downloader.stream_finished_event.wait()
|
||||
await self.downloader.stop()
|
||||
self.assertTrue(os.path.isfile(self.downloader.output_path))
|
||||
with open(self.downloader.output_path, 'rb') as f:
|
||||
self.assertEqual(f.read(), self.stream_bytes)
|
||||
|
||||
@@ -21,7 +21,7 @@ class TestStreamAssembler(AsyncioTestCase):
|
||||
self.storage = SQLiteStorage(Config(), os.path.join(tmp_dir, "lbrynet.sqlite"))
|
||||
await self.storage.open()
|
||||
self.blob_manager = BlobFileManager(self.loop, tmp_dir, self.storage)
|
||||
self.stream_manager = StreamManager(self.loop, self.blob_manager, None, self.storage, None, 3.0, 3.0)
|
||||
self.stream_manager = StreamManager(self.loop, Config(), self.blob_manager, None, self.storage, None)
|
||||
|
||||
server_tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(server_tmp_dir))
|
||||
|
||||
Reference in new issue
Block a user