Compare commits

...
Author SHA1 Message Date
Jack Robison e5e903a9bf v0.37.5 2019-06-19 09:27:13 -04:00
Jack Robison 9f729c29c3 paginated deterministically shuffled find_value
- fixes https://github.com/lbryio/lbry/issues/2244
- reduce the max DHT datagram size to 1400 bytes
- truncate `contacts` field of find_value response datagrams to the k closest (8)
- truncate peers in find_node response datagrams to the 2k closest (16)
- remove `contacts` field from find_value responses beyond `page` 0 (the first/default)
- deterministically shuffle the peers for a blob in a find_value response
- add optional `page` argument to `find_value` and `p` field to find_value responses containing the number of pages of k peers for the blob
- test one blob being announced by 150 different peers to one peer
- speed up pylint and remove some disabled checks
2019-06-18 23:39:50 -04:00
Jack Robison b468a82b53 v0.37.4 2019-06-11 15:21:01 -04:00
Jack Robison e0e9c4efd0 fix from lex 2019-06-10 18:15:43 -04:00
Jack Robison c16433357b fix attribute error if availability_response field is missing 2019-06-10 16:41:29 -04:00
Jack Robison a8cb873de5 change a few instances of lbry.io to lbry.com. fixes #2135
# Conflicts:
#	lbrynet/schema/url.py
2019-06-10 16:36:50 -04:00
Jack Robison d364431619 --blocking
# Conflicts:
#	lbrynet/extras/daemon/Daemon.py
2019-06-10 12:35:39 -04:00
Jack Robison bec75a957d v0.37.4rc1 2019-06-07 16:34:10 -04:00
Jack Robison 338ea79618 handle chunked responses from reflector server 2019-06-07 16:33:28 -04:00
Jack Robison 61bdbceca3 test chunked reflector server response 2019-06-07 16:32:08 -04:00
Jack Robison 8683e96156 fix connections status raising an error during startup 2019-06-07 16:31:46 -04:00
Jack Robison 4ae59b15e0 v0.37.3 2019-06-05 21:01:23 -04:00
Jack Robison 3771263179 update version docstring, add torba_version 2019-06-05 20:48:50 -04:00
Jack Robison 80ab0ba46d v0.37.3rc6 2019-06-05 20:22:50 -04:00
Jack Robison bd31c4c395 pin aioupnp requirement 2019-06-05 20:22:11 -04:00
Jack Robison 9bcb1d7d95 require torba 0.5.4a0 2019-06-05 20:22:07 -04:00
Jack Robison e9ce41bea6 fix torba version on travis
# Conflicts:
#	.travis.yml
2019-06-05 18:21:44 -04:00
Thomas Zarebczan 734b80b2b2 add MKV to mime types 2019-06-05 15:39:26 -04:00
Jack Robison a90919a9ba mbps instead of mbs, remove time from connection status 2019-06-04 19:35:34 -04:00
Jack Robison 347415dceb add connection status 2019-06-04 19:34:09 -04:00
Jack Robison fe55187696 0.37.2 2019-05-24 18:59:06 -04:00
Jack Robison db8fb15cc5 0.37.2rc4 2019-05-24 18:12:19 -04:00
Jack Robison a165ad910f improve set_build.py, pin torba version
- adds BUILD_COMMIT to lbrynet.build_type
2019-05-24 18:11:50 -04:00
Jack Robison b711069456 fix false negative with file completed field 2019-05-24 17:31:26 -04:00
Jack Robison c977a672b0 start returning range request bytes at the requested offset 2019-05-24 17:31:26 -04:00
Jack Robison ac5794d96d add blob_lru_cache_size config setting, defaults to 32 2019-05-24 17:31:26 -04:00
42 changed files with 568 additions and 172 deletions

No files matched your search

+1 -1
View File
@@ -7,7 +7,7 @@ Before reporting any issues, please make sure that you're using the latest versi
- App: https://github.com/lbryio/lbry-desktop/releases
- Daemon: https://github.com/lbryio/lbry/releases
We are also available on Discord at https://chat.lbry.io
We are also available on Discord at https://chat.lbry.com
-->
+1 -5
View File
@@ -24,7 +24,7 @@ persistent=yes
load-plugins=
# Use multiple processes to speed up Pylint.
jobs=1
jobs=4
# Allow loading of arbitrary C extensions. Extensions are imported into the
# active Python interpreter and may run arbitrary code.
@@ -77,8 +77,6 @@ disable=
dangerous-default-value,
duplicate-code,
fixme,
global-statement,
inherit-non-class,
invalid-name,
len-as-condition,
locally-disabled,
@@ -111,13 +109,11 @@ disable=
unnecessary-lambda,
unused-argument,
unused-variable,
wildcard-import,
wrong-import-order,
wrong-import-position,
deprecated-lambda,
simplifiable-if-statement,
unidiomatic-typecheck,
global-at-module-level,
inconsistent-return-statements,
keyword-arg-before-vararg,
assignment-from-no-return,
+13 -7
View File
@@ -3,6 +3,11 @@ dist: xenial
language: python
python: "3.7"
env:
global:
# must also be updated in wine_build.sh
- TORBA=v0.5.4a0
jobs:
include:
@@ -10,7 +15,7 @@ jobs:
name: "pylint lbrynet"
install:
- pip install astroid pylint
- pip install git+https://github.com/lbryio/torba.git#egg=torba
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
- pip install -e .
script: pylint lbrynet
@@ -18,7 +23,7 @@ jobs:
name: "Unit Tests"
install:
- pip install coverage
- pip install git+https://github.com/lbryio/torba.git#egg=torba
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
- pip install -e .
script:
- HOME=/tmp coverage run -p --source=lbrynet -m unittest discover -vv tests.unit
@@ -29,7 +34,8 @@ jobs:
- name: "Integration Tests"
install:
- pip install tox-travis coverage
- pushd .. && git clone https://github.com/lbryio/torba.git && popd
- pushd .. && git clone --single-branch --branch ${TORBA} https://github.com/lbryio/torba.git && popd
script: tox
after_success:
- coverage combine tests/
@@ -38,7 +44,7 @@ jobs:
- name: "Run Examples"
install:
- pip install coverage
- pip install git+https://github.com/lbryio/torba.git#egg=torba
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
- pip install -e .
script:
- HOME=/tmp coverage run -p --source=lbrynet scripts/generate_json_api.py
@@ -79,7 +85,7 @@ jobs:
env: OS=linux
install:
- pip3 install pyinstaller
- pip3 install git+https://github.com/lbryio/torba.git
- pip3 install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
- python3 scripts/set_build.py
- pip3 install -e .
script:
@@ -121,7 +127,7 @@ jobs:
script:
- set -e
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f scripts/Dockerfile.wallet_server .
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f scripts/Dockerfile.wallet_server --build-arg TORBA_VERSION=${TORBA} .
- docker push lbry/wallet-server:$TRAVIS_TAG
- if: tag IS blank AND branch = master AND NOT type IN (pull_request)
stage: build
@@ -129,7 +135,7 @@ jobs:
script:
- set -e
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
- travis_retry docker build -t lbry/wallet-server:master -f scripts/Dockerfile.wallet_server .
- travis_retry docker build -t lbry/wallet-server:master -f scripts/Dockerfile.wallet_server --build-arg TORBA_VERSION=${TORBA} .
- docker push lbry/wallet-server:master
cache:
+1 -1
View File
@@ -41,7 +41,7 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
## Security
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.io/faq/gpg-key) if you need it.
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.com/faq/gpg-key) if you need it.
## Contact
+1 -1
View File
@@ -1,4 +1,4 @@
__name__ = "lbrynet"
__version__ = "0.37.1"
__version__ = "0.37.5"
version = tuple(__version__.split('.'))
+7
View File
@@ -2,8 +2,10 @@ import os
import typing
import asyncio
import logging
from lbrynet.utils import LRUCache
from lbrynet.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.connection_manager import ConnectionManager
if typing.TYPE_CHECKING:
from lbrynet.conf import Config
@@ -30,6 +32,9 @@ class BlobManager:
else self._node_data_store.completed_blobs
self.blobs: typing.Dict[str, AbstractBlob] = {}
self.config = config
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCache(
self.config.blob_lru_cache_size)
self.connection_manager = ConnectionManager(loop)
def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None):
if self.config.save_blobs:
@@ -81,9 +86,11 @@ class BlobManager:
to_add = await self.storage.sync_missing_blobs(in_blobfiles_dir)
if to_add:
self.completed_blob_hashes.update(to_add)
self.connection_manager.start()
return True
def stop(self):
self.connection_manager.stop()
while self.blobs:
_, blob = self.blobs.popitem()
blob.close()
+34 -8
View File
@@ -8,18 +8,20 @@ from lbrynet.utils import cache_concurrent
if typing.TYPE_CHECKING:
from lbrynet.blob.blob_file import AbstractBlob
from lbrynet.blob.writer import HashBlobWriter
from lbrynet.connection_manager import ConnectionManager
log = logging.getLogger(__name__)
class BlobExchangeClientProtocol(asyncio.Protocol):
def __init__(self, loop: asyncio.BaseEventLoop, peer_timeout: typing.Optional[float] = 10):
def __init__(self, loop: asyncio.BaseEventLoop, peer_timeout: typing.Optional[float] = 10,
connection_manager: typing.Optional['ConnectionManager'] = None):
self.loop = loop
self.peer_port: typing.Optional[int] = None
self.peer_address: typing.Optional[str] = None
self.peer_timeout = peer_timeout
self.transport: typing.Optional[asyncio.Transport] = None
self.peer_timeout = peer_timeout
self.connection_manager = connection_manager
self.writer: typing.Optional['HashBlobWriter'] = None
self.blob: typing.Optional['AbstractBlob'] = None
@@ -31,6 +33,12 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
self.closed = asyncio.Event(loop=self.loop)
def data_received(self, data: bytes):
if self.connection_manager:
if not self.peer_address:
addr_info = self.transport.get_extra_info('peername')
self.peer_address, self.peer_port = addr_info
# assert self.peer_address is not None
self.connection_manager.received_data(f"{self.peer_address}:{self.peer_port}", len(data))
#log.debug("%s:%d -- got %s bytes -- %s bytes on buffer -- %s blob bytes received",
# self.peer_address, self.peer_port, len(data), len(self.buf), self._blob_bytes_received)
if not self.transport or self.transport.is_closing():
@@ -94,10 +102,15 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
"""
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
blob_hash = self.blob.blob_hash
if not self.peer_address:
addr_info = self.transport.get_extra_info('peername')
self.peer_address, self.peer_port = addr_info
try:
msg = request.serialize()
log.debug("send request to %s:%i -> %s", self.peer_address, self.peer_port, msg.decode())
self.transport.write(msg)
if self.connection_manager:
self.connection_manager.sent_data(f"{self.peer_address}:{self.peer_port}", len(msg))
response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop)
availability_response = response.get_availability_response()
price_response = response.get_price_response()
@@ -112,11 +125,16 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
self.peer_port)
log.warning(response.to_dict())
return self._blob_bytes_received, self.close()
elif availability_response.available_blobs and \
elif availability_response and availability_response.available_blobs and \
availability_response.available_blobs != [self.blob.blob_hash]:
log.warning("blob availability response doesn't match our request from %s:%i",
self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
elif not availability_response:
log.warning("response from %s:%i did not include an availability response (we requested %s)",
self.peer_address, self.peer_port, blob_hash)
return self._blob_bytes_received, self.close()
if not price_response or price_response.blob_data_payment_rate != 'RATE_ACCEPTED':
log.warning("data rate rejected by %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
@@ -186,11 +204,16 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
self.writer = None
def connection_made(self, transport: asyncio.Transport):
addr = transport.get_extra_info('peername')
self.peer_address, self.peer_port = addr[0], addr[1]
self.transport = transport
self.peer_address, self.peer_port = self.transport.get_extra_info('peername')
if self.connection_manager:
self.connection_manager.connection_made(f"{self.peer_address}:{self.peer_port}")
log.debug("connection made to %s:%i", self.peer_address, self.peer_port)
def connection_lost(self, reason):
if self.connection_manager:
self.connection_manager.outgoing_connection_lost(f"{self.peer_address}:{self.peer_port}")
log.debug("connection lost to %s:%i (reason: %s, %s)", self.peer_address, self.peer_port, str(reason),
str(type(reason)))
self.close()
@@ -199,16 +222,19 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
@cache_concurrent
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'AbstractBlob', address: str, tcp_port: int,
peer_connect_timeout: float, blob_download_timeout: float,
connected_transport: asyncio.Transport = None, connection_id: int = 0)\
connected_transport: asyncio.Transport = None, connection_id: int = 0,
connection_manager: typing.Optional['ConnectionManager'] = None)\
-> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
"""
Returns [<downloaded blob>, <keep connection>]
"""
protocol = BlobExchangeClientProtocol(loop, blob_download_timeout)
protocol = BlobExchangeClientProtocol(
loop, blob_download_timeout, connection_manager
)
if connected_transport and not connected_transport.is_closing():
connected_transport.set_protocol(protocol)
protocol.connection_made(connected_transport)
protocol.transport = connected_transport
log.debug("reusing connection for %s:%d", address, tcp_port)
else:
connected_transport = None
+3 -1
View File
@@ -41,7 +41,9 @@ class BlobDownloader:
start = self.loop.time()
bytes_received, transport = await request_blob(
self.loop, blob, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
self.config.blob_download_timeout, connected_transport=transport, connection_id=connection_id
self.config.blob_download_timeout, connected_transport=transport, connection_id=connection_id,
connection_manager=self.blob_manager.connection_manager
)
if not transport and peer not in self.ignored:
self.ignored[peer] = self.loop.time()
+11 -2
View File
@@ -22,15 +22,23 @@ class BlobServerProtocol(asyncio.Protocol):
self.buf = b''
self.transport = None
self.lbrycrd_address = lbrycrd_address
self.peer_address_and_port: typing.Optional[str] = None
def connection_made(self, transport):
self.transport = transport
self.peer_address_and_port = "%s:%i" % self.transport.get_extra_info('peername')
self.blob_manager.connection_manager.connection_received(self.peer_address_and_port)
def connection_lost(self, exc: typing.Optional[Exception]) -> None:
self.blob_manager.connection_manager.incoming_connection_lost(self.peer_address_and_port)
def send_response(self, responses: typing.List[blob_response_types]):
to_send = []
while responses:
to_send.append(responses.pop())
self.transport.write(BlobResponse(to_send).serialize())
serialized = BlobResponse(to_send).serialize()
self.transport.write(serialized)
self.blob_manager.connection_manager.sent_data(self.peer_address_and_port, len(serialized))
async def handle_request(self, request: BlobRequest):
addr = self.transport.get_extra_info('peername')
@@ -72,6 +80,7 @@ class BlobServerProtocol(asyncio.Protocol):
def data_received(self, data):
request = None
if data:
self.blob_manager.connection_manager.received_data(self.peer_address_and_port, len(data))
message, separator, remainder = data.rpartition(b'}')
if not separator:
self.buf += data
@@ -97,7 +106,7 @@ class BlobServer:
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str):
self.loop = loop
self.blob_manager = blob_manager
self.server_task: asyncio.Task = None
self.server_task: typing.Optional[asyncio.Task] = None
self.started_listening = asyncio.Event(loop=self.loop)
self.lbrycrd_address = lbrycrd_address
self.server_protocol_class = BlobServerProtocol
+1
View File
@@ -1,2 +1,3 @@
# don't touch this. CI server changes this during build/deployment
BUILD = "dev"
BUILD_COMMIT = "source installation"
+5 -2
View File
@@ -485,7 +485,10 @@ class Config(CLIConfig):
# blob announcement and download
save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True)
blob_lru_cache_size = Integer(
"LRU cache size for decrypted downloaded blobs used to minimize re-downloading the same blobs when "
"replying to a range request. Set to 0 to disable.", 32
)
announce_head_and_sd_only = Toggle(
"Announce only the descriptor and first (rather than all) data blob for a stream to the DHT", True,
previous_names=['announce_head_blobs_only']
@@ -517,7 +520,7 @@ class Config(CLIConfig):
# servers
reflector_servers = Servers("Reflector re-hosting servers", [
('reflector.lbry.io', 5566)
('reflector.lbry.com', 5566)
])
lbryum_servers = Servers("SPV wallet servers", [
('lbryumx1.lbry.com', 50001), # US EAST
+89
View File
@@ -0,0 +1,89 @@
import asyncio
import typing
import collections
import logging
log = logging.getLogger(__name__)
CONNECTED_EVENT = "connected"
DISCONNECTED_EVENT = "disconnected"
TRANSFERRED_EVENT = "transferred"
class ConnectionManager:
def __init__(self, loop: asyncio.AbstractEventLoop):
self.loop = loop
self.incoming_connected: typing.Set[str] = set()
self.incoming: typing.DefaultDict[str, int] = collections.defaultdict(int)
self.outgoing_connected: typing.Set[str] = set()
self.outgoing: typing.DefaultDict[str, int] = collections.defaultdict(int)
self._status = {}
self._task: typing.Optional[asyncio.Task] = None
@property
def status(self):
return self._status
def sent_data(self, host_and_port: str, size: int):
self.outgoing[host_and_port] += size
def received_data(self, host_and_port: str, size: int):
self.incoming[host_and_port] += size
def connection_made(self, host_and_port: str):
self.outgoing_connected.add(host_and_port)
def connection_received(self, host_and_port: str):
# self.incoming_connected.add(host_and_port)
pass
def outgoing_connection_lost(self, host_and_port: str):
if host_and_port in self.outgoing_connected:
self.outgoing_connected.remove(host_and_port)
def incoming_connection_lost(self, host_and_port: str):
if host_and_port in self.incoming_connected:
self.incoming_connected.remove(host_and_port)
async def _update(self):
self._status = {
'incoming_bps': {},
'outgoing_bps': {},
'total_incoming_mbps': 0.0,
'total_outgoing_mbps': 0.0,
}
while True:
last = self.loop.time()
await asyncio.sleep(1, loop=self.loop)
self._status['incoming_bps'].clear()
self._status['outgoing_bps'].clear()
while self.outgoing:
k, v = self.outgoing.popitem()
self._status['outgoing_bps'][k] = v
while self.incoming:
k, v = self.incoming.popitem()
self._status['incoming_bps'][k] = v
now = self.loop.time()
self._status['total_outgoing_mbps'] = int(sum(list(self._status['outgoing_bps'].values()))
/ (now - last)) / 1000000.0
self._status['total_incoming_mbps'] = int(sum(list(self._status['incoming_bps'].values()))
/ (now - last)) / 1000000.0
self._status['time'] = now
def stop(self):
if self._task:
self._task.cancel()
self._task = None
self.outgoing.clear()
self.outgoing_connected.clear()
self.incoming.clear()
self.incoming_connected.clear()
self._status.clear()
def start(self):
self.stop()
self._task = self.loop.create_task(self._update())
+1 -2
View File
@@ -18,11 +18,10 @@ data_expiration = 86400 # 24 hours
token_secret_refresh_interval = 300 # 5 minutes
maybe_ping_delay = 300 # 5 minutes
check_refresh_interval = refresh_interval / 5
max_datagram_size = 8192 # 8 KB
rpc_id_length = 20
protocol_version = 1
bottom_out_limit = 3
msg_size_limit = max_datagram_size - 26
msg_size_limit = 1400
def digest(data: bytes) -> bytes:
+38 -23
View File
@@ -5,12 +5,13 @@ import hashlib
import asyncio
import typing
import binascii
import random
from asyncio.protocols import DatagramProtocol
from asyncio.transports import DatagramTransport
from lbrynet.dht import constants
from lbrynet.dht.serialization.datagram import decode_datagram, ErrorDatagram, ResponseDatagram, RequestDatagram
from lbrynet.dht.serialization.datagram import RESPONSE_TYPE, ERROR_TYPE
from lbrynet.dht.serialization.datagram import RESPONSE_TYPE, ERROR_TYPE, PAGE_KEY
from lbrynet.dht.error import RemoteException, TransportNotConnected
from lbrynet.dht.protocol.routing_table import TreeRoutingTable
from lbrynet.dht.protocol.data_store import DictDataStore
@@ -67,19 +68,23 @@ class KademliaRPC:
contacts = self.protocol.routing_table.find_close_peers(key, sender_node_id=rpc_contact.node_id)
contact_triples = []
for contact in contacts:
for contact in contacts[:constants.k * 2]:
contact_triples.append((contact.node_id, contact.address, contact.udp_port))
return contact_triples
def find_value(self, rpc_contact: 'KademliaPeer', key: bytes):
def find_value(self, rpc_contact: 'KademliaPeer', key: bytes, page: int = 0):
page = page if page > 0 else 0
if len(key) != constants.hash_length:
raise ValueError("invalid blob_exchange hash length: %i" % len(key))
response = {
b'token': self.make_token(rpc_contact.compact_ip()),
b'contacts': self.find_node(rpc_contact, key)
}
if not page:
response[b'contacts'] = self.find_node(rpc_contact, key)[:constants.k]
if self.protocol.protocol_version:
response[b'protocolVersion'] = self.protocol.protocol_version
@@ -92,8 +97,14 @@ class KademliaRPC:
# if we don't have k storing peers to return and we have this hash locally, include our contact information
if len(peers) < constants.k and binascii.hexlify(key).decode() in self.protocol.data_store.completed_blobs:
peers.append(self.compact_address())
if peers:
response[key] = peers
if not peers:
response[PAGE_KEY] = 0
else:
response[PAGE_KEY] = (len(peers) // (constants.k + 1)) + 1 # how many pages of peers we have for the blob
if len(peers) > constants.k:
random.Random(self.protocol.node_id).shuffle(peers)
if page * constants.k < len(peers):
response[key] = peers[page * constants.k:page * constants.k + constants.k]
return response
def refresh_token(self): # TODO: this needs to be called periodically
@@ -166,7 +177,7 @@ class RemoteKademliaRPC:
)
return [(node_id, address.decode(), udp_port) for node_id, address, udp_port in response.response]
async def find_value(self, key: bytes) -> typing.Union[typing.Dict]:
async def find_value(self, key: bytes, page: int = 0) -> typing.Union[typing.Dict]:
"""
:return: {
b'token': <token bytes>,
@@ -177,7 +188,7 @@ class RemoteKademliaRPC:
if len(key) != constants.hash_bits // 8:
raise ValueError(f"invalid length of find value key: {len(key)}")
response = await self.protocol.send_request(
self.peer, RequestDatagram.make_find_value(self.protocol.node_id, key)
self.peer, RequestDatagram.make_find_value(self.protocol.node_id, key, page=page)
)
self.peer_tracker.update_token(self.peer.node_id, response.response[b'token'])
return response.response
@@ -406,24 +417,25 @@ class KademliaProtocol(DatagramProtocol):
raise AttributeError('Invalid method: %s' % message.method.decode())
if message.args and isinstance(message.args[-1], dict) and b'protocolVersion' in message.args[-1]:
# args don't need reformatting
a, kw = tuple(message.args[:-1]), message.args[-1]
args, kw = tuple(message.args[:-1]), message.args[-1]
else:
a, kw = self._migrate_incoming_rpc_args(sender_contact, message.method, *message.args)
args, kw = self._migrate_incoming_rpc_args(sender_contact, message.method, *message.args)
log.debug("%s:%i RECV CALL %s %s:%i", self.external_ip, self.udp_port, message.method.decode(),
sender_contact.address, sender_contact.udp_port)
if method == b'ping':
result = self.node_rpc.ping()
elif method == b'store':
blob_hash, token, port, original_publisher_id, age = a
blob_hash, token, port, original_publisher_id, age = args[:5]
result = self.node_rpc.store(sender_contact, blob_hash, token, port)
elif method == b'findNode':
key, = a
result = self.node_rpc.find_node(sender_contact, key)
else:
assert method == b'findValue'
key, = a
result = self.node_rpc.find_value(sender_contact, key)
key = args[0]
page = kw.get(PAGE_KEY, 0)
if method == b'findNode':
result = self.node_rpc.find_node(sender_contact, key)
else:
assert method == b'findValue'
result = self.node_rpc.find_value(sender_contact, key, page)
self.send_response(
sender_contact, ResponseDatagram(RESPONSE_TYPE, message.rpc_id, self.node_id, result),
@@ -569,15 +581,18 @@ class KademliaProtocol(DatagramProtocol):
def send_error(self, peer: 'KademliaPeer', error: ErrorDatagram):
self._send(peer, error)
def _send(self, peer: 'KademliaPeer', message: typing.Union[RequestDatagram, ResponseDatagram,
ErrorDatagram]):
def _send(self, peer: 'KademliaPeer', message: typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]):
if not self.transport or self.transport.is_closing():
raise TransportNotConnected()
data = message.bencode()
if len(data) > constants.msg_size_limit:
log.exception("unexpected: %i vs %i", len(data), constants.msg_size_limit)
raise ValueError()
log.warning("cannot send datagram larger than %i bytes (packet is %i bytes)",
constants.msg_size_limit, len(data))
log.debug("Packet is too large to send: %s", binascii.hexlify(data[:3500]).decode())
raise ValueError(
f"cannot send datagram larger than {constants.msg_size_limit} bytes (packet is {len(data)} bytes)"
)
if isinstance(message, (RequestDatagram, ResponseDatagram)):
assert message.node_id == self.node_id, message
if isinstance(message, RequestDatagram):
@@ -642,12 +657,12 @@ class KademliaProtocol(DatagramProtocol):
log.debug("Timeout while storing blob_hash %s at %s", binascii.hexlify(hash_value).decode()[:8], peer)
except ValueError as err:
log.error("Unexpected response: %s" % err)
except Exception as err:
except RemoteException as err:
if 'Invalid token' in str(err):
self.peer_manager.clear_token(peer.node_id)
try:
return await __store()
except:
except (ValueError, asyncio.TimeoutError, RemoteException):
return peer.node_id, False
else:
log.exception("Unexpected error while storing blob_hash")
+1 -1
View File
@@ -63,7 +63,7 @@ def bencode(data: typing.Dict) -> bytes:
def bdecode(data: bytes, allow_non_dict_return: typing.Optional[bool] = False) -> typing.Dict:
assert type(data) == bytes, DecodeError(f"invalid data type: {str(type(data))}")
assert isinstance(data, bytes), DecodeError(f"invalid data type: {str(type(data))}")
if len(data) == 0:
raise DecodeError('Cannot decode empty string')
+7 -2
View File
@@ -7,6 +7,9 @@ REQUEST_TYPE = 0
RESPONSE_TYPE = 1
ERROR_TYPE = 2
# bencode representation of argument keys
PAGE_KEY = b'p'
class KademliaDatagramBase:
"""
@@ -91,11 +94,13 @@ class RequestDatagram(KademliaDatagramBase):
@classmethod
def make_find_value(cls, from_node_id: bytes, key: bytes,
rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
rpc_id: typing.Optional[bytes] = None, page: int = 0) -> 'RequestDatagram':
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
if len(key) != constants.hash_bits // 8:
raise ValueError(f"invalid key length: {len(key)}")
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'findValue', [key])
if page < 0:
raise ValueError(f"cannot request a negative page ({page})")
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'findValue', [key, {PAGE_KEY: page}])
class ResponseDatagram(KademliaDatagramBase):
+10 -7
View File
@@ -278,7 +278,7 @@ class BlobComponent(Component):
def __init__(self, component_manager):
super().__init__(component_manager)
self.blob_manager: BlobManager = None
self.blob_manager: typing.Optional[BlobManager] = None
@property
def component(self) -> typing.Optional[BlobManager]:
@@ -294,7 +294,7 @@ class BlobComponent(Component):
blob_dir = os.path.join(self.conf.data_dir, 'blobfiles')
if not os.path.isdir(blob_dir):
os.mkdir(blob_dir)
self.blob_manager = BlobManager(asyncio.get_event_loop(), blob_dir, storage, self.conf, data_store)
self.blob_manager = BlobManager(self.component_manager.loop, blob_dir, storage, self.conf, data_store)
return await self.blob_manager.setup()
async def stop(self):
@@ -304,7 +304,10 @@ class BlobComponent(Component):
count = 0
if self.blob_manager:
count = len(self.blob_manager.completed_blob_hashes)
return {'finished_blobs': count}
return {
'finished_blobs': count,
'connections': {} if not self.blob_manager else self.blob_manager.connection_manager.status
}
class DHTComponent(Component):
@@ -405,7 +408,7 @@ class StreamManagerComponent(Component):
def __init__(self, component_manager):
super().__init__(component_manager)
self.stream_manager: StreamManager = None
self.stream_manager: typing.Optional[StreamManager] = None
@property
def component(self) -> typing.Optional[StreamManager]:
@@ -415,7 +418,7 @@ class StreamManagerComponent(Component):
if not self.stream_manager:
return
return {
'managed_files': len(self.stream_manager.streams)
'managed_files': len(self.stream_manager.streams),
}
async def start(self):
@@ -444,7 +447,7 @@ class PeerProtocolServerComponent(Component):
def __init__(self, component_manager):
super().__init__(component_manager)
self.blob_server: BlobServer = None
self.blob_server: typing.Optional[BlobServer] = None
@property
def component(self) -> typing.Optional[BlobServer]:
@@ -510,7 +513,7 @@ class UPnPComponent(Component):
pass
if external_ip == "0.0.0.0" or not external_ip:
log.warning("unable to get external ip from UPnP, checking lbry.io fallback")
log.warning("unable to get external ip from UPnP, checking lbry.com fallback")
external_ip = await utils.get_external_ip()
if self.external_ip and self.external_ip != external_ip:
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
+73 -35
View File
@@ -485,6 +485,7 @@ class Daemon(metaclass=JSONRPCServerType):
async def handle_stream_get_request(self, request: web.Request):
if not self.conf.streaming_get:
log.warning("streaming_get is disabled, rejecting request")
raise web.HTTPForbidden()
name_and_claim_id = request.path.split("/get/")[1]
if "/" not in name_and_claim_id:
@@ -500,6 +501,20 @@ class Daemon(metaclass=JSONRPCServerType):
raise web.HTTPFound(f"/stream/{stream.sd_hash}")
async def handle_stream_range_request(self, request: web.Request):
try:
return await self._handle_stream_range_request(request)
except web.HTTPException as err:
log.warning("http code during /stream range request: %s", err)
raise err
except asyncio.CancelledError:
log.debug("/stream range request cancelled")
except Exception:
log.exception("error handling /stream range request")
raise
finally:
log.debug("finished handling /stream range request")
async def _handle_stream_range_request(self, request: web.Request):
sd_hash = request.path.split("/stream/")[1]
if not self.stream_manager.started.is_set():
await self.stream_manager.started.wait()
@@ -725,6 +740,16 @@ class Daemon(metaclass=JSONRPCServerType):
},
'blob_manager': {
'finished_blobs': (int) number of finished blobs in the blob manager,
'connections': {
'incoming_bps': {
<source ip and tcp port>: (int) bytes per second received,
},
'outgoing_bps': {
<destination ip and tcp port>: (int) bytes per second sent,
},
'total_outgoing_mbps': (float) megabytes per second sent,
'total_incoming_mbps': (float) megabytes per second received
}
},
'hash_announcer': {
'announce_queue_size': (int) number of blobs currently queued to be announced
@@ -776,16 +801,14 @@ class Daemon(metaclass=JSONRPCServerType):
Returns:
(dict) Dictionary of lbry version information
{
'build': (str) build type (e.g. "dev", "rc", "release"),
'ip': (str) remote ip, if available,
'lbrynet_version': (str) lbrynet_version,
'lbryum_version': (str) lbryum_version,
'lbryschema_version': (str) lbryschema_version,
'os_release': (str) os release string
'os_system': (str) os name
'platform': (str) platform string
'processor': (str) processor type,
'python_version': (str) python version,
'platform': (str) platform string,
'os_release': (str) os release string,
'os_system': (str) os name,
'lbrynet_version': (str) lbrynet version,
'torba_version': (str) torba version,
'build': (str) "dev" | "qa" | "rc" | "release",
}
"""
platform_info = system_info.get_platform()
@@ -1734,7 +1757,7 @@ class Daemon(metaclass=JSONRPCServerType):
@requires(WALLET_COMPONENT, conditions=[WALLET_IS_UNLOCKED])
async def jsonrpc_channel_create(
self, name, bid, allow_duplicate_name=False, account_id=None, claim_address=None,
preview=False, **kwargs):
preview=False, blocking=False, **kwargs):
"""
Create a new channel by generating a channel private key and establishing an '@' prefixed claim.
@@ -1745,7 +1768,8 @@ class Daemon(metaclass=JSONRPCServerType):
[--website_url=<website_url>] [--featured=<featured>...]
[--tags=<tags>...] [--languages=<languages>...] [--locations=<locations>...]
[--thumbnail_url=<thumbnail_url>] [--cover_url=<cover_url>]
[--account_id=<account_id>] [--claim_address=<claim_address>] [--preview]
[--account_id=<account_id>] [--claim_address=<claim_address>]
[--preview] [--blocking]
Options:
--name=<name> : (str) name of the channel prefixed with '@'
@@ -1802,6 +1826,7 @@ class Daemon(metaclass=JSONRPCServerType):
--claim_address=<claim_address>: (str) address where the channel is sent to, if not specified
it will be determined automatically from the account
--preview : (bool) do not broadcast the transaction
--blocking : (bool) wait until transaction is in mempool
Returns: {Transaction}
"""
@@ -1828,7 +1853,7 @@ class Daemon(metaclass=JSONRPCServerType):
if not preview:
await tx.sign([account])
await account.ledger.broadcast(tx)
await self.broadcast_or_release(account, tx, blocking)
account.add_channel_private_key(txo.ref, txo.private_key)
self.default_wallet.save()
await self.storage.save_claims([self._old_get_temp_claim_info(
@@ -1843,7 +1868,7 @@ class Daemon(metaclass=JSONRPCServerType):
@requires(WALLET_COMPONENT, conditions=[WALLET_IS_UNLOCKED])
async def jsonrpc_channel_update(
self, claim_id, bid=None, account_id=None, claim_address=None,
new_signing_key=False, preview=False, replace=False, **kwargs):
new_signing_key=False, preview=False, blocking=False, replace=False, **kwargs):
"""
Update an existing channel claim.
@@ -1857,7 +1882,7 @@ class Daemon(metaclass=JSONRPCServerType):
[--locations=<locations>...] [--clear_locations]
[--thumbnail_url=<thumbnail_url>] [--cover_url=<cover_url>]
[--account_id=<account_id>] [--claim_address=<claim_address>] [--new_signing_key]
[--preview] [--replace]
[--preview] [--blocking] [--replace]
Options:
--claim_id=<claim_id> : (str) claim_id of the channel to update
@@ -1916,6 +1941,7 @@ class Daemon(metaclass=JSONRPCServerType):
--claim_address=<claim_address>: (str) address where the channel is sent
--new_signing_key : (bool) generate a new signing key, will invalidate all previous publishes
--preview : (bool) do not broadcast the transaction
--blocking : (bool) wait until transaction is in mempool
--replace : (bool) instead of modifying specific values on
the channel, this will clear all existing values
and only save passed in values, useful for form
@@ -1966,7 +1992,7 @@ class Daemon(metaclass=JSONRPCServerType):
if not preview:
await tx.sign([account])
await account.ledger.broadcast(tx)
await self.broadcast_or_release(account, tx, blocking)
account.add_channel_private_key(new_txo.ref, new_txo.private_key)
self.default_wallet.save()
await self.storage.save_claims([self._old_get_temp_claim_info(
@@ -2018,10 +2044,8 @@ class Daemon(metaclass=JSONRPCServerType):
)
if not preview:
await account.ledger.broadcast(tx)
await self.broadcast_or_release(account, tx, blocking)
await self.analytics_manager.send_claim_action('abandon')
if blocking:
await account.ledger.wait(tx)
else:
await account.ledger.release_tx(tx)
@@ -2103,7 +2127,8 @@ class Daemon(metaclass=JSONRPCServerType):
[--release_time=<release_time>] [--width=<width>] [--height=<height>] [--duration=<duration>]
[--channel_id=<channel_id> | --channel_name=<channel_name>]
[--channel_account_id=<channel_account_id>...]
[--account_id=<account_id>] [--claim_address=<claim_address>] [--preview]
[--account_id=<account_id>] [--claim_address=<claim_address>]
[--preview] [--blocking]
Options:
--name=<name> : (str) name of the content (can only consist of a-z A-Z 0-9 and -(dash))
@@ -2174,6 +2199,7 @@ class Daemon(metaclass=JSONRPCServerType):
--claim_address=<claim_address>: (str) address where the claim is sent to, if not specified
it will be determined automatically from the account
--preview : (bool) do not broadcast the transaction
--blocking : (bool) wait until transaction is in mempool
Returns: {Transaction}
"""
@@ -2200,7 +2226,8 @@ class Daemon(metaclass=JSONRPCServerType):
async def jsonrpc_stream_create(
self, name, bid, file_path, allow_duplicate_name=False,
channel_id=None, channel_name=None, channel_account_id=None,
account_id=None, claim_address=None, preview=False, **kwargs):
account_id=None, claim_address=None, preview=False, blocking=False,
**kwargs):
"""
Make a new stream claim and announce the associated file to lbrynet.
@@ -2214,7 +2241,8 @@ class Daemon(metaclass=JSONRPCServerType):
[--release_time=<release_time>] [--width=<width>] [--height=<height>] [--duration=<duration>]
[--channel_id=<channel_id> | --channel_name=<channel_name>]
[--channel_account_id=<channel_account_id>...]
[--account_id=<account_id>] [--claim_address=<claim_address>] [--preview]
[--account_id=<account_id>] [--claim_address=<claim_address>]
[--preview] [--blocking]
Options:
--name=<name> : (str) name of the content (can only consist of a-z A-Z 0-9 and -(dash))
@@ -2287,6 +2315,7 @@ class Daemon(metaclass=JSONRPCServerType):
--claim_address=<claim_address>: (str) address where the claim is sent to, if not specified
it will be determined automatically from the account
--preview : (bool) do not broadcast the transaction
--blocking : (bool) wait until transaction is in mempool
Returns: {Transaction}
"""
@@ -2323,7 +2352,7 @@ class Daemon(metaclass=JSONRPCServerType):
await tx.sign([account])
if not preview:
await account.ledger.broadcast(tx)
await self.broadcast_or_release(account, tx, blocking)
await self.storage.save_claims([self._old_get_temp_claim_info(
tx, new_txo, claim_address, claim, name, dewies_to_lbc(amount)
)])
@@ -2340,7 +2369,7 @@ class Daemon(metaclass=JSONRPCServerType):
self, claim_id, bid=None, file_path=None,
channel_id=None, channel_name=None, channel_account_id=None, clear_channel=False,
account_id=None, claim_address=None,
preview=False, replace=False, **kwargs):
preview=False, blocking=False, replace=False, **kwargs):
"""
Update an existing stream claim and if a new file is provided announce it to lbrynet.
@@ -2358,7 +2387,7 @@ class Daemon(metaclass=JSONRPCServerType):
[--channel_id=<channel_id> | --channel_name=<channel_name> | --clear_channel]
[--channel_account_id=<channel_account_id>...]
[--account_id=<account_id>] [--claim_address=<claim_address>]
[--preview] [--replace]
[--preview] [--blocking] [--replace]
Options:
--claim_id=<claim_id> : (str) id of the stream claim to update
@@ -2437,6 +2466,7 @@ class Daemon(metaclass=JSONRPCServerType):
--claim_address=<claim_address>: (str) address where the claim is sent to, if not specified
it will be determined automatically from the account
--preview : (bool) do not broadcast the transaction
--blocking : (bool) wait until transaction is in mempool
--replace : (bool) instead of modifying specific values on
the stream, this will clear all existing values
and only save passed in values, useful for form
@@ -2514,7 +2544,7 @@ class Daemon(metaclass=JSONRPCServerType):
await tx.sign([account])
if not preview:
await account.ledger.broadcast(tx)
await self.broadcast_or_release(account, tx, blocking)
await self.storage.save_claims([self._old_get_temp_claim_info(
tx, new_txo, claim_address, new_txo.claim, new_txo.claim_name, dewies_to_lbc(amount)
)])
@@ -2529,7 +2559,7 @@ class Daemon(metaclass=JSONRPCServerType):
@requires(WALLET_COMPONENT, conditions=[WALLET_IS_UNLOCKED])
async def jsonrpc_stream_abandon(
self, claim_id=None, txid=None, nout=None, account_id=None,
preview=False, blocking=True):
preview=False, blocking=False):
"""
Abandon one of my stream claims.
@@ -2566,10 +2596,8 @@ class Daemon(metaclass=JSONRPCServerType):
)
if not preview:
await account.ledger.broadcast(tx)
await self.broadcast_or_release(account, tx, blocking)
await self.analytics_manager.send_claim_action('abandon')
if blocking:
await account.ledger.wait(tx)
else:
await account.ledger.release_tx(tx)
@@ -2622,13 +2650,15 @@ class Daemon(metaclass=JSONRPCServerType):
"""
@requires(WALLET_COMPONENT, conditions=[WALLET_IS_UNLOCKED])
async def jsonrpc_support_create(self, claim_id, amount, tip=False, account_id=None, preview=False):
async def jsonrpc_support_create(
self, claim_id, amount, tip=False, account_id=None,
preview=False, blocking=False):
"""
Create a support or a tip for name claim.
Usage:
support_create (<claim_id> | --claim_id=<claim_id>) (<amount> | --amount=<amount>)
[--tip] [--account_id=<account_id>] [--preview]
[--tip] [--account_id=<account_id>] [--preview] [--blocking]
Options:
--claim_id=<claim_id> : (str) claim_id of the claim to support
@@ -2636,6 +2666,7 @@ class Daemon(metaclass=JSONRPCServerType):
--tip : (bool) send support to claim owner, default: false.
--account_id=<account_id> : (str) id of the account to use
--preview : (bool) do not broadcast the transaction
--blocking : (bool) wait until transaction is in mempool
Returns: {Transaction}
"""
@@ -2653,7 +2684,7 @@ class Daemon(metaclass=JSONRPCServerType):
if not preview:
await tx.sign([account])
await account.ledger.broadcast(tx)
await self.broadcast_or_release(account, tx, blocking)
await self.storage.save_supports({claim_id: [{
'txid': tx.id,
'nout': tx.position,
@@ -2693,7 +2724,7 @@ class Daemon(metaclass=JSONRPCServerType):
@requires(WALLET_COMPONENT, conditions=[WALLET_IS_UNLOCKED])
async def jsonrpc_support_abandon(
self, claim_id=None, txid=None, nout=None, keep=None,
account_id=None, preview=False, blocking=True):
account_id=None, preview=False, blocking=False):
"""
Abandon supports, including tips, of a specific claim, optionally
keeping some amount as supports.
@@ -2743,10 +2774,8 @@ class Daemon(metaclass=JSONRPCServerType):
)
if not preview:
await account.ledger.broadcast(tx)
await self.broadcast_or_release(account, tx, blocking)
await self.analytics_manager.send_claim_action('abandon')
if blocking:
await account.ledger.wait(tx)
else:
await account.ledger.release_tx(tx)
@@ -3387,6 +3416,15 @@ class Daemon(metaclass=JSONRPCServerType):
f"The given claim_id ({claim_id}) may be invalid")
return await jsonrpc_post(url, 'get_comment_data', comm_index=comment_id, better_keys=True)
async def broadcast_or_release(self, account, tx, blocking=False):
try:
await account.ledger.broadcast(tx)
if blocking:
await account.ledger.wait(tx)
except:
await account.ledger.release_tx(tx)
raise
def valid_address_or_error(self, address):
try:
assert self.ledger.is_valid_address(address)
@@ -123,8 +123,8 @@ class LBRYioFeed(MarketFeed):
def __init__(self):
super().__init__(
"BTCLBC",
"lbry.io",
"https://api.lbry.io/lbc/exchange_rate",
"lbry.com",
"https://api.lbry.com/lbc/exchange_rate",
{},
0.0,
)
@@ -140,8 +140,8 @@ class LBRYioBTCFeed(MarketFeed):
def __init__(self):
super().__init__(
"USDBTC",
"lbry.io",
"https://api.lbry.io/lbc/exchange_rate",
"lbry.com",
"https://api.lbry.com/lbc/exchange_rate",
{},
0.0,
)
+2
View File
@@ -3,6 +3,7 @@ import os
import logging.handlers
from lbrynet import build_type, __version__ as lbrynet_version
from torba import __version__ as torba_version
log = logging.getLogger(__name__)
@@ -18,6 +19,7 @@ def get_platform() -> dict:
"os_release": platform.release(),
"os_system": os_system,
"lbrynet_version": lbrynet_version,
"torba_version": torba_version,
"build": build_type.BUILD, # CI server sets this during build step
}
if p["os_system"] == "Linux":
+3 -2
View File
@@ -140,8 +140,10 @@ types_map = {
'.cbr': ('application/vnd.comicbook-rar', 'document'),
'.cbz': ('application/vnd.comicbook+zip', 'document'),
'.lbry': ('application/x-ext-lbry', 'document'),
'.m4v': ('video/m4v', 'video'),
'.mid': ('audio/midi', 'audio'),
'.midi': ('audio/midi', 'audio'),
'.mkv': ('video/x-matroska', 'video'),
'.mobi': ('application/x-mobipocket-ebook', 'document'),
'.pct': ('image/pict', 'image'),
'.pic': ('image/pict', 'image'),
@@ -149,8 +151,7 @@ types_map = {
'.prc': ('application/x-mobipocket-ebook', 'document'),
'.rtf': ('application/rtf', 'document'),
'.xul': ('text/xul', 'document'),
'.m4v': ('video/m4v', 'video'),
# microsoft is special and has its own 'standard'
# https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
'.wmv': ('video/x-ms-wmv', 'video')
+11 -1
View File
@@ -3,7 +3,7 @@ import typing
import logging
import binascii
from lbrynet.error import DownloadSDTimeout
from lbrynet.utils import resolve_host
from lbrynet.utils import resolve_host, lru_cache_concurrent
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.blob_exchange.downloader import BlobDownloader
from lbrynet.dht.peer import KademliaPeer
@@ -36,6 +36,16 @@ class StreamDownloader:
self.time_to_descriptor: typing.Optional[float] = None
self.time_to_first_bytes: typing.Optional[float] = None
async def cached_read_blob(blob_info: 'BlobInfo') -> bytes:
return await self.read_blob(blob_info, 2)
if self.blob_manager.decrypted_blob_lru_cache:
cached_read_blob = lru_cache_concurrent(override_lru_cache=self.blob_manager.decrypted_blob_lru_cache)(
cached_read_blob
)
self.cached_read_blob = cached_read_blob
async def add_fixed_peers(self):
def _delayed_add_fixed_peers():
self.added_fixed_peers = True
+31 -10
View File
@@ -3,7 +3,7 @@ import asyncio
import typing
import logging
import binascii
from aiohttp.web import Request, StreamResponse
from aiohttp.web import Request, StreamResponse, HTTPRequestRangeNotSatisfiable
from lbrynet.utils import generate_id
from lbrynet.error import DownloadSDTimeout
from lbrynet.schema.mime_types import guess_media_type
@@ -42,6 +42,9 @@ class ManagedStream:
STATUS_STOPPED = "stopped"
STATUS_FINISHED = "finished"
SAVING_ID = 1
STREAMING_ID = 2
__slots__ = [
'loop',
'config',
@@ -214,7 +217,8 @@ class ManagedStream:
written_bytes = None
return {
'streaming_url': f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash}",
'completed': (self.output_file_exists and self.status in ('stopped', 'finished')) or all(
'completed': (self.output_file_exists and (self.status in ('stopped', 'finished'))
or not self.saving.is_set()) or all(
self.blob_manager.is_blob_verified(b.blob_hash) for b in self.descriptor.blobs[:-1]),
'file_name': file_name,
'download_directory': download_directory,
@@ -304,14 +308,19 @@ class ManagedStream:
raise IndexError(start_blob_num)
for i, blob_info in enumerate(self.descriptor.blobs[start_blob_num:-1]):
assert i + start_blob_num == blob_info.blob_num
decrypted = await self.downloader.read_blob(blob_info, connection_id)
if connection_id == self.STREAMING_ID:
decrypted = await self.downloader.cached_read_blob(blob_info)
else:
decrypted = await self.downloader.read_blob(blob_info, connection_id)
yield (blob_info, decrypted)
async def stream_file(self, request: Request, node: typing.Optional['Node'] = None) -> StreamResponse:
log.info("stream file to browser for lbry://%s#%s (sd hash %s...)", self.claim_name, self.claim_id,
self.sd_hash[:6])
headers, size, skip_blobs, first_blob_start_offset = self._prepare_range_response_headers(
request.headers.get('range', 'bytes=0-')
)
await self.start(node)
headers, size, skip_blobs = self._prepare_range_response_headers(request.headers.get('range', 'bytes=0-'))
response = StreamResponse(
status=206,
headers=headers
@@ -321,11 +330,16 @@ class ManagedStream:
self.streaming.set()
try:
wrote = 0
async for blob_info, decrypted in self._aiter_read_stream(skip_blobs, connection_id=2):
async for blob_info, decrypted in self._aiter_read_stream(skip_blobs, connection_id=self.STREAMING_ID):
if not wrote:
decrypted = decrypted[first_blob_start_offset:]
if (blob_info.blob_num == len(self.descriptor.blobs) - 2) or (len(decrypted) + wrote >= size):
decrypted += (b'\x00' * (size - len(decrypted) - wrote - (skip_blobs * 2097151)))
log.debug("sending browser final blob (%i/%i)", blob_info.blob_num + 1,
len(self.descriptor.blobs) - 1)
await response.write_eof(decrypted)
else:
log.debug("sending browser blob (%i/%i)", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
await response.write(decrypted)
wrote += len(decrypted)
log.info("sent browser %sblob %i/%i", "(final) " if response._eof_sent else "",
@@ -354,7 +368,7 @@ class ManagedStream:
self.started_writing.clear()
try:
with open(output_path, 'wb') as file_write_handle:
async for blob_info, decrypted in self._aiter_read_stream(connection_id=1):
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
await self.loop.run_in_executor(None, self._write_decrypted_blob, file_write_handle, decrypted)
self.written_bytes += len(decrypted)
@@ -485,7 +499,7 @@ class ManagedStream:
return
await asyncio.sleep(1, loop=self.loop)
def _prepare_range_response_headers(self, get_range: str) -> typing.Tuple[typing.Dict[str, str], int, int]:
def _prepare_range_response_headers(self, get_range: str) -> typing.Tuple[typing.Dict[str, str], int, int, int]:
if '=' in get_range:
get_range = get_range.split('=')[1]
start, end = get_range.split('-')
@@ -503,16 +517,23 @@ class ManagedStream:
log.debug("estimating stream size")
start = int(start)
if not 0 <= start < size:
raise HTTPRequestRangeNotSatisfiable()
end = int(end) if end else size - 1
if end >= size:
raise HTTPRequestRangeNotSatisfiable()
skip_blobs = start // 2097150
skip = skip_blobs * 2097151
start = skip
skip_first_blob = start - skip
start = skip_first_blob + skip
final_size = end - start + 1
headers = {
'Accept-Ranges': 'bytes',
'Content-Range': f'bytes {start}-{end}/{size}',
'Content-Length': str(final_size),
'Content-Type': self.mime_type
}
return headers, size, skip_blobs
return headers, size, skip_blobs, skip_first_blob
+15 -3
View File
@@ -10,13 +10,15 @@ if typing.TYPE_CHECKING:
REFLECTOR_V1 = 0
REFLECTOR_V2 = 1
MAX_RESPONSE_SIZE = 2000000
log = logging.getLogger(__name__)
class StreamReflectorClient(asyncio.Protocol):
def __init__(self, blob_manager: 'BlobManager', descriptor: 'StreamDescriptor'):
self.loop = asyncio.get_event_loop()
self.transport: asyncio.StreamWriter = None
self.transport: typing.Optional[asyncio.WriteTransport] = None
self.blob_manager = blob_manager
self.descriptor = descriptor
self.response_buff = b''
@@ -37,11 +39,21 @@ class StreamReflectorClient(asyncio.Protocol):
log.info("Finished sending reflector %i blobs", len(self.reflected_blobs))
def data_received(self, data):
if len(self.response_buff + (data or b'')) > MAX_RESPONSE_SIZE:
log.warning("response message to large from reflector server: %i bytes",
len(self.response_buff + (data or b'')))
self.response_buff = b''
self.transport.close()
return
self.response_buff += (data or b'')
try:
response = json.loads(data.decode())
response = json.loads(self.response_buff.decode())
self.response_buff = b''
self.response_queue.put_nowait(response)
except ValueError:
self.transport.close()
if not data:
log.warning("got undecodable response from reflector server")
self.response_buff = b''
return
async def send_request(self, request_dict: typing.Dict):
+15 -5
View File
@@ -15,7 +15,7 @@ log = logging.getLogger(__name__)
class ReflectorServerProtocol(asyncio.Protocol):
def __init__(self, blob_manager: 'BlobManager'):
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000):
self.loop = asyncio.get_event_loop()
self.blob_manager = blob_manager
self.server_task: asyncio.Task = None
@@ -28,6 +28,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
self.sd_blob: typing.Optional['BlobFile'] = None
self.received = []
self.incoming = asyncio.Event(loop=self.loop)
self.chunk_size = response_chunk_size
def connection_made(self, transport):
self.transport = transport
@@ -47,7 +48,15 @@ class ReflectorServerProtocol(asyncio.Protocol):
self.loop.create_task(self.handle_request(request))
def send_response(self, response: typing.Dict):
self.transport.write(json.dumps(response).encode())
def chunk_response(remaining: bytes):
f = self.loop.create_future()
f.add_done_callback(lambda _: self.transport.write(remaining[:self.chunk_size]))
if len(remaining) > self.chunk_size:
f.add_done_callback(lambda _: self.loop.call_soon(chunk_response, remaining[self.chunk_size:]))
self.loop.call_soon(f.set_result, None)
response_bytes = json.dumps(response).encode()
chunk_response(response_bytes)
async def handle_request(self, request: typing.Dict):
if self.client_version is None:
@@ -121,11 +130,12 @@ class ReflectorServerProtocol(asyncio.Protocol):
class ReflectorServer:
def __init__(self, blob_manager: 'BlobManager'):
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000):
self.loop = asyncio.get_event_loop()
self.blob_manager = blob_manager
self.server_task: asyncio.Task = None
self.server_task: typing.Optional[asyncio.Task] = None
self.started_listening = asyncio.Event(loop=self.loop)
self.response_chunk_size = response_chunk_size
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
if self.server_task is not None:
@@ -133,7 +143,7 @@ class ReflectorServer:
async def _start_server():
server = await self.loop.create_server(
lambda: ReflectorServerProtocol(self.blob_manager),
lambda: ReflectorServerProtocol(self.blob_manager, self.response_chunk_size),
interface, port
)
self.started_listening.set()
+2
View File
@@ -62,6 +62,7 @@ class CommandTestCase(IntegrationTestCase):
LEDGER = lbrynet.wallet
MANAGER = LbryWalletManager
VERBOSITY = logging.WARN
blob_lru_cache_size = 0
async def asyncSetUp(self):
await super().asyncSetUp()
@@ -81,6 +82,7 @@ class CommandTestCase(IntegrationTestCase):
conf.lbryum_servers = [('127.0.0.1', 50001)]
conf.reflector_servers = [('127.0.0.1', 5566)]
conf.known_dht_nodes = []
conf.blob_lru_cache_size = self.blob_lru_cache_size
await self.account.ensure_address_gap()
address = (await self.account.receiving.get_addresses(limit=1, only_usable=True))[0]
+7 -6
View File
@@ -75,7 +75,7 @@ def obfuscate(plain):
return rot13(base64.b64encode(plain).decode())
def check_connection(server="lbry.io", port=80, timeout=5) -> bool:
def check_connection(server="lbry.com", port=80, timeout=5) -> bool:
"""Attempts to open a socket to server:port and returns True if successful."""
log.debug('Checking connection to %s:%s', server, port)
try:
@@ -102,7 +102,7 @@ def check_connection(server="lbry.io", port=80, timeout=5) -> bool:
return False
async def async_check_connection(server="lbry.io", port=80, timeout=5) -> bool:
async def async_check_connection(server="lbry.com", port=80, timeout=5) -> bool:
return await asyncio.get_event_loop().run_in_executor(None, check_connection, server, port, timeout)
@@ -229,11 +229,12 @@ class LRUCache:
return item in self.cache
def lru_cache_concurrent(cache_size: int):
if not cache_size > 0:
def lru_cache_concurrent(cache_size: typing.Optional[int] = None,
override_lru_cache: typing.Optional[LRUCache] = None):
if not cache_size and override_lru_cache is None:
raise ValueError("invalid cache size")
concurrent_cache = {}
lru_cache = LRUCache(cache_size)
lru_cache = override_lru_cache or LRUCache(cache_size)
def wrapper(async_fn):
@@ -270,7 +271,7 @@ async def aiohttp_request(method, url, **kwargs) -> typing.AsyncContextManager[a
async def get_external_ip() -> typing.Optional[str]: # used if upnp is disabled or non-functioning
try:
async with aiohttp_request("get", "https://api.lbry.io/ip") as resp:
async with aiohttp_request("get", "https://api.lbry.com/ip") as resp:
response = await resp.json()
if response['success']:
return response['data']['ip']
-2
View File
@@ -46,8 +46,6 @@ class LBRYElectrumX(ElectrumX):
handlers.update({
'blockchain.numblocks.subscribe': self.numblocks_subscribe,
'blockchain.utxo.get_address': self.utxo_get_address,
'blockchain.transaction.broadcast':
self.transaction_broadcast_1_0,
'blockchain.transaction.get': self.transaction_get,
})
self.request_handlers.update(handlers)
+1 -1
View File
@@ -24,6 +24,6 @@ theme:
- type: 'telegram'
link: 'https://t.me/lbryofficial'
- type: 'discord'
link: 'https://chat.lbry.io'
link: 'https://chat.lbry.com'
- type: 'reddit'
link: 'https://reddit.com/r/lbry'
+3 -1
View File
@@ -1,5 +1,7 @@
FROM debian:buster-slim
ARG TORBA_VERSION=master
RUN apt-get update && \
apt-get upgrade -y && \
apt-get install -y --no-install-recommends \
@@ -15,7 +17,7 @@ RUN python3.7 -m pip install --upgrade pip setuptools wheel
COPY . /app
WORKDIR /app
RUN python3.7 -m pip install git+https://github.com/lbryio/torba.git#egg=torba
RUN python3.7 -m pip install --user git+https://github.com/lbryio/torba.git@${TORBA_VERSION}#egg=torba
RUN python3.7 -m pip install -e .
# Orchstr8 API
+2 -1
View File
@@ -1,6 +1,7 @@
FROM debian:buster-slim
ARG user=lbry
ARG TORBA_VERSION=master
# create an unprivileged user
RUN groupadd -g 999 $user && useradd -r -u 999 -g $user $user
@@ -23,7 +24,7 @@ WORKDIR /home/$user
RUN python3.7 -m pip install --upgrade pip setuptools
# get torba
RUN python3.7 -m pip install --user git+https://github.com/lbryio/torba.git#egg=torba
RUN python3.7 -m pip install --user git+https://github.com/lbryio/torba.git@${TORBA_VERSION}#egg=torba
# get uvloop
RUN python3.7 -m pip install --user uvloop
+1 -1
View File
@@ -47,7 +47,7 @@ async def main(blob_hash: str, url: str):
if __name__ == "__main__": # usage: python download_blob_from_peer.py <blob_hash> [host url:port]
url = 'reflector.lbry.io:5567'
url = 'reflector.lbry.com:5567'
if len(sys.argv) > 2:
url = sys.argv[2]
asyncio.run(main(sys.argv[1], url))
+26 -24
View File
@@ -1,33 +1,35 @@
"""Set the build version to be 'qa', 'rc', 'release'"""
"""Set the build version to be 'dev', 'qa', 'rc', 'release'"""
import os.path
import re
import subprocess
import sys
import os
import re
import logging
log = logging.getLogger()
log.addHandler(logging.StreamHandler())
log.setLevel(logging.DEBUG)
def get_build_type(travis_tag=None):
if not travis_tag:
return "qa"
log.debug("getting build type for tag: \"%s\"", travis_tag)
if re.match('v\d+\.\d+\.\d+rc\d+$', travis_tag):
return 'rc'
elif re.match('v\d+\.\d+\.\d+$', travis_tag):
return 'release'
return 'qa'
def main():
build = get_build()
root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
with open(os.path.join(root_dir, 'lbrynet', 'build_type.py'), 'w') as f:
f.write("BUILD = '{}'\n".format(build))
def get_build():
try:
tag = subprocess.check_output(['git', 'describe', '--exact-match', '--all']).strip()
if re.match('tags\/v\d+\.\d+\.\d+rc\d+$', tag.decode()):
print('Build: rc')
return 'rc'
elif re.match('tags\/v\d+\.\d+\.\d+$', tag.decode()):
print('Build: release')
return 'release'
print('Build: qa')
return 'qa'
except subprocess.CalledProcessError:
print("Couldn't determine build type, defaulting to qa.")
return 'qa'
build_type_path = os.path.join(root_dir, 'lbrynet', 'build_type.py')
log.debug("configuring build type file: %s", build_type_path)
travis_commit = os.environ['TRAVIS_COMMIT'][:6]
build_type = get_build_type(os.environ.get('TRAVIS_TAG', None))
log.debug("setting build type=%s, build commit=%s", build_type, travis_commit)
with open(build_type_path, 'w') as f:
f.write("BUILD = \"{}\"\nBUILD_COMMIT = \"{}\"\n".format(build_type, travis_commit))
if __name__ == '__main__':
+1 -1
View File
@@ -26,7 +26,7 @@ def extract_uris(response):
async def get_frontpage_uris():
session = aiohttp.ClientSession()
try:
response = await session.get("https://api.lbry.io/file/list_homepage", timeout=10.0)
response = await session.get("https://api.lbry.com/file/list_homepage", timeout=10.0)
if response.status != 200:
print("API returned non 200 code!!")
return
+9 -3
View File
@@ -1,13 +1,12 @@
set -x
# must also be updated in travis.yml
TORBA_VERSION=v0.5.4a0
rm -rf /tmp/.wine-*
apt-get -qq update
apt-get -qq install -y git
pip install setuptools_scm
git clone https://github.com/lbryio/torba.git --depth 1
cd torba && pip install -e . && cd ..
cd lbry
@@ -15,6 +14,13 @@ cd lbry
wget -Onetifaces-0.10.7-cp37-cp37m-win32.whl https://ci.appveyor.com/api/buildjobs/6hworunifsymrhp2/artifacts/dist%2Fnetifaces-0.10.7-cp37-cp37m-win32.whl
pip install netifaces-0.10.7-cp37-cp37m-win32.whl
git clone --depth=1 --single-branch --branch ${TORBA_VERSION} https://github.com/lbryio/torba.git
cd torba
pip install .
cd ..
rm -rf torba
pip show torba
pip install -e .
pip install pywin32
+4 -4
View File
@@ -9,8 +9,8 @@ setup(
name=__name__,
version=__version__,
author="LBRY Inc.",
author_email="hello@lbry.io",
url="https://lbry.io",
author_email="hello@lbry.com",
url="https://lbry.com",
description="A decentralized media library and marketplace",
long_description=open(README_PATH, encoding='utf-8').read(),
long_description_content_type="text/markdown",
@@ -23,9 +23,9 @@ setup(
'console_scripts': 'lbrynet=lbrynet.extras.cli:main'
},
install_requires=[
'torba',
'torba==0.5.4a0',
'aiohttp==3.5.4',
'aioupnp',
'aioupnp==0.0.12',
'appdirs==1.4.3',
'certifi>=2018.11.29',
'colorama==0.3.7',
+44
View File
@@ -2,6 +2,7 @@ import os
import hashlib
import aiohttp
import aiohttp.web
import asyncio
from lbrynet.utils import aiohttp_request
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
@@ -373,3 +374,46 @@ class RangeRequests(CommandTestCase):
await stream.finished_writing.wait()
with open(stream.full_path, 'rb') as f:
self.assertEqual(self.data, f.read())
class RangeRequestsLRUCache(CommandTestCase):
blob_lru_cache_size = 32
async def _request_stream(self):
name = 'foo'
url = f'http://{self.daemon.conf.streaming_host}:{self.daemon.conf.streaming_port}/get/{name}'
async with aiohttp_request('get', url) as req:
self.assertEqual(req.headers.get('Content-Type'), 'application/octet-stream')
content_range = req.headers.get('Content-Range')
content_length = int(req.headers.get('Content-Length'))
streamed_bytes = await req.content.read()
self.assertEqual(content_length, len(streamed_bytes))
self.assertEqual(15, content_length)
self.assertEqual(b'hi\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', streamed_bytes)
self.assertEqual('bytes 0-14/15', content_range)
async def test_range_requests_with_blob_lru_cache(self):
self.data = b'hi'
self.daemon.conf.save_blobs = False
self.daemon.conf.save_files = False
await self.stream_create('foo', '0.01', data=self.data, file_size=0)
await self.daemon.jsonrpc_file_list()[0].fully_reflected.wait()
await self.daemon.jsonrpc_file_delete(delete_from_download_dir=True, claim_name='foo')
self.assertEqual(0, len(os.listdir(self.daemon.blob_manager.blob_dir)))
await self.daemon.streaming_runner.setup()
site = aiohttp.web.TCPSite(self.daemon.streaming_runner, self.daemon.conf.streaming_host,
self.daemon.conf.streaming_port)
await site.start()
self.assertListEqual(self.daemon.jsonrpc_file_list(), [])
await self._request_stream()
self.assertEqual(1, len(self.daemon.jsonrpc_file_list()))
self.server.stop_server()
# running with cache size 0 gets through without errors without
# this since the server doesnt stop immediately
await asyncio.sleep(1, loop=self.loop)
await self._request_stream()
+4 -1
View File
@@ -83,12 +83,15 @@ class TestProtocol(AsyncioTestCase):
find_value_response = peer1.node_rpc.find_value(peer3, b'2' * 48)
self.assertEqual(len(find_value_response[b'contacts']), 0)
self.assertSetEqual(
{b'2' * 48, b'token', b'protocolVersion', b'contacts'}, set(find_value_response.keys())
{b'2' * 48, b'token', b'protocolVersion', b'contacts', b'p'}, set(find_value_response.keys())
)
self.assertEqual(2, len(find_value_response[b'2' * 48]))
self.assertEqual(find_value_response[b'2' * 48][0], peer2_from_peer1.compact_address_tcp())
self.assertDictEqual(bdecode(bencode(find_value_response)), find_value_response)
find_value_page_above_pages_response = peer1.node_rpc.find_value(peer3, b'2' * 48, page=10)
self.assertNotIn(b'2' * 48, find_value_page_above_pages_response)
peer1.stop()
peer2.stop()
peer1.disconnect()
+22 -2
View File
@@ -59,17 +59,28 @@ class TestDatagram(unittest.TestCase):
self.assertRaises(ValueError, RequestDatagram.make_find_value, b'1' * 49, b'2' * 48, b'1' * 20)
self.assertRaises(ValueError, RequestDatagram.make_find_value, b'1' * 48, b'2' * 49, b'1' * 20)
self.assertRaises(ValueError, RequestDatagram.make_find_value, b'1' * 48, b'2' * 48, b'1' * 21)
self.assertRaises(ValueError, RequestDatagram.make_find_value, b'1' * 48, b'2' * 48, b'1' * 20, -1)
self.assertEqual(20, len(RequestDatagram.make_find_value(b'1' * 48, b'2' * 48).rpc_id))
# default page argument
serialized = RequestDatagram.make_find_value(b'1' * 48, b'2' * 48, b'1' * 20).bencode()
decoded = decode_datagram(serialized)
self.assertEqual(decoded.packet_type, REQUEST_TYPE)
self.assertEqual(decoded.rpc_id, b'1' * 20)
self.assertEqual(decoded.node_id, b'1' * 48)
self.assertEqual(decoded.method, b'findValue')
self.assertListEqual(decoded.args, [b'2' * 48, {b'protocolVersion': 1}])
self.assertListEqual(decoded.args, [b'2' * 48, {b'protocolVersion': 1, b'p': 0}])
def test_find_value_response(self):
# nondefault page argument
serialized = RequestDatagram.make_find_value(b'1' * 48, b'2' * 48, b'1' * 20, 1).bencode()
decoded = decode_datagram(serialized)
self.assertEqual(decoded.packet_type, REQUEST_TYPE)
self.assertEqual(decoded.rpc_id, b'1' * 20)
self.assertEqual(decoded.node_id, b'1' * 48)
self.assertEqual(decoded.method, b'findValue')
self.assertListEqual(decoded.args, [b'2' * 48, {b'protocolVersion': 1, b'p': 1}])
def test_find_value_response_without_pages_field(self):
found_value_response = {b'2' * 48: [b'\x7f\x00\x00\x01']}
serialized = ResponseDatagram(RESPONSE_TYPE, b'1' * 20, b'1' * 48, found_value_response).bencode()
decoded = decode_datagram(serialized)
@@ -78,6 +89,15 @@ class TestDatagram(unittest.TestCase):
self.assertEqual(decoded.node_id, b'1' * 48)
self.assertDictEqual(decoded.response, found_value_response)
def test_find_value_response_with_pages_field(self):
found_value_response = {b'2' * 48: [b'\x7f\x00\x00\x01'], b'p': 1}
serialized = ResponseDatagram(RESPONSE_TYPE, b'1' * 20, b'1' * 48, found_value_response).bencode()
decoded = decode_datagram(serialized)
self.assertEqual(decoded.packet_type, RESPONSE_TYPE)
self.assertEqual(decoded.rpc_id, b'1' * 20)
self.assertEqual(decoded.node_id, b'1' * 48)
self.assertDictEqual(decoded.response, found_value_response)
def test_store_request(self):
self.assertRaises(ValueError, RequestDatagram.make_store, b'1' * 47, b'2' * 48, b'3' * 48, 3333, b'1' * 20)
self.assertRaises(ValueError, RequestDatagram.make_store, b'1' * 48, b'2' * 49, b'3' * 48, 3333, b'1' * 20)
+55
View File
@@ -1,6 +1,7 @@
import contextlib
import typing
import binascii
import socket
import asyncio
from torba.testcase import AsyncioTestCase
from tests import dht_mocks
@@ -26,6 +27,7 @@ class TestBlobAnnouncer(AsyncioTestCase):
for node_id, address in peer_addresses:
await self.add_peer(node_id, address)
self.node.joined.set()
self.node._refresh_task = self.loop.create_task(self.node.refresh_node())
async def add_peer(self, node_id, address, add_to_routing_table=True):
n = Node(self.loop, PeerManager(self.loop), node_id, 4444, 4444, 3333, address)
@@ -113,3 +115,56 @@ class TestBlobAnnouncer(AsyncioTestCase):
self.assertEqual(self.node.protocol.node_id, found_peers[0].node_id)
self.assertEqual(self.node.protocol.external_ip, found_peers[0].address)
self.assertEqual(self.node.protocol.peer_port, found_peers[0].tcp_port)
async def test_popular_blob(self):
peer_count = 150
addresses = [
(constants.generate_id(i + 1), socket.inet_ntoa(int(i + 1).to_bytes(length=4, byteorder='big')))
for i in range(peer_count)
]
blob_hash = b'1' * 48
async with self._test_network_context(peer_addresses=addresses):
total_seen = set()
announced_to = self.nodes[0]
for i in range(1, peer_count):
node = self.nodes[i]
kad_peer = announced_to.protocol.peer_manager.get_kademlia_peer(
node.protocol.node_id, node.protocol.external_ip, node.protocol.udp_port
)
await announced_to.protocol._add_peer(kad_peer)
peer = node.protocol.get_rpc_peer(
node.protocol.peer_manager.get_kademlia_peer(
announced_to.protocol.node_id,
announced_to.protocol.external_ip,
announced_to.protocol.udp_port
)
)
response = await peer.store(blob_hash)
self.assertEqual(response, b'OK')
peers_for_blob = await peer.find_value(blob_hash, 0)
if i == 1:
self.assertTrue(blob_hash not in peers_for_blob)
self.assertEqual(peers_for_blob[b'p'], 0)
else:
self.assertEqual(len(peers_for_blob[blob_hash]), min(i - 1, constants.k))
self.assertEqual(len(announced_to.protocol.data_store.get_peers_for_blob(blob_hash)), i)
if i - 1 > constants.k:
self.assertEqual(len(peers_for_blob[b'contacts']), constants.k)
self.assertEqual(peers_for_blob[b'p'], ((i - 1) // (constants.k + 1)) + 1)
seen = set(peers_for_blob[blob_hash])
self.assertEqual(len(seen), constants.k)
self.assertEqual(len(peers_for_blob[blob_hash]), len(seen))
for pg in range(1, peers_for_blob[b'p']):
page_x = await peer.find_value(blob_hash, pg)
self.assertNotIn(b'contacts', page_x)
page_x_set = set(page_x[blob_hash])
self.assertEqual(len(page_x[blob_hash]), len(page_x_set))
self.assertTrue(len(page_x_set) > 0)
self.assertSetEqual(seen.intersection(page_x_set), set())
seen.intersection_update(page_x_set)
total_seen.update(page_x_set)
else:
self.assertEqual(len(peers_for_blob[b'contacts']), i - 1)
self.assertEqual(len(total_seen), peer_count - 2)
+8 -2
View File
@@ -41,8 +41,8 @@ class TestStreamAssembler(AsyncioTestCase):
self.stream = await self.stream_manager.create_stream(file_path)
async def test_reflect_stream(self):
reflector = ReflectorServer(self.server_blob_manager)
async def _test_reflect_stream(self, response_chunk_size):
reflector = ReflectorServer(self.server_blob_manager, response_chunk_size=response_chunk_size)
reflector.start_server(5566, '127.0.0.1')
await reflector.started_listening.wait()
self.addCleanup(reflector.stop_server)
@@ -63,6 +63,12 @@ class TestStreamAssembler(AsyncioTestCase):
sent = await self.stream.upload_to_reflector('127.0.0.1', 5566)
self.assertListEqual(sent, [])
async def test_reflect_stream(self):
return await asyncio.wait_for(self._test_reflect_stream(response_chunk_size=50), 3, loop=self.loop)
async def test_reflect_stream_small_response_chunks(self):
return await asyncio.wait_for(self._test_reflect_stream(response_chunk_size=30), 3, loop=self.loop)
async def test_announces(self):
to_announce = await self.storage.get_blobs_to_announce()
self.assertIn(self.stream.sd_hash, to_announce, "sd blob not set to announce")
+1
View File
@@ -5,6 +5,7 @@ envlist = py37-integration
deps =
coverage
../torba
extras = test
changedir = {toxinidir}/tests
setenv =