Compare commits
26
Commits
master
...
v0.37.5rc2
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bbc1f95ff1 | ||
|
|
9f729c29c3 | ||
|
|
b468a82b53 | ||
|
|
e0e9c4efd0 | ||
|
|
c16433357b | ||
|
|
a8cb873de5 | ||
|
|
d364431619 | ||
|
|
bec75a957d | ||
|
|
338ea79618 | ||
|
|
61bdbceca3 | ||
|
|
8683e96156 | ||
|
|
4ae59b15e0 | ||
|
|
3771263179 | ||
|
|
80ab0ba46d | ||
|
|
bd31c4c395 | ||
|
|
9bcb1d7d95 | ||
|
|
e9ce41bea6 | ||
|
|
734b80b2b2 | ||
|
|
a90919a9ba | ||
|
|
347415dceb | ||
|
|
fe55187696 | ||
|
|
db8fb15cc5 | ||
|
|
a165ad910f | ||
|
|
b711069456 | ||
|
|
c977a672b0 | ||
|
|
ac5794d96d |
No files matched your search
@@ -7,7 +7,7 @@ Before reporting any issues, please make sure that you're using the latest versi
|
||||
- App: https://github.com/lbryio/lbry-desktop/releases
|
||||
- Daemon: https://github.com/lbryio/lbry/releases
|
||||
|
||||
We are also available on Discord at https://chat.lbry.io
|
||||
We are also available on Discord at https://chat.lbry.com
|
||||
-->
|
||||
|
||||
|
||||
|
||||
@@ -24,7 +24,7 @@ persistent=yes
|
||||
load-plugins=
|
||||
|
||||
# Use multiple processes to speed up Pylint.
|
||||
jobs=1
|
||||
jobs=4
|
||||
|
||||
# Allow loading of arbitrary C extensions. Extensions are imported into the
|
||||
# active Python interpreter and may run arbitrary code.
|
||||
@@ -77,8 +77,6 @@ disable=
|
||||
dangerous-default-value,
|
||||
duplicate-code,
|
||||
fixme,
|
||||
global-statement,
|
||||
inherit-non-class,
|
||||
invalid-name,
|
||||
len-as-condition,
|
||||
locally-disabled,
|
||||
@@ -111,13 +109,11 @@ disable=
|
||||
unnecessary-lambda,
|
||||
unused-argument,
|
||||
unused-variable,
|
||||
wildcard-import,
|
||||
wrong-import-order,
|
||||
wrong-import-position,
|
||||
deprecated-lambda,
|
||||
simplifiable-if-statement,
|
||||
unidiomatic-typecheck,
|
||||
global-at-module-level,
|
||||
inconsistent-return-statements,
|
||||
keyword-arg-before-vararg,
|
||||
assignment-from-no-return,
|
||||
|
||||
+13
-7
@@ -3,6 +3,11 @@ dist: xenial
|
||||
language: python
|
||||
python: "3.7"
|
||||
|
||||
env:
|
||||
global:
|
||||
# must also be updated in wine_build.sh
|
||||
- TORBA=v0.5.4a0
|
||||
|
||||
jobs:
|
||||
include:
|
||||
|
||||
@@ -10,7 +15,7 @@ jobs:
|
||||
name: "pylint lbrynet"
|
||||
install:
|
||||
- pip install astroid pylint
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
|
||||
- pip install -e .
|
||||
script: pylint lbrynet
|
||||
|
||||
@@ -18,7 +23,7 @@ jobs:
|
||||
name: "Unit Tests"
|
||||
install:
|
||||
- pip install coverage
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
|
||||
- pip install -e .
|
||||
script:
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m unittest discover -vv tests.unit
|
||||
@@ -29,7 +34,8 @@ jobs:
|
||||
- name: "Integration Tests"
|
||||
install:
|
||||
- pip install tox-travis coverage
|
||||
- pushd .. && git clone https://github.com/lbryio/torba.git && popd
|
||||
- pushd .. && git clone --single-branch --branch ${TORBA} https://github.com/lbryio/torba.git && popd
|
||||
|
||||
script: tox
|
||||
after_success:
|
||||
- coverage combine tests/
|
||||
@@ -38,7 +44,7 @@ jobs:
|
||||
- name: "Run Examples"
|
||||
install:
|
||||
- pip install coverage
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
|
||||
- pip install -e .
|
||||
script:
|
||||
- HOME=/tmp coverage run -p --source=lbrynet scripts/generate_json_api.py
|
||||
@@ -79,7 +85,7 @@ jobs:
|
||||
env: OS=linux
|
||||
install:
|
||||
- pip3 install pyinstaller
|
||||
- pip3 install git+https://github.com/lbryio/torba.git
|
||||
- pip3 install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
|
||||
- python3 scripts/set_build.py
|
||||
- pip3 install -e .
|
||||
script:
|
||||
@@ -121,7 +127,7 @@ jobs:
|
||||
script:
|
||||
- set -e
|
||||
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
|
||||
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f scripts/Dockerfile.wallet_server .
|
||||
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f scripts/Dockerfile.wallet_server --build-arg TORBA_VERSION=${TORBA} .
|
||||
- docker push lbry/wallet-server:$TRAVIS_TAG
|
||||
- if: tag IS blank AND branch = master AND NOT type IN (pull_request)
|
||||
stage: build
|
||||
@@ -129,7 +135,7 @@ jobs:
|
||||
script:
|
||||
- set -e
|
||||
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
|
||||
- travis_retry docker build -t lbry/wallet-server:master -f scripts/Dockerfile.wallet_server .
|
||||
- travis_retry docker build -t lbry/wallet-server:master -f scripts/Dockerfile.wallet_server --build-arg TORBA_VERSION=${TORBA} .
|
||||
- docker push lbry/wallet-server:master
|
||||
|
||||
cache:
|
||||
|
||||
@@ -41,7 +41,7 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
|
||||
|
||||
## Security
|
||||
|
||||
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.io/faq/gpg-key) if you need it.
|
||||
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.com/faq/gpg-key) if you need it.
|
||||
|
||||
## Contact
|
||||
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
__name__ = "lbrynet"
|
||||
__version__ = "0.37.1"
|
||||
__version__ = "0.37.5rc2"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
@@ -2,8 +2,10 @@ import os
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
from lbrynet.utils import LRUCache
|
||||
from lbrynet.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.connection_manager import ConnectionManager
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.conf import Config
|
||||
@@ -30,6 +32,9 @@ class BlobManager:
|
||||
else self._node_data_store.completed_blobs
|
||||
self.blobs: typing.Dict[str, AbstractBlob] = {}
|
||||
self.config = config
|
||||
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCache(
|
||||
self.config.blob_lru_cache_size)
|
||||
self.connection_manager = ConnectionManager(loop)
|
||||
|
||||
def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None):
|
||||
if self.config.save_blobs:
|
||||
@@ -81,9 +86,11 @@ class BlobManager:
|
||||
to_add = await self.storage.sync_missing_blobs(in_blobfiles_dir)
|
||||
if to_add:
|
||||
self.completed_blob_hashes.update(to_add)
|
||||
self.connection_manager.start()
|
||||
return True
|
||||
|
||||
def stop(self):
|
||||
self.connection_manager.stop()
|
||||
while self.blobs:
|
||||
_, blob = self.blobs.popitem()
|
||||
blob.close()
|
||||
|
||||
@@ -8,18 +8,20 @@ from lbrynet.utils import cache_concurrent
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.blob.blob_file import AbstractBlob
|
||||
from lbrynet.blob.writer import HashBlobWriter
|
||||
from lbrynet.connection_manager import ConnectionManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_timeout: typing.Optional[float] = 10):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_timeout: typing.Optional[float] = 10,
|
||||
connection_manager: typing.Optional['ConnectionManager'] = None):
|
||||
self.loop = loop
|
||||
self.peer_port: typing.Optional[int] = None
|
||||
self.peer_address: typing.Optional[str] = None
|
||||
self.peer_timeout = peer_timeout
|
||||
self.transport: typing.Optional[asyncio.Transport] = None
|
||||
|
||||
self.peer_timeout = peer_timeout
|
||||
self.connection_manager = connection_manager
|
||||
self.writer: typing.Optional['HashBlobWriter'] = None
|
||||
self.blob: typing.Optional['AbstractBlob'] = None
|
||||
|
||||
@@ -31,6 +33,12 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
self.closed = asyncio.Event(loop=self.loop)
|
||||
|
||||
def data_received(self, data: bytes):
|
||||
if self.connection_manager:
|
||||
if not self.peer_address:
|
||||
addr_info = self.transport.get_extra_info('peername')
|
||||
self.peer_address, self.peer_port = addr_info
|
||||
# assert self.peer_address is not None
|
||||
self.connection_manager.received_data(f"{self.peer_address}:{self.peer_port}", len(data))
|
||||
#log.debug("%s:%d -- got %s bytes -- %s bytes on buffer -- %s blob bytes received",
|
||||
# self.peer_address, self.peer_port, len(data), len(self.buf), self._blob_bytes_received)
|
||||
if not self.transport or self.transport.is_closing():
|
||||
@@ -94,10 +102,15 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
"""
|
||||
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
|
||||
blob_hash = self.blob.blob_hash
|
||||
if not self.peer_address:
|
||||
addr_info = self.transport.get_extra_info('peername')
|
||||
self.peer_address, self.peer_port = addr_info
|
||||
try:
|
||||
msg = request.serialize()
|
||||
log.debug("send request to %s:%i -> %s", self.peer_address, self.peer_port, msg.decode())
|
||||
self.transport.write(msg)
|
||||
if self.connection_manager:
|
||||
self.connection_manager.sent_data(f"{self.peer_address}:{self.peer_port}", len(msg))
|
||||
response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop)
|
||||
availability_response = response.get_availability_response()
|
||||
price_response = response.get_price_response()
|
||||
@@ -112,11 +125,16 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
self.peer_port)
|
||||
log.warning(response.to_dict())
|
||||
return self._blob_bytes_received, self.close()
|
||||
elif availability_response.available_blobs and \
|
||||
elif availability_response and availability_response.available_blobs and \
|
||||
availability_response.available_blobs != [self.blob.blob_hash]:
|
||||
log.warning("blob availability response doesn't match our request from %s:%i",
|
||||
self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
elif not availability_response:
|
||||
log.warning("response from %s:%i did not include an availability response (we requested %s)",
|
||||
self.peer_address, self.peer_port, blob_hash)
|
||||
return self._blob_bytes_received, self.close()
|
||||
|
||||
if not price_response or price_response.blob_data_payment_rate != 'RATE_ACCEPTED':
|
||||
log.warning("data rate rejected by %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
@@ -186,11 +204,16 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
self.writer = None
|
||||
|
||||
def connection_made(self, transport: asyncio.Transport):
|
||||
addr = transport.get_extra_info('peername')
|
||||
self.peer_address, self.peer_port = addr[0], addr[1]
|
||||
self.transport = transport
|
||||
self.peer_address, self.peer_port = self.transport.get_extra_info('peername')
|
||||
if self.connection_manager:
|
||||
self.connection_manager.connection_made(f"{self.peer_address}:{self.peer_port}")
|
||||
log.debug("connection made to %s:%i", self.peer_address, self.peer_port)
|
||||
|
||||
def connection_lost(self, reason):
|
||||
if self.connection_manager:
|
||||
self.connection_manager.outgoing_connection_lost(f"{self.peer_address}:{self.peer_port}")
|
||||
log.debug("connection lost to %s:%i (reason: %s, %s)", self.peer_address, self.peer_port, str(reason),
|
||||
str(type(reason)))
|
||||
self.close()
|
||||
@@ -199,16 +222,19 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
@cache_concurrent
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'AbstractBlob', address: str, tcp_port: int,
|
||||
peer_connect_timeout: float, blob_download_timeout: float,
|
||||
connected_transport: asyncio.Transport = None, connection_id: int = 0)\
|
||||
connected_transport: asyncio.Transport = None, connection_id: int = 0,
|
||||
connection_manager: typing.Optional['ConnectionManager'] = None)\
|
||||
-> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
|
||||
"""
|
||||
Returns [<downloaded blob>, <keep connection>]
|
||||
"""
|
||||
|
||||
protocol = BlobExchangeClientProtocol(loop, blob_download_timeout)
|
||||
protocol = BlobExchangeClientProtocol(
|
||||
loop, blob_download_timeout, connection_manager
|
||||
)
|
||||
if connected_transport and not connected_transport.is_closing():
|
||||
connected_transport.set_protocol(protocol)
|
||||
protocol.connection_made(connected_transport)
|
||||
protocol.transport = connected_transport
|
||||
log.debug("reusing connection for %s:%d", address, tcp_port)
|
||||
else:
|
||||
connected_transport = None
|
||||
|
||||
@@ -41,7 +41,9 @@ class BlobDownloader:
|
||||
start = self.loop.time()
|
||||
bytes_received, transport = await request_blob(
|
||||
self.loop, blob, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
|
||||
self.config.blob_download_timeout, connected_transport=transport, connection_id=connection_id
|
||||
self.config.blob_download_timeout, connected_transport=transport, connection_id=connection_id,
|
||||
connection_manager=self.blob_manager.connection_manager
|
||||
|
||||
)
|
||||
if not transport and peer not in self.ignored:
|
||||
self.ignored[peer] = self.loop.time()
|
||||
|
||||
@@ -22,15 +22,23 @@ class BlobServerProtocol(asyncio.Protocol):
|
||||
self.buf = b''
|
||||
self.transport = None
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
self.peer_address_and_port: typing.Optional[str] = None
|
||||
|
||||
def connection_made(self, transport):
|
||||
self.transport = transport
|
||||
self.peer_address_and_port = "%s:%i" % self.transport.get_extra_info('peername')
|
||||
self.blob_manager.connection_manager.connection_received(self.peer_address_and_port)
|
||||
|
||||
def connection_lost(self, exc: typing.Optional[Exception]) -> None:
|
||||
self.blob_manager.connection_manager.incoming_connection_lost(self.peer_address_and_port)
|
||||
|
||||
def send_response(self, responses: typing.List[blob_response_types]):
|
||||
to_send = []
|
||||
while responses:
|
||||
to_send.append(responses.pop())
|
||||
self.transport.write(BlobResponse(to_send).serialize())
|
||||
serialized = BlobResponse(to_send).serialize()
|
||||
self.transport.write(serialized)
|
||||
self.blob_manager.connection_manager.sent_data(self.peer_address_and_port, len(serialized))
|
||||
|
||||
async def handle_request(self, request: BlobRequest):
|
||||
addr = self.transport.get_extra_info('peername')
|
||||
@@ -72,6 +80,7 @@ class BlobServerProtocol(asyncio.Protocol):
|
||||
def data_received(self, data):
|
||||
request = None
|
||||
if data:
|
||||
self.blob_manager.connection_manager.received_data(self.peer_address_and_port, len(data))
|
||||
message, separator, remainder = data.rpartition(b'}')
|
||||
if not separator:
|
||||
self.buf += data
|
||||
@@ -97,7 +106,7 @@ class BlobServer:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str):
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
self.server_task: typing.Optional[asyncio.Task] = None
|
||||
self.started_listening = asyncio.Event(loop=self.loop)
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
self.server_protocol_class = BlobServerProtocol
|
||||
|
||||
@@ -1,2 +1,3 @@
|
||||
# don't touch this. CI server changes this during build/deployment
|
||||
BUILD = "dev"
|
||||
BUILD_COMMIT = "source installation"
|
||||
+5
-2
@@ -485,7 +485,10 @@ class Config(CLIConfig):
|
||||
|
||||
# blob announcement and download
|
||||
save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True)
|
||||
|
||||
blob_lru_cache_size = Integer(
|
||||
"LRU cache size for decrypted downloaded blobs used to minimize re-downloading the same blobs when "
|
||||
"replying to a range request. Set to 0 to disable.", 32
|
||||
)
|
||||
announce_head_and_sd_only = Toggle(
|
||||
"Announce only the descriptor and first (rather than all) data blob for a stream to the DHT", True,
|
||||
previous_names=['announce_head_blobs_only']
|
||||
@@ -517,7 +520,7 @@ class Config(CLIConfig):
|
||||
|
||||
# servers
|
||||
reflector_servers = Servers("Reflector re-hosting servers", [
|
||||
('reflector.lbry.io', 5566)
|
||||
('reflector.lbry.com', 5566)
|
||||
])
|
||||
lbryum_servers = Servers("SPV wallet servers", [
|
||||
('lbryumx1.lbry.com', 50001), # US EAST
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import collections
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
CONNECTED_EVENT = "connected"
|
||||
DISCONNECTED_EVENT = "disconnected"
|
||||
TRANSFERRED_EVENT = "transferred"
|
||||
|
||||
|
||||
class ConnectionManager:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop):
|
||||
self.loop = loop
|
||||
self.incoming_connected: typing.Set[str] = set()
|
||||
self.incoming: typing.DefaultDict[str, int] = collections.defaultdict(int)
|
||||
self.outgoing_connected: typing.Set[str] = set()
|
||||
self.outgoing: typing.DefaultDict[str, int] = collections.defaultdict(int)
|
||||
self._status = {}
|
||||
|
||||
self._task: typing.Optional[asyncio.Task] = None
|
||||
|
||||
@property
|
||||
def status(self):
|
||||
return self._status
|
||||
|
||||
def sent_data(self, host_and_port: str, size: int):
|
||||
self.outgoing[host_and_port] += size
|
||||
|
||||
def received_data(self, host_and_port: str, size: int):
|
||||
self.incoming[host_and_port] += size
|
||||
|
||||
def connection_made(self, host_and_port: str):
|
||||
self.outgoing_connected.add(host_and_port)
|
||||
|
||||
def connection_received(self, host_and_port: str):
|
||||
# self.incoming_connected.add(host_and_port)
|
||||
pass
|
||||
|
||||
def outgoing_connection_lost(self, host_and_port: str):
|
||||
if host_and_port in self.outgoing_connected:
|
||||
self.outgoing_connected.remove(host_and_port)
|
||||
|
||||
def incoming_connection_lost(self, host_and_port: str):
|
||||
if host_and_port in self.incoming_connected:
|
||||
self.incoming_connected.remove(host_and_port)
|
||||
|
||||
async def _update(self):
|
||||
|
||||
self._status = {
|
||||
'incoming_bps': {},
|
||||
'outgoing_bps': {},
|
||||
'total_incoming_mbps': 0.0,
|
||||
'total_outgoing_mbps': 0.0,
|
||||
}
|
||||
|
||||
while True:
|
||||
last = self.loop.time()
|
||||
await asyncio.sleep(1, loop=self.loop)
|
||||
self._status['incoming_bps'].clear()
|
||||
self._status['outgoing_bps'].clear()
|
||||
while self.outgoing:
|
||||
k, v = self.outgoing.popitem()
|
||||
self._status['outgoing_bps'][k] = v
|
||||
while self.incoming:
|
||||
k, v = self.incoming.popitem()
|
||||
self._status['incoming_bps'][k] = v
|
||||
now = self.loop.time()
|
||||
self._status['total_outgoing_mbps'] = int(sum(list(self._status['outgoing_bps'].values()))
|
||||
/ (now - last)) / 1000000.0
|
||||
self._status['total_incoming_mbps'] = int(sum(list(self._status['incoming_bps'].values()))
|
||||
/ (now - last)) / 1000000.0
|
||||
self._status['time'] = now
|
||||
|
||||
def stop(self):
|
||||
if self._task:
|
||||
self._task.cancel()
|
||||
self._task = None
|
||||
self.outgoing.clear()
|
||||
self.outgoing_connected.clear()
|
||||
self.incoming.clear()
|
||||
self.incoming_connected.clear()
|
||||
self._status.clear()
|
||||
|
||||
def start(self):
|
||||
self.stop()
|
||||
self._task = self.loop.create_task(self._update())
|
||||
@@ -18,11 +18,10 @@ data_expiration = 86400 # 24 hours
|
||||
token_secret_refresh_interval = 300 # 5 minutes
|
||||
maybe_ping_delay = 300 # 5 minutes
|
||||
check_refresh_interval = refresh_interval / 5
|
||||
max_datagram_size = 8192 # 8 KB
|
||||
rpc_id_length = 20
|
||||
protocol_version = 1
|
||||
bottom_out_limit = 3
|
||||
msg_size_limit = max_datagram_size - 26
|
||||
msg_size_limit = 1400
|
||||
|
||||
|
||||
def digest(data: bytes) -> bytes:
|
||||
|
||||
@@ -5,12 +5,13 @@ import hashlib
|
||||
import asyncio
|
||||
import typing
|
||||
import binascii
|
||||
import random
|
||||
from asyncio.protocols import DatagramProtocol
|
||||
from asyncio.transports import DatagramTransport
|
||||
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.serialization.datagram import decode_datagram, ErrorDatagram, ResponseDatagram, RequestDatagram
|
||||
from lbrynet.dht.serialization.datagram import RESPONSE_TYPE, ERROR_TYPE
|
||||
from lbrynet.dht.serialization.datagram import RESPONSE_TYPE, ERROR_TYPE, PAGE_KEY
|
||||
from lbrynet.dht.error import RemoteException, TransportNotConnected
|
||||
from lbrynet.dht.protocol.routing_table import TreeRoutingTable
|
||||
from lbrynet.dht.protocol.data_store import DictDataStore
|
||||
@@ -67,19 +68,23 @@ class KademliaRPC:
|
||||
|
||||
contacts = self.protocol.routing_table.find_close_peers(key, sender_node_id=rpc_contact.node_id)
|
||||
contact_triples = []
|
||||
for contact in contacts:
|
||||
for contact in contacts[:constants.k * 2]:
|
||||
contact_triples.append((contact.node_id, contact.address, contact.udp_port))
|
||||
return contact_triples
|
||||
|
||||
def find_value(self, rpc_contact: 'KademliaPeer', key: bytes):
|
||||
def find_value(self, rpc_contact: 'KademliaPeer', key: bytes, page: int = 0):
|
||||
page = page if page > 0 else 0
|
||||
|
||||
if len(key) != constants.hash_length:
|
||||
raise ValueError("invalid blob_exchange hash length: %i" % len(key))
|
||||
|
||||
response = {
|
||||
b'token': self.make_token(rpc_contact.compact_ip()),
|
||||
b'contacts': self.find_node(rpc_contact, key)
|
||||
}
|
||||
|
||||
if not page:
|
||||
response[b'contacts'] = self.find_node(rpc_contact, key)[:constants.k]
|
||||
|
||||
if self.protocol.protocol_version:
|
||||
response[b'protocolVersion'] = self.protocol.protocol_version
|
||||
|
||||
@@ -92,8 +97,14 @@ class KademliaRPC:
|
||||
# if we don't have k storing peers to return and we have this hash locally, include our contact information
|
||||
if len(peers) < constants.k and binascii.hexlify(key).decode() in self.protocol.data_store.completed_blobs:
|
||||
peers.append(self.compact_address())
|
||||
if peers:
|
||||
response[key] = peers
|
||||
if not peers:
|
||||
response[PAGE_KEY] = 0
|
||||
else:
|
||||
response[PAGE_KEY] = (len(peers) // (constants.k + 1)) + 1 # how many pages of peers we have for the blob
|
||||
if len(peers) > constants.k:
|
||||
random.Random(self.protocol.node_id).shuffle(peers)
|
||||
if page * constants.k < len(peers):
|
||||
response[key] = peers[page * constants.k:page * constants.k + constants.k]
|
||||
return response
|
||||
|
||||
def refresh_token(self): # TODO: this needs to be called periodically
|
||||
@@ -166,7 +177,7 @@ class RemoteKademliaRPC:
|
||||
)
|
||||
return [(node_id, address.decode(), udp_port) for node_id, address, udp_port in response.response]
|
||||
|
||||
async def find_value(self, key: bytes) -> typing.Union[typing.Dict]:
|
||||
async def find_value(self, key: bytes, page: int = 0) -> typing.Union[typing.Dict]:
|
||||
"""
|
||||
:return: {
|
||||
b'token': <token bytes>,
|
||||
@@ -177,7 +188,7 @@ class RemoteKademliaRPC:
|
||||
if len(key) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid length of find value key: {len(key)}")
|
||||
response = await self.protocol.send_request(
|
||||
self.peer, RequestDatagram.make_find_value(self.protocol.node_id, key)
|
||||
self.peer, RequestDatagram.make_find_value(self.protocol.node_id, key, page=page)
|
||||
)
|
||||
self.peer_tracker.update_token(self.peer.node_id, response.response[b'token'])
|
||||
return response.response
|
||||
@@ -406,24 +417,25 @@ class KademliaProtocol(DatagramProtocol):
|
||||
raise AttributeError('Invalid method: %s' % message.method.decode())
|
||||
if message.args and isinstance(message.args[-1], dict) and b'protocolVersion' in message.args[-1]:
|
||||
# args don't need reformatting
|
||||
a, kw = tuple(message.args[:-1]), message.args[-1]
|
||||
args, kw = tuple(message.args[:-1]), message.args[-1]
|
||||
else:
|
||||
a, kw = self._migrate_incoming_rpc_args(sender_contact, message.method, *message.args)
|
||||
args, kw = self._migrate_incoming_rpc_args(sender_contact, message.method, *message.args)
|
||||
log.debug("%s:%i RECV CALL %s %s:%i", self.external_ip, self.udp_port, message.method.decode(),
|
||||
sender_contact.address, sender_contact.udp_port)
|
||||
|
||||
if method == b'ping':
|
||||
result = self.node_rpc.ping()
|
||||
elif method == b'store':
|
||||
blob_hash, token, port, original_publisher_id, age = a
|
||||
blob_hash, token, port, original_publisher_id, age = args[:5]
|
||||
result = self.node_rpc.store(sender_contact, blob_hash, token, port)
|
||||
elif method == b'findNode':
|
||||
key, = a
|
||||
result = self.node_rpc.find_node(sender_contact, key)
|
||||
else:
|
||||
assert method == b'findValue'
|
||||
key, = a
|
||||
result = self.node_rpc.find_value(sender_contact, key)
|
||||
key = args[0]
|
||||
page = kw.get(PAGE_KEY, 0)
|
||||
if method == b'findNode':
|
||||
result = self.node_rpc.find_node(sender_contact, key)
|
||||
else:
|
||||
assert method == b'findValue'
|
||||
result = self.node_rpc.find_value(sender_contact, key, page)
|
||||
|
||||
self.send_response(
|
||||
sender_contact, ResponseDatagram(RESPONSE_TYPE, message.rpc_id, self.node_id, result),
|
||||
@@ -569,15 +581,18 @@ class KademliaProtocol(DatagramProtocol):
|
||||
def send_error(self, peer: 'KademliaPeer', error: ErrorDatagram):
|
||||
self._send(peer, error)
|
||||
|
||||
def _send(self, peer: 'KademliaPeer', message: typing.Union[RequestDatagram, ResponseDatagram,
|
||||
ErrorDatagram]):
|
||||
def _send(self, peer: 'KademliaPeer', message: typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]):
|
||||
if not self.transport or self.transport.is_closing():
|
||||
raise TransportNotConnected()
|
||||
|
||||
data = message.bencode()
|
||||
if len(data) > constants.msg_size_limit:
|
||||
log.exception("unexpected: %i vs %i", len(data), constants.msg_size_limit)
|
||||
raise ValueError()
|
||||
log.warning("cannot send datagram larger than %i bytes (packet is %i bytes)",
|
||||
constants.msg_size_limit, len(data))
|
||||
log.debug("Packet is too large to send: %s", binascii.hexlify(data[:3500]).decode())
|
||||
raise ValueError(
|
||||
f"cannot send datagram larger than {constants.msg_size_limit} bytes (packet is {len(data)} bytes)"
|
||||
)
|
||||
if isinstance(message, (RequestDatagram, ResponseDatagram)):
|
||||
assert message.node_id == self.node_id, message
|
||||
if isinstance(message, RequestDatagram):
|
||||
@@ -642,12 +657,12 @@ class KademliaProtocol(DatagramProtocol):
|
||||
log.debug("Timeout while storing blob_hash %s at %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
except ValueError as err:
|
||||
log.error("Unexpected response: %s" % err)
|
||||
except Exception as err:
|
||||
except RemoteException as err:
|
||||
if 'Invalid token' in str(err):
|
||||
self.peer_manager.clear_token(peer.node_id)
|
||||
try:
|
||||
return await __store()
|
||||
except:
|
||||
except (ValueError, asyncio.TimeoutError, RemoteException):
|
||||
return peer.node_id, False
|
||||
else:
|
||||
log.exception("Unexpected error while storing blob_hash")
|
||||
|
||||
@@ -63,7 +63,7 @@ def bencode(data: typing.Dict) -> bytes:
|
||||
|
||||
|
||||
def bdecode(data: bytes, allow_non_dict_return: typing.Optional[bool] = False) -> typing.Dict:
|
||||
assert type(data) == bytes, DecodeError(f"invalid data type: {str(type(data))}")
|
||||
assert isinstance(data, bytes), DecodeError(f"invalid data type: {str(type(data))}")
|
||||
|
||||
if len(data) == 0:
|
||||
raise DecodeError('Cannot decode empty string')
|
||||
|
||||
@@ -7,6 +7,9 @@ REQUEST_TYPE = 0
|
||||
RESPONSE_TYPE = 1
|
||||
ERROR_TYPE = 2
|
||||
|
||||
# bencode representation of argument keys
|
||||
PAGE_KEY = b'p'
|
||||
|
||||
|
||||
class KademliaDatagramBase:
|
||||
"""
|
||||
@@ -91,11 +94,13 @@ class RequestDatagram(KademliaDatagramBase):
|
||||
|
||||
@classmethod
|
||||
def make_find_value(cls, from_node_id: bytes, key: bytes,
|
||||
rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
|
||||
rpc_id: typing.Optional[bytes] = None, page: int = 0) -> 'RequestDatagram':
|
||||
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
|
||||
if len(key) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid key length: {len(key)}")
|
||||
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'findValue', [key])
|
||||
if page < 0:
|
||||
raise ValueError(f"cannot request a negative page ({page})")
|
||||
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'findValue', [key, {PAGE_KEY: page}])
|
||||
|
||||
|
||||
class ResponseDatagram(KademliaDatagramBase):
|
||||
|
||||
@@ -278,7 +278,7 @@ class BlobComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.blob_manager: BlobManager = None
|
||||
self.blob_manager: typing.Optional[BlobManager] = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[BlobManager]:
|
||||
@@ -294,7 +294,7 @@ class BlobComponent(Component):
|
||||
blob_dir = os.path.join(self.conf.data_dir, 'blobfiles')
|
||||
if not os.path.isdir(blob_dir):
|
||||
os.mkdir(blob_dir)
|
||||
self.blob_manager = BlobManager(asyncio.get_event_loop(), blob_dir, storage, self.conf, data_store)
|
||||
self.blob_manager = BlobManager(self.component_manager.loop, blob_dir, storage, self.conf, data_store)
|
||||
return await self.blob_manager.setup()
|
||||
|
||||
async def stop(self):
|
||||
@@ -304,7 +304,10 @@ class BlobComponent(Component):
|
||||
count = 0
|
||||
if self.blob_manager:
|
||||
count = len(self.blob_manager.completed_blob_hashes)
|
||||
return {'finished_blobs': count}
|
||||
return {
|
||||
'finished_blobs': count,
|
||||
'connections': {} if not self.blob_manager else self.blob_manager.connection_manager.status
|
||||
}
|
||||
|
||||
|
||||
class DHTComponent(Component):
|
||||
@@ -405,7 +408,7 @@ class StreamManagerComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.stream_manager: StreamManager = None
|
||||
self.stream_manager: typing.Optional[StreamManager] = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[StreamManager]:
|
||||
@@ -415,7 +418,7 @@ class StreamManagerComponent(Component):
|
||||
if not self.stream_manager:
|
||||
return
|
||||
return {
|
||||
'managed_files': len(self.stream_manager.streams)
|
||||
'managed_files': len(self.stream_manager.streams),
|
||||
}
|
||||
|
||||
async def start(self):
|
||||
@@ -444,7 +447,7 @@ class PeerProtocolServerComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.blob_server: BlobServer = None
|
||||
self.blob_server: typing.Optional[BlobServer] = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[BlobServer]:
|
||||
@@ -510,7 +513,7 @@ class UPnPComponent(Component):
|
||||
pass
|
||||
|
||||
if external_ip == "0.0.0.0" or not external_ip:
|
||||
log.warning("unable to get external ip from UPnP, checking lbry.io fallback")
|
||||
log.warning("unable to get external ip from UPnP, checking lbry.com fallback")
|
||||
external_ip = await utils.get_external_ip()
|
||||
if self.external_ip and self.external_ip != external_ip:
|
||||
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
|
||||
|
||||
@@ -485,6 +485,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
async def handle_stream_get_request(self, request: web.Request):
|
||||
if not self.conf.streaming_get:
|
||||
log.warning("streaming_get is disabled, rejecting request")
|
||||
raise web.HTTPForbidden()
|
||||
name_and_claim_id = request.path.split("/get/")[1]
|
||||
if "/" not in name_and_claim_id:
|
||||
@@ -500,6 +501,20 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
raise web.HTTPFound(f"/stream/{stream.sd_hash}")
|
||||
|
||||
async def handle_stream_range_request(self, request: web.Request):
|
||||
try:
|
||||
return await self._handle_stream_range_request(request)
|
||||
except web.HTTPException as err:
|
||||
log.warning("http code during /stream range request: %s", err)
|
||||
raise err
|
||||
except asyncio.CancelledError:
|
||||
log.debug("/stream range request cancelled")
|
||||
except Exception:
|
||||
log.exception("error handling /stream range request")
|
||||
raise
|
||||
finally:
|
||||
log.debug("finished handling /stream range request")
|
||||
|
||||
async def _handle_stream_range_request(self, request: web.Request):
|
||||
sd_hash = request.path.split("/stream/")[1]
|
||||
if not self.stream_manager.started.is_set():
|
||||
await self.stream_manager.started.wait()
|
||||
@@ -725,6 +740,16 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
},
|
||||
'blob_manager': {
|
||||
'finished_blobs': (int) number of finished blobs in the blob manager,
|
||||
'connections': {
|
||||
'incoming_bps': {
|
||||
<source ip and tcp port>: (int) bytes per second received,
|
||||
},
|
||||
'outgoing_bps': {
|
||||
<destination ip and tcp port>: (int) bytes per second sent,
|
||||
},
|
||||
'total_outgoing_mbps': (float) megabytes per second sent,
|
||||
'total_incoming_mbps': (float) megabytes per second received
|
||||
}
|
||||
},
|
||||
'hash_announcer': {
|
||||
'announce_queue_size': (int) number of blobs currently queued to be announced
|
||||
@@ -776,16 +801,14 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
Returns:
|
||||
(dict) Dictionary of lbry version information
|
||||
{
|
||||
'build': (str) build type (e.g. "dev", "rc", "release"),
|
||||
'ip': (str) remote ip, if available,
|
||||
'lbrynet_version': (str) lbrynet_version,
|
||||
'lbryum_version': (str) lbryum_version,
|
||||
'lbryschema_version': (str) lbryschema_version,
|
||||
'os_release': (str) os release string
|
||||
'os_system': (str) os name
|
||||
'platform': (str) platform string
|
||||
'processor': (str) processor type,
|
||||
'python_version': (str) python version,
|
||||
'platform': (str) platform string,
|
||||
'os_release': (str) os release string,
|
||||
'os_system': (str) os name,
|
||||
'lbrynet_version': (str) lbrynet version,
|
||||
'torba_version': (str) torba version,
|
||||
'build': (str) "dev" | "qa" | "rc" | "release",
|
||||
}
|
||||
"""
|
||||
platform_info = system_info.get_platform()
|
||||
@@ -1734,7 +1757,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
@requires(WALLET_COMPONENT, conditions=[WALLET_IS_UNLOCKED])
|
||||
async def jsonrpc_channel_create(
|
||||
self, name, bid, allow_duplicate_name=False, account_id=None, claim_address=None,
|
||||
preview=False, **kwargs):
|
||||
preview=False, blocking=False, **kwargs):
|
||||
"""
|
||||
Create a new channel by generating a channel private key and establishing an '@' prefixed claim.
|
||||
|
||||
@@ -1745,7 +1768,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
[--website_url=<website_url>] [--featured=<featured>...]
|
||||
[--tags=<tags>...] [--languages=<languages>...] [--locations=<locations>...]
|
||||
[--thumbnail_url=<thumbnail_url>] [--cover_url=<cover_url>]
|
||||
[--account_id=<account_id>] [--claim_address=<claim_address>] [--preview]
|
||||
[--account_id=<account_id>] [--claim_address=<claim_address>]
|
||||
[--preview] [--blocking]
|
||||
|
||||
Options:
|
||||
--name=<name> : (str) name of the channel prefixed with '@'
|
||||
@@ -1802,6 +1826,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--claim_address=<claim_address>: (str) address where the channel is sent to, if not specified
|
||||
it will be determined automatically from the account
|
||||
--preview : (bool) do not broadcast the transaction
|
||||
--blocking : (bool) wait until transaction is in mempool
|
||||
|
||||
Returns: {Transaction}
|
||||
"""
|
||||
@@ -1828,7 +1853,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
if not preview:
|
||||
await tx.sign([account])
|
||||
await account.ledger.broadcast(tx)
|
||||
await self.broadcast_or_release(account, tx, blocking)
|
||||
account.add_channel_private_key(txo.ref, txo.private_key)
|
||||
self.default_wallet.save()
|
||||
await self.storage.save_claims([self._old_get_temp_claim_info(
|
||||
@@ -1843,7 +1868,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
@requires(WALLET_COMPONENT, conditions=[WALLET_IS_UNLOCKED])
|
||||
async def jsonrpc_channel_update(
|
||||
self, claim_id, bid=None, account_id=None, claim_address=None,
|
||||
new_signing_key=False, preview=False, replace=False, **kwargs):
|
||||
new_signing_key=False, preview=False, blocking=False, replace=False, **kwargs):
|
||||
"""
|
||||
Update an existing channel claim.
|
||||
|
||||
@@ -1857,7 +1882,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
[--locations=<locations>...] [--clear_locations]
|
||||
[--thumbnail_url=<thumbnail_url>] [--cover_url=<cover_url>]
|
||||
[--account_id=<account_id>] [--claim_address=<claim_address>] [--new_signing_key]
|
||||
[--preview] [--replace]
|
||||
[--preview] [--blocking] [--replace]
|
||||
|
||||
Options:
|
||||
--claim_id=<claim_id> : (str) claim_id of the channel to update
|
||||
@@ -1916,6 +1941,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--claim_address=<claim_address>: (str) address where the channel is sent
|
||||
--new_signing_key : (bool) generate a new signing key, will invalidate all previous publishes
|
||||
--preview : (bool) do not broadcast the transaction
|
||||
--blocking : (bool) wait until transaction is in mempool
|
||||
--replace : (bool) instead of modifying specific values on
|
||||
the channel, this will clear all existing values
|
||||
and only save passed in values, useful for form
|
||||
@@ -1966,7 +1992,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
if not preview:
|
||||
await tx.sign([account])
|
||||
await account.ledger.broadcast(tx)
|
||||
await self.broadcast_or_release(account, tx, blocking)
|
||||
account.add_channel_private_key(new_txo.ref, new_txo.private_key)
|
||||
self.default_wallet.save()
|
||||
await self.storage.save_claims([self._old_get_temp_claim_info(
|
||||
@@ -2018,10 +2044,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
)
|
||||
|
||||
if not preview:
|
||||
await account.ledger.broadcast(tx)
|
||||
await self.broadcast_or_release(account, tx, blocking)
|
||||
await self.analytics_manager.send_claim_action('abandon')
|
||||
if blocking:
|
||||
await account.ledger.wait(tx)
|
||||
else:
|
||||
await account.ledger.release_tx(tx)
|
||||
|
||||
@@ -2103,7 +2127,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
[--release_time=<release_time>] [--width=<width>] [--height=<height>] [--duration=<duration>]
|
||||
[--channel_id=<channel_id> | --channel_name=<channel_name>]
|
||||
[--channel_account_id=<channel_account_id>...]
|
||||
[--account_id=<account_id>] [--claim_address=<claim_address>] [--preview]
|
||||
[--account_id=<account_id>] [--claim_address=<claim_address>]
|
||||
[--preview] [--blocking]
|
||||
|
||||
Options:
|
||||
--name=<name> : (str) name of the content (can only consist of a-z A-Z 0-9 and -(dash))
|
||||
@@ -2174,6 +2199,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--claim_address=<claim_address>: (str) address where the claim is sent to, if not specified
|
||||
it will be determined automatically from the account
|
||||
--preview : (bool) do not broadcast the transaction
|
||||
--blocking : (bool) wait until transaction is in mempool
|
||||
|
||||
Returns: {Transaction}
|
||||
"""
|
||||
@@ -2200,7 +2226,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
async def jsonrpc_stream_create(
|
||||
self, name, bid, file_path, allow_duplicate_name=False,
|
||||
channel_id=None, channel_name=None, channel_account_id=None,
|
||||
account_id=None, claim_address=None, preview=False, **kwargs):
|
||||
account_id=None, claim_address=None, preview=False, blocking=False,
|
||||
**kwargs):
|
||||
"""
|
||||
Make a new stream claim and announce the associated file to lbrynet.
|
||||
|
||||
@@ -2214,7 +2241,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
[--release_time=<release_time>] [--width=<width>] [--height=<height>] [--duration=<duration>]
|
||||
[--channel_id=<channel_id> | --channel_name=<channel_name>]
|
||||
[--channel_account_id=<channel_account_id>...]
|
||||
[--account_id=<account_id>] [--claim_address=<claim_address>] [--preview]
|
||||
[--account_id=<account_id>] [--claim_address=<claim_address>]
|
||||
[--preview] [--blocking]
|
||||
|
||||
Options:
|
||||
--name=<name> : (str) name of the content (can only consist of a-z A-Z 0-9 and -(dash))
|
||||
@@ -2287,6 +2315,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--claim_address=<claim_address>: (str) address where the claim is sent to, if not specified
|
||||
it will be determined automatically from the account
|
||||
--preview : (bool) do not broadcast the transaction
|
||||
--blocking : (bool) wait until transaction is in mempool
|
||||
|
||||
Returns: {Transaction}
|
||||
"""
|
||||
@@ -2323,7 +2352,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
await tx.sign([account])
|
||||
|
||||
if not preview:
|
||||
await account.ledger.broadcast(tx)
|
||||
await self.broadcast_or_release(account, tx, blocking)
|
||||
await self.storage.save_claims([self._old_get_temp_claim_info(
|
||||
tx, new_txo, claim_address, claim, name, dewies_to_lbc(amount)
|
||||
)])
|
||||
@@ -2340,7 +2369,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
self, claim_id, bid=None, file_path=None,
|
||||
channel_id=None, channel_name=None, channel_account_id=None, clear_channel=False,
|
||||
account_id=None, claim_address=None,
|
||||
preview=False, replace=False, **kwargs):
|
||||
preview=False, blocking=False, replace=False, **kwargs):
|
||||
"""
|
||||
Update an existing stream claim and if a new file is provided announce it to lbrynet.
|
||||
|
||||
@@ -2358,7 +2387,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
[--channel_id=<channel_id> | --channel_name=<channel_name> | --clear_channel]
|
||||
[--channel_account_id=<channel_account_id>...]
|
||||
[--account_id=<account_id>] [--claim_address=<claim_address>]
|
||||
[--preview] [--replace]
|
||||
[--preview] [--blocking] [--replace]
|
||||
|
||||
Options:
|
||||
--claim_id=<claim_id> : (str) id of the stream claim to update
|
||||
@@ -2437,6 +2466,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--claim_address=<claim_address>: (str) address where the claim is sent to, if not specified
|
||||
it will be determined automatically from the account
|
||||
--preview : (bool) do not broadcast the transaction
|
||||
--blocking : (bool) wait until transaction is in mempool
|
||||
--replace : (bool) instead of modifying specific values on
|
||||
the stream, this will clear all existing values
|
||||
and only save passed in values, useful for form
|
||||
@@ -2514,7 +2544,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
await tx.sign([account])
|
||||
|
||||
if not preview:
|
||||
await account.ledger.broadcast(tx)
|
||||
await self.broadcast_or_release(account, tx, blocking)
|
||||
await self.storage.save_claims([self._old_get_temp_claim_info(
|
||||
tx, new_txo, claim_address, new_txo.claim, new_txo.claim_name, dewies_to_lbc(amount)
|
||||
)])
|
||||
@@ -2529,7 +2559,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
@requires(WALLET_COMPONENT, conditions=[WALLET_IS_UNLOCKED])
|
||||
async def jsonrpc_stream_abandon(
|
||||
self, claim_id=None, txid=None, nout=None, account_id=None,
|
||||
preview=False, blocking=True):
|
||||
preview=False, blocking=False):
|
||||
"""
|
||||
Abandon one of my stream claims.
|
||||
|
||||
@@ -2566,10 +2596,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
)
|
||||
|
||||
if not preview:
|
||||
await account.ledger.broadcast(tx)
|
||||
await self.broadcast_or_release(account, tx, blocking)
|
||||
await self.analytics_manager.send_claim_action('abandon')
|
||||
if blocking:
|
||||
await account.ledger.wait(tx)
|
||||
else:
|
||||
await account.ledger.release_tx(tx)
|
||||
|
||||
@@ -2622,13 +2650,15 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
"""
|
||||
|
||||
@requires(WALLET_COMPONENT, conditions=[WALLET_IS_UNLOCKED])
|
||||
async def jsonrpc_support_create(self, claim_id, amount, tip=False, account_id=None, preview=False):
|
||||
async def jsonrpc_support_create(
|
||||
self, claim_id, amount, tip=False, account_id=None,
|
||||
preview=False, blocking=False):
|
||||
"""
|
||||
Create a support or a tip for name claim.
|
||||
|
||||
Usage:
|
||||
support_create (<claim_id> | --claim_id=<claim_id>) (<amount> | --amount=<amount>)
|
||||
[--tip] [--account_id=<account_id>] [--preview]
|
||||
[--tip] [--account_id=<account_id>] [--preview] [--blocking]
|
||||
|
||||
Options:
|
||||
--claim_id=<claim_id> : (str) claim_id of the claim to support
|
||||
@@ -2636,6 +2666,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--tip : (bool) send support to claim owner, default: false.
|
||||
--account_id=<account_id> : (str) id of the account to use
|
||||
--preview : (bool) do not broadcast the transaction
|
||||
--blocking : (bool) wait until transaction is in mempool
|
||||
|
||||
Returns: {Transaction}
|
||||
"""
|
||||
@@ -2653,7 +2684,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
if not preview:
|
||||
await tx.sign([account])
|
||||
await account.ledger.broadcast(tx)
|
||||
await self.broadcast_or_release(account, tx, blocking)
|
||||
await self.storage.save_supports({claim_id: [{
|
||||
'txid': tx.id,
|
||||
'nout': tx.position,
|
||||
@@ -2693,7 +2724,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
@requires(WALLET_COMPONENT, conditions=[WALLET_IS_UNLOCKED])
|
||||
async def jsonrpc_support_abandon(
|
||||
self, claim_id=None, txid=None, nout=None, keep=None,
|
||||
account_id=None, preview=False, blocking=True):
|
||||
account_id=None, preview=False, blocking=False):
|
||||
"""
|
||||
Abandon supports, including tips, of a specific claim, optionally
|
||||
keeping some amount as supports.
|
||||
@@ -2743,10 +2774,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
)
|
||||
|
||||
if not preview:
|
||||
await account.ledger.broadcast(tx)
|
||||
await self.broadcast_or_release(account, tx, blocking)
|
||||
await self.analytics_manager.send_claim_action('abandon')
|
||||
if blocking:
|
||||
await account.ledger.wait(tx)
|
||||
else:
|
||||
await account.ledger.release_tx(tx)
|
||||
|
||||
@@ -3387,6 +3416,15 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
f"The given claim_id ({claim_id}) may be invalid")
|
||||
return await jsonrpc_post(url, 'get_comment_data', comm_index=comment_id, better_keys=True)
|
||||
|
||||
async def broadcast_or_release(self, account, tx, blocking=False):
|
||||
try:
|
||||
await account.ledger.broadcast(tx)
|
||||
if blocking:
|
||||
await account.ledger.wait(tx)
|
||||
except:
|
||||
await account.ledger.release_tx(tx)
|
||||
raise
|
||||
|
||||
def valid_address_or_error(self, address):
|
||||
try:
|
||||
assert self.ledger.is_valid_address(address)
|
||||
|
||||
@@ -123,8 +123,8 @@ class LBRYioFeed(MarketFeed):
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
"BTCLBC",
|
||||
"lbry.io",
|
||||
"https://api.lbry.io/lbc/exchange_rate",
|
||||
"lbry.com",
|
||||
"https://api.lbry.com/lbc/exchange_rate",
|
||||
{},
|
||||
0.0,
|
||||
)
|
||||
@@ -140,8 +140,8 @@ class LBRYioBTCFeed(MarketFeed):
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
"USDBTC",
|
||||
"lbry.io",
|
||||
"https://api.lbry.io/lbc/exchange_rate",
|
||||
"lbry.com",
|
||||
"https://api.lbry.com/lbc/exchange_rate",
|
||||
{},
|
||||
0.0,
|
||||
)
|
||||
|
||||
@@ -3,6 +3,7 @@ import os
|
||||
import logging.handlers
|
||||
|
||||
from lbrynet import build_type, __version__ as lbrynet_version
|
||||
from torba import __version__ as torba_version
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -18,6 +19,7 @@ def get_platform() -> dict:
|
||||
"os_release": platform.release(),
|
||||
"os_system": os_system,
|
||||
"lbrynet_version": lbrynet_version,
|
||||
"torba_version": torba_version,
|
||||
"build": build_type.BUILD, # CI server sets this during build step
|
||||
}
|
||||
if p["os_system"] == "Linux":
|
||||
|
||||
@@ -140,8 +140,10 @@ types_map = {
|
||||
'.cbr': ('application/vnd.comicbook-rar', 'document'),
|
||||
'.cbz': ('application/vnd.comicbook+zip', 'document'),
|
||||
'.lbry': ('application/x-ext-lbry', 'document'),
|
||||
'.m4v': ('video/m4v', 'video'),
|
||||
'.mid': ('audio/midi', 'audio'),
|
||||
'.midi': ('audio/midi', 'audio'),
|
||||
'.mkv': ('video/x-matroska', 'video'),
|
||||
'.mobi': ('application/x-mobipocket-ebook', 'document'),
|
||||
'.pct': ('image/pict', 'image'),
|
||||
'.pic': ('image/pict', 'image'),
|
||||
@@ -149,8 +151,7 @@ types_map = {
|
||||
'.prc': ('application/x-mobipocket-ebook', 'document'),
|
||||
'.rtf': ('application/rtf', 'document'),
|
||||
'.xul': ('text/xul', 'document'),
|
||||
'.m4v': ('video/m4v', 'video'),
|
||||
|
||||
|
||||
# microsoft is special and has its own 'standard'
|
||||
# https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
|
||||
'.wmv': ('video/x-ms-wmv', 'video')
|
||||
|
||||
@@ -3,7 +3,7 @@ import typing
|
||||
import logging
|
||||
import binascii
|
||||
from lbrynet.error import DownloadSDTimeout
|
||||
from lbrynet.utils import resolve_host
|
||||
from lbrynet.utils import resolve_host, lru_cache_concurrent
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
from lbrynet.blob_exchange.downloader import BlobDownloader
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
@@ -36,6 +36,16 @@ class StreamDownloader:
|
||||
self.time_to_descriptor: typing.Optional[float] = None
|
||||
self.time_to_first_bytes: typing.Optional[float] = None
|
||||
|
||||
async def cached_read_blob(blob_info: 'BlobInfo') -> bytes:
|
||||
return await self.read_blob(blob_info, 2)
|
||||
|
||||
if self.blob_manager.decrypted_blob_lru_cache:
|
||||
cached_read_blob = lru_cache_concurrent(override_lru_cache=self.blob_manager.decrypted_blob_lru_cache)(
|
||||
cached_read_blob
|
||||
)
|
||||
|
||||
self.cached_read_blob = cached_read_blob
|
||||
|
||||
async def add_fixed_peers(self):
|
||||
def _delayed_add_fixed_peers():
|
||||
self.added_fixed_peers = True
|
||||
|
||||
@@ -3,7 +3,7 @@ import asyncio
|
||||
import typing
|
||||
import logging
|
||||
import binascii
|
||||
from aiohttp.web import Request, StreamResponse
|
||||
from aiohttp.web import Request, StreamResponse, HTTPRequestRangeNotSatisfiable
|
||||
from lbrynet.utils import generate_id
|
||||
from lbrynet.error import DownloadSDTimeout
|
||||
from lbrynet.schema.mime_types import guess_media_type
|
||||
@@ -42,6 +42,9 @@ class ManagedStream:
|
||||
STATUS_STOPPED = "stopped"
|
||||
STATUS_FINISHED = "finished"
|
||||
|
||||
SAVING_ID = 1
|
||||
STREAMING_ID = 2
|
||||
|
||||
__slots__ = [
|
||||
'loop',
|
||||
'config',
|
||||
@@ -214,7 +217,8 @@ class ManagedStream:
|
||||
written_bytes = None
|
||||
return {
|
||||
'streaming_url': f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash}",
|
||||
'completed': (self.output_file_exists and self.status in ('stopped', 'finished')) or all(
|
||||
'completed': (self.output_file_exists and (self.status in ('stopped', 'finished'))
|
||||
or not self.saving.is_set()) or all(
|
||||
self.blob_manager.is_blob_verified(b.blob_hash) for b in self.descriptor.blobs[:-1]),
|
||||
'file_name': file_name,
|
||||
'download_directory': download_directory,
|
||||
@@ -304,14 +308,19 @@ class ManagedStream:
|
||||
raise IndexError(start_blob_num)
|
||||
for i, blob_info in enumerate(self.descriptor.blobs[start_blob_num:-1]):
|
||||
assert i + start_blob_num == blob_info.blob_num
|
||||
decrypted = await self.downloader.read_blob(blob_info, connection_id)
|
||||
if connection_id == self.STREAMING_ID:
|
||||
decrypted = await self.downloader.cached_read_blob(blob_info)
|
||||
else:
|
||||
decrypted = await self.downloader.read_blob(blob_info, connection_id)
|
||||
yield (blob_info, decrypted)
|
||||
|
||||
async def stream_file(self, request: Request, node: typing.Optional['Node'] = None) -> StreamResponse:
|
||||
log.info("stream file to browser for lbry://%s#%s (sd hash %s...)", self.claim_name, self.claim_id,
|
||||
self.sd_hash[:6])
|
||||
headers, size, skip_blobs, first_blob_start_offset = self._prepare_range_response_headers(
|
||||
request.headers.get('range', 'bytes=0-')
|
||||
)
|
||||
await self.start(node)
|
||||
headers, size, skip_blobs = self._prepare_range_response_headers(request.headers.get('range', 'bytes=0-'))
|
||||
response = StreamResponse(
|
||||
status=206,
|
||||
headers=headers
|
||||
@@ -321,11 +330,16 @@ class ManagedStream:
|
||||
self.streaming.set()
|
||||
try:
|
||||
wrote = 0
|
||||
async for blob_info, decrypted in self._aiter_read_stream(skip_blobs, connection_id=2):
|
||||
async for blob_info, decrypted in self._aiter_read_stream(skip_blobs, connection_id=self.STREAMING_ID):
|
||||
if not wrote:
|
||||
decrypted = decrypted[first_blob_start_offset:]
|
||||
if (blob_info.blob_num == len(self.descriptor.blobs) - 2) or (len(decrypted) + wrote >= size):
|
||||
decrypted += (b'\x00' * (size - len(decrypted) - wrote - (skip_blobs * 2097151)))
|
||||
log.debug("sending browser final blob (%i/%i)", blob_info.blob_num + 1,
|
||||
len(self.descriptor.blobs) - 1)
|
||||
await response.write_eof(decrypted)
|
||||
else:
|
||||
log.debug("sending browser blob (%i/%i)", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
|
||||
await response.write(decrypted)
|
||||
wrote += len(decrypted)
|
||||
log.info("sent browser %sblob %i/%i", "(final) " if response._eof_sent else "",
|
||||
@@ -354,7 +368,7 @@ class ManagedStream:
|
||||
self.started_writing.clear()
|
||||
try:
|
||||
with open(output_path, 'wb') as file_write_handle:
|
||||
async for blob_info, decrypted in self._aiter_read_stream(connection_id=1):
|
||||
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
|
||||
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
|
||||
await self.loop.run_in_executor(None, self._write_decrypted_blob, file_write_handle, decrypted)
|
||||
self.written_bytes += len(decrypted)
|
||||
@@ -485,7 +499,7 @@ class ManagedStream:
|
||||
return
|
||||
await asyncio.sleep(1, loop=self.loop)
|
||||
|
||||
def _prepare_range_response_headers(self, get_range: str) -> typing.Tuple[typing.Dict[str, str], int, int]:
|
||||
def _prepare_range_response_headers(self, get_range: str) -> typing.Tuple[typing.Dict[str, str], int, int, int]:
|
||||
if '=' in get_range:
|
||||
get_range = get_range.split('=')[1]
|
||||
start, end = get_range.split('-')
|
||||
@@ -503,16 +517,23 @@ class ManagedStream:
|
||||
log.debug("estimating stream size")
|
||||
|
||||
start = int(start)
|
||||
if not 0 <= start < size:
|
||||
raise HTTPRequestRangeNotSatisfiable()
|
||||
|
||||
end = int(end) if end else size - 1
|
||||
|
||||
if end >= size:
|
||||
raise HTTPRequestRangeNotSatisfiable()
|
||||
|
||||
skip_blobs = start // 2097150
|
||||
skip = skip_blobs * 2097151
|
||||
start = skip
|
||||
skip_first_blob = start - skip
|
||||
start = skip_first_blob + skip
|
||||
final_size = end - start + 1
|
||||
|
||||
headers = {
|
||||
'Accept-Ranges': 'bytes',
|
||||
'Content-Range': f'bytes {start}-{end}/{size}',
|
||||
'Content-Length': str(final_size),
|
||||
'Content-Type': self.mime_type
|
||||
}
|
||||
return headers, size, skip_blobs
|
||||
return headers, size, skip_blobs, skip_first_blob
|
||||
@@ -10,13 +10,15 @@ if typing.TYPE_CHECKING:
|
||||
REFLECTOR_V1 = 0
|
||||
REFLECTOR_V2 = 1
|
||||
|
||||
MAX_RESPONSE_SIZE = 2000000
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StreamReflectorClient(asyncio.Protocol):
|
||||
def __init__(self, blob_manager: 'BlobManager', descriptor: 'StreamDescriptor'):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.transport: asyncio.StreamWriter = None
|
||||
self.transport: typing.Optional[asyncio.WriteTransport] = None
|
||||
self.blob_manager = blob_manager
|
||||
self.descriptor = descriptor
|
||||
self.response_buff = b''
|
||||
@@ -37,11 +39,21 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
log.info("Finished sending reflector %i blobs", len(self.reflected_blobs))
|
||||
|
||||
def data_received(self, data):
|
||||
if len(self.response_buff + (data or b'')) > MAX_RESPONSE_SIZE:
|
||||
log.warning("response message to large from reflector server: %i bytes",
|
||||
len(self.response_buff + (data or b'')))
|
||||
self.response_buff = b''
|
||||
self.transport.close()
|
||||
return
|
||||
self.response_buff += (data or b'')
|
||||
try:
|
||||
response = json.loads(data.decode())
|
||||
response = json.loads(self.response_buff.decode())
|
||||
self.response_buff = b''
|
||||
self.response_queue.put_nowait(response)
|
||||
except ValueError:
|
||||
self.transport.close()
|
||||
if not data:
|
||||
log.warning("got undecodable response from reflector server")
|
||||
self.response_buff = b''
|
||||
return
|
||||
|
||||
async def send_request(self, request_dict: typing.Dict):
|
||||
|
||||
@@ -15,7 +15,7 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ReflectorServerProtocol(asyncio.Protocol):
|
||||
def __init__(self, blob_manager: 'BlobManager'):
|
||||
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
@@ -28,6 +28,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
self.sd_blob: typing.Optional['BlobFile'] = None
|
||||
self.received = []
|
||||
self.incoming = asyncio.Event(loop=self.loop)
|
||||
self.chunk_size = response_chunk_size
|
||||
|
||||
def connection_made(self, transport):
|
||||
self.transport = transport
|
||||
@@ -47,7 +48,15 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
self.loop.create_task(self.handle_request(request))
|
||||
|
||||
def send_response(self, response: typing.Dict):
|
||||
self.transport.write(json.dumps(response).encode())
|
||||
def chunk_response(remaining: bytes):
|
||||
f = self.loop.create_future()
|
||||
f.add_done_callback(lambda _: self.transport.write(remaining[:self.chunk_size]))
|
||||
if len(remaining) > self.chunk_size:
|
||||
f.add_done_callback(lambda _: self.loop.call_soon(chunk_response, remaining[self.chunk_size:]))
|
||||
self.loop.call_soon(f.set_result, None)
|
||||
|
||||
response_bytes = json.dumps(response).encode()
|
||||
chunk_response(response_bytes)
|
||||
|
||||
async def handle_request(self, request: typing.Dict):
|
||||
if self.client_version is None:
|
||||
@@ -121,11 +130,12 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
|
||||
|
||||
class ReflectorServer:
|
||||
def __init__(self, blob_manager: 'BlobManager'):
|
||||
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
self.server_task: typing.Optional[asyncio.Task] = None
|
||||
self.started_listening = asyncio.Event(loop=self.loop)
|
||||
self.response_chunk_size = response_chunk_size
|
||||
|
||||
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
|
||||
if self.server_task is not None:
|
||||
@@ -133,7 +143,7 @@ class ReflectorServer:
|
||||
|
||||
async def _start_server():
|
||||
server = await self.loop.create_server(
|
||||
lambda: ReflectorServerProtocol(self.blob_manager),
|
||||
lambda: ReflectorServerProtocol(self.blob_manager, self.response_chunk_size),
|
||||
interface, port
|
||||
)
|
||||
self.started_listening.set()
|
||||
|
||||
@@ -62,6 +62,7 @@ class CommandTestCase(IntegrationTestCase):
|
||||
LEDGER = lbrynet.wallet
|
||||
MANAGER = LbryWalletManager
|
||||
VERBOSITY = logging.WARN
|
||||
blob_lru_cache_size = 0
|
||||
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
@@ -81,6 +82,7 @@ class CommandTestCase(IntegrationTestCase):
|
||||
conf.lbryum_servers = [('127.0.0.1', 50001)]
|
||||
conf.reflector_servers = [('127.0.0.1', 5566)]
|
||||
conf.known_dht_nodes = []
|
||||
conf.blob_lru_cache_size = self.blob_lru_cache_size
|
||||
|
||||
await self.account.ensure_address_gap()
|
||||
address = (await self.account.receiving.get_addresses(limit=1, only_usable=True))[0]
|
||||
|
||||
+7
-6
@@ -75,7 +75,7 @@ def obfuscate(plain):
|
||||
return rot13(base64.b64encode(plain).decode())
|
||||
|
||||
|
||||
def check_connection(server="lbry.io", port=80, timeout=5) -> bool:
|
||||
def check_connection(server="lbry.com", port=80, timeout=5) -> bool:
|
||||
"""Attempts to open a socket to server:port and returns True if successful."""
|
||||
log.debug('Checking connection to %s:%s', server, port)
|
||||
try:
|
||||
@@ -102,7 +102,7 @@ def check_connection(server="lbry.io", port=80, timeout=5) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
async def async_check_connection(server="lbry.io", port=80, timeout=5) -> bool:
|
||||
async def async_check_connection(server="lbry.com", port=80, timeout=5) -> bool:
|
||||
return await asyncio.get_event_loop().run_in_executor(None, check_connection, server, port, timeout)
|
||||
|
||||
|
||||
@@ -229,11 +229,12 @@ class LRUCache:
|
||||
return item in self.cache
|
||||
|
||||
|
||||
def lru_cache_concurrent(cache_size: int):
|
||||
if not cache_size > 0:
|
||||
def lru_cache_concurrent(cache_size: typing.Optional[int] = None,
|
||||
override_lru_cache: typing.Optional[LRUCache] = None):
|
||||
if not cache_size and override_lru_cache is None:
|
||||
raise ValueError("invalid cache size")
|
||||
concurrent_cache = {}
|
||||
lru_cache = LRUCache(cache_size)
|
||||
lru_cache = override_lru_cache or LRUCache(cache_size)
|
||||
|
||||
def wrapper(async_fn):
|
||||
|
||||
@@ -270,7 +271,7 @@ async def aiohttp_request(method, url, **kwargs) -> typing.AsyncContextManager[a
|
||||
|
||||
async def get_external_ip() -> typing.Optional[str]: # used if upnp is disabled or non-functioning
|
||||
try:
|
||||
async with aiohttp_request("get", "https://api.lbry.io/ip") as resp:
|
||||
async with aiohttp_request("get", "https://api.lbry.com/ip") as resp:
|
||||
response = await resp.json()
|
||||
if response['success']:
|
||||
return response['data']['ip']
|
||||
|
||||
@@ -46,8 +46,6 @@ class LBRYElectrumX(ElectrumX):
|
||||
handlers.update({
|
||||
'blockchain.numblocks.subscribe': self.numblocks_subscribe,
|
||||
'blockchain.utxo.get_address': self.utxo_get_address,
|
||||
'blockchain.transaction.broadcast':
|
||||
self.transaction_broadcast_1_0,
|
||||
'blockchain.transaction.get': self.transaction_get,
|
||||
})
|
||||
self.request_handlers.update(handlers)
|
||||
|
||||
+1
-1
@@ -24,6 +24,6 @@ theme:
|
||||
- type: 'telegram'
|
||||
link: 'https://t.me/lbryofficial'
|
||||
- type: 'discord'
|
||||
link: 'https://chat.lbry.io'
|
||||
link: 'https://chat.lbry.com'
|
||||
- type: 'reddit'
|
||||
link: 'https://reddit.com/r/lbry'
|
||||
@@ -1,5 +1,7 @@
|
||||
FROM debian:buster-slim
|
||||
|
||||
ARG TORBA_VERSION=master
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get upgrade -y && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
@@ -15,7 +17,7 @@ RUN python3.7 -m pip install --upgrade pip setuptools wheel
|
||||
COPY . /app
|
||||
WORKDIR /app
|
||||
|
||||
RUN python3.7 -m pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
RUN python3.7 -m pip install --user git+https://github.com/lbryio/torba.git@${TORBA_VERSION}#egg=torba
|
||||
RUN python3.7 -m pip install -e .
|
||||
|
||||
# Orchstr8 API
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
FROM debian:buster-slim
|
||||
|
||||
ARG user=lbry
|
||||
ARG TORBA_VERSION=master
|
||||
|
||||
# create an unprivileged user
|
||||
RUN groupadd -g 999 $user && useradd -r -u 999 -g $user $user
|
||||
@@ -23,7 +24,7 @@ WORKDIR /home/$user
|
||||
RUN python3.7 -m pip install --upgrade pip setuptools
|
||||
|
||||
# get torba
|
||||
RUN python3.7 -m pip install --user git+https://github.com/lbryio/torba.git#egg=torba
|
||||
RUN python3.7 -m pip install --user git+https://github.com/lbryio/torba.git@${TORBA_VERSION}#egg=torba
|
||||
|
||||
# get uvloop
|
||||
RUN python3.7 -m pip install --user uvloop
|
||||
|
||||
@@ -47,7 +47,7 @@ async def main(blob_hash: str, url: str):
|
||||
|
||||
|
||||
if __name__ == "__main__": # usage: python download_blob_from_peer.py <blob_hash> [host url:port]
|
||||
url = 'reflector.lbry.io:5567'
|
||||
url = 'reflector.lbry.com:5567'
|
||||
if len(sys.argv) > 2:
|
||||
url = sys.argv[2]
|
||||
asyncio.run(main(sys.argv[1], url))
|
||||
+26
-24
@@ -1,33 +1,35 @@
|
||||
"""Set the build version to be 'qa', 'rc', 'release'"""
|
||||
|
||||
"""Set the build version to be 'dev', 'qa', 'rc', 'release'"""
|
||||
|
||||
import os.path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import os
|
||||
import re
|
||||
import logging
|
||||
|
||||
log = logging.getLogger()
|
||||
log.addHandler(logging.StreamHandler())
|
||||
log.setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
def get_build_type(travis_tag=None):
|
||||
if not travis_tag:
|
||||
return "qa"
|
||||
log.debug("getting build type for tag: \"%s\"", travis_tag)
|
||||
if re.match('v\d+\.\d+\.\d+rc\d+$', travis_tag):
|
||||
return 'rc'
|
||||
elif re.match('v\d+\.\d+\.\d+$', travis_tag):
|
||||
return 'release'
|
||||
return 'qa'
|
||||
|
||||
|
||||
def main():
|
||||
build = get_build()
|
||||
root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||
with open(os.path.join(root_dir, 'lbrynet', 'build_type.py'), 'w') as f:
|
||||
f.write("BUILD = '{}'\n".format(build))
|
||||
|
||||
|
||||
def get_build():
|
||||
try:
|
||||
tag = subprocess.check_output(['git', 'describe', '--exact-match', '--all']).strip()
|
||||
if re.match('tags\/v\d+\.\d+\.\d+rc\d+$', tag.decode()):
|
||||
print('Build: rc')
|
||||
return 'rc'
|
||||
elif re.match('tags\/v\d+\.\d+\.\d+$', tag.decode()):
|
||||
print('Build: release')
|
||||
return 'release'
|
||||
print('Build: qa')
|
||||
return 'qa'
|
||||
except subprocess.CalledProcessError:
|
||||
print("Couldn't determine build type, defaulting to qa.")
|
||||
return 'qa'
|
||||
build_type_path = os.path.join(root_dir, 'lbrynet', 'build_type.py')
|
||||
log.debug("configuring build type file: %s", build_type_path)
|
||||
travis_commit = os.environ['TRAVIS_COMMIT'][:6]
|
||||
build_type = get_build_type(os.environ.get('TRAVIS_TAG', None))
|
||||
log.debug("setting build type=%s, build commit=%s", build_type, travis_commit)
|
||||
with open(build_type_path, 'w') as f:
|
||||
f.write("BUILD = \"{}\"\nBUILD_COMMIT = \"{}\"\n".format(build_type, travis_commit))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@@ -26,7 +26,7 @@ def extract_uris(response):
|
||||
async def get_frontpage_uris():
|
||||
session = aiohttp.ClientSession()
|
||||
try:
|
||||
response = await session.get("https://api.lbry.io/file/list_homepage", timeout=10.0)
|
||||
response = await session.get("https://api.lbry.com/file/list_homepage", timeout=10.0)
|
||||
if response.status != 200:
|
||||
print("API returned non 200 code!!")
|
||||
return
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
set -x
|
||||
|
||||
# must also be updated in travis.yml
|
||||
TORBA_VERSION=v0.5.4a0
|
||||
rm -rf /tmp/.wine-*
|
||||
|
||||
apt-get -qq update
|
||||
apt-get -qq install -y git
|
||||
|
||||
pip install setuptools_scm
|
||||
git clone https://github.com/lbryio/torba.git --depth 1
|
||||
cd torba && pip install -e . && cd ..
|
||||
|
||||
cd lbry
|
||||
|
||||
@@ -15,6 +14,13 @@ cd lbry
|
||||
wget -Onetifaces-0.10.7-cp37-cp37m-win32.whl https://ci.appveyor.com/api/buildjobs/6hworunifsymrhp2/artifacts/dist%2Fnetifaces-0.10.7-cp37-cp37m-win32.whl
|
||||
pip install netifaces-0.10.7-cp37-cp37m-win32.whl
|
||||
|
||||
git clone --depth=1 --single-branch --branch ${TORBA_VERSION} https://github.com/lbryio/torba.git
|
||||
cd torba
|
||||
pip install .
|
||||
cd ..
|
||||
rm -rf torba
|
||||
|
||||
pip show torba
|
||||
pip install -e .
|
||||
pip install pywin32
|
||||
|
||||
|
||||
@@ -9,8 +9,8 @@ setup(
|
||||
name=__name__,
|
||||
version=__version__,
|
||||
author="LBRY Inc.",
|
||||
author_email="hello@lbry.io",
|
||||
url="https://lbry.io",
|
||||
author_email="hello@lbry.com",
|
||||
url="https://lbry.com",
|
||||
description="A decentralized media library and marketplace",
|
||||
long_description=open(README_PATH, encoding='utf-8').read(),
|
||||
long_description_content_type="text/markdown",
|
||||
@@ -23,9 +23,9 @@ setup(
|
||||
'console_scripts': 'lbrynet=lbrynet.extras.cli:main'
|
||||
},
|
||||
install_requires=[
|
||||
'torba',
|
||||
'torba==0.5.4a0',
|
||||
'aiohttp==3.5.4',
|
||||
'aioupnp',
|
||||
'aioupnp==0.0.12',
|
||||
'appdirs==1.4.3',
|
||||
'certifi>=2018.11.29',
|
||||
'colorama==0.3.7',
|
||||
|
||||
@@ -2,6 +2,7 @@ import os
|
||||
import hashlib
|
||||
import aiohttp
|
||||
import aiohttp.web
|
||||
import asyncio
|
||||
|
||||
from lbrynet.utils import aiohttp_request
|
||||
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
|
||||
@@ -373,3 +374,46 @@ class RangeRequests(CommandTestCase):
|
||||
await stream.finished_writing.wait()
|
||||
with open(stream.full_path, 'rb') as f:
|
||||
self.assertEqual(self.data, f.read())
|
||||
|
||||
|
||||
class RangeRequestsLRUCache(CommandTestCase):
|
||||
blob_lru_cache_size = 32
|
||||
|
||||
async def _request_stream(self):
|
||||
name = 'foo'
|
||||
url = f'http://{self.daemon.conf.streaming_host}:{self.daemon.conf.streaming_port}/get/{name}'
|
||||
|
||||
async with aiohttp_request('get', url) as req:
|
||||
self.assertEqual(req.headers.get('Content-Type'), 'application/octet-stream')
|
||||
content_range = req.headers.get('Content-Range')
|
||||
content_length = int(req.headers.get('Content-Length'))
|
||||
streamed_bytes = await req.content.read()
|
||||
self.assertEqual(content_length, len(streamed_bytes))
|
||||
self.assertEqual(15, content_length)
|
||||
self.assertEqual(b'hi\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', streamed_bytes)
|
||||
self.assertEqual('bytes 0-14/15', content_range)
|
||||
|
||||
async def test_range_requests_with_blob_lru_cache(self):
|
||||
self.data = b'hi'
|
||||
self.daemon.conf.save_blobs = False
|
||||
self.daemon.conf.save_files = False
|
||||
await self.stream_create('foo', '0.01', data=self.data, file_size=0)
|
||||
await self.daemon.jsonrpc_file_list()[0].fully_reflected.wait()
|
||||
await self.daemon.jsonrpc_file_delete(delete_from_download_dir=True, claim_name='foo')
|
||||
self.assertEqual(0, len(os.listdir(self.daemon.blob_manager.blob_dir)))
|
||||
|
||||
await self.daemon.streaming_runner.setup()
|
||||
site = aiohttp.web.TCPSite(self.daemon.streaming_runner, self.daemon.conf.streaming_host,
|
||||
self.daemon.conf.streaming_port)
|
||||
await site.start()
|
||||
self.assertListEqual(self.daemon.jsonrpc_file_list(), [])
|
||||
|
||||
await self._request_stream()
|
||||
self.assertEqual(1, len(self.daemon.jsonrpc_file_list()))
|
||||
self.server.stop_server()
|
||||
|
||||
# running with cache size 0 gets through without errors without
|
||||
# this since the server doesnt stop immediately
|
||||
await asyncio.sleep(1, loop=self.loop)
|
||||
|
||||
await self._request_stream()
|
||||
@@ -83,12 +83,15 @@ class TestProtocol(AsyncioTestCase):
|
||||
find_value_response = peer1.node_rpc.find_value(peer3, b'2' * 48)
|
||||
self.assertEqual(len(find_value_response[b'contacts']), 0)
|
||||
self.assertSetEqual(
|
||||
{b'2' * 48, b'token', b'protocolVersion', b'contacts'}, set(find_value_response.keys())
|
||||
{b'2' * 48, b'token', b'protocolVersion', b'contacts', b'p'}, set(find_value_response.keys())
|
||||
)
|
||||
self.assertEqual(2, len(find_value_response[b'2' * 48]))
|
||||
self.assertEqual(find_value_response[b'2' * 48][0], peer2_from_peer1.compact_address_tcp())
|
||||
self.assertDictEqual(bdecode(bencode(find_value_response)), find_value_response)
|
||||
|
||||
find_value_page_above_pages_response = peer1.node_rpc.find_value(peer3, b'2' * 48, page=10)
|
||||
self.assertNotIn(b'2' * 48, find_value_page_above_pages_response)
|
||||
|
||||
peer1.stop()
|
||||
peer2.stop()
|
||||
peer1.disconnect()
|
||||
|
||||
@@ -59,17 +59,28 @@ class TestDatagram(unittest.TestCase):
|
||||
self.assertRaises(ValueError, RequestDatagram.make_find_value, b'1' * 49, b'2' * 48, b'1' * 20)
|
||||
self.assertRaises(ValueError, RequestDatagram.make_find_value, b'1' * 48, b'2' * 49, b'1' * 20)
|
||||
self.assertRaises(ValueError, RequestDatagram.make_find_value, b'1' * 48, b'2' * 48, b'1' * 21)
|
||||
self.assertRaises(ValueError, RequestDatagram.make_find_value, b'1' * 48, b'2' * 48, b'1' * 20, -1)
|
||||
self.assertEqual(20, len(RequestDatagram.make_find_value(b'1' * 48, b'2' * 48).rpc_id))
|
||||
|
||||
# default page argument
|
||||
serialized = RequestDatagram.make_find_value(b'1' * 48, b'2' * 48, b'1' * 20).bencode()
|
||||
decoded = decode_datagram(serialized)
|
||||
self.assertEqual(decoded.packet_type, REQUEST_TYPE)
|
||||
self.assertEqual(decoded.rpc_id, b'1' * 20)
|
||||
self.assertEqual(decoded.node_id, b'1' * 48)
|
||||
self.assertEqual(decoded.method, b'findValue')
|
||||
self.assertListEqual(decoded.args, [b'2' * 48, {b'protocolVersion': 1}])
|
||||
self.assertListEqual(decoded.args, [b'2' * 48, {b'protocolVersion': 1, b'p': 0}])
|
||||
|
||||
def test_find_value_response(self):
|
||||
# nondefault page argument
|
||||
serialized = RequestDatagram.make_find_value(b'1' * 48, b'2' * 48, b'1' * 20, 1).bencode()
|
||||
decoded = decode_datagram(serialized)
|
||||
self.assertEqual(decoded.packet_type, REQUEST_TYPE)
|
||||
self.assertEqual(decoded.rpc_id, b'1' * 20)
|
||||
self.assertEqual(decoded.node_id, b'1' * 48)
|
||||
self.assertEqual(decoded.method, b'findValue')
|
||||
self.assertListEqual(decoded.args, [b'2' * 48, {b'protocolVersion': 1, b'p': 1}])
|
||||
|
||||
def test_find_value_response_without_pages_field(self):
|
||||
found_value_response = {b'2' * 48: [b'\x7f\x00\x00\x01']}
|
||||
serialized = ResponseDatagram(RESPONSE_TYPE, b'1' * 20, b'1' * 48, found_value_response).bencode()
|
||||
decoded = decode_datagram(serialized)
|
||||
@@ -78,6 +89,15 @@ class TestDatagram(unittest.TestCase):
|
||||
self.assertEqual(decoded.node_id, b'1' * 48)
|
||||
self.assertDictEqual(decoded.response, found_value_response)
|
||||
|
||||
def test_find_value_response_with_pages_field(self):
|
||||
found_value_response = {b'2' * 48: [b'\x7f\x00\x00\x01'], b'p': 1}
|
||||
serialized = ResponseDatagram(RESPONSE_TYPE, b'1' * 20, b'1' * 48, found_value_response).bencode()
|
||||
decoded = decode_datagram(serialized)
|
||||
self.assertEqual(decoded.packet_type, RESPONSE_TYPE)
|
||||
self.assertEqual(decoded.rpc_id, b'1' * 20)
|
||||
self.assertEqual(decoded.node_id, b'1' * 48)
|
||||
self.assertDictEqual(decoded.response, found_value_response)
|
||||
|
||||
def test_store_request(self):
|
||||
self.assertRaises(ValueError, RequestDatagram.make_store, b'1' * 47, b'2' * 48, b'3' * 48, 3333, b'1' * 20)
|
||||
self.assertRaises(ValueError, RequestDatagram.make_store, b'1' * 48, b'2' * 49, b'3' * 48, 3333, b'1' * 20)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import contextlib
|
||||
import typing
|
||||
import binascii
|
||||
import socket
|
||||
import asyncio
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from tests import dht_mocks
|
||||
@@ -26,6 +27,7 @@ class TestBlobAnnouncer(AsyncioTestCase):
|
||||
for node_id, address in peer_addresses:
|
||||
await self.add_peer(node_id, address)
|
||||
self.node.joined.set()
|
||||
self.node._refresh_task = self.loop.create_task(self.node.refresh_node())
|
||||
|
||||
async def add_peer(self, node_id, address, add_to_routing_table=True):
|
||||
n = Node(self.loop, PeerManager(self.loop), node_id, 4444, 4444, 3333, address)
|
||||
@@ -113,3 +115,56 @@ class TestBlobAnnouncer(AsyncioTestCase):
|
||||
self.assertEqual(self.node.protocol.node_id, found_peers[0].node_id)
|
||||
self.assertEqual(self.node.protocol.external_ip, found_peers[0].address)
|
||||
self.assertEqual(self.node.protocol.peer_port, found_peers[0].tcp_port)
|
||||
|
||||
async def test_popular_blob(self):
|
||||
peer_count = 150
|
||||
addresses = [
|
||||
(constants.generate_id(i + 1), socket.inet_ntoa(int(i + 1).to_bytes(length=4, byteorder='big')))
|
||||
for i in range(peer_count)
|
||||
]
|
||||
blob_hash = b'1' * 48
|
||||
|
||||
async with self._test_network_context(peer_addresses=addresses):
|
||||
total_seen = set()
|
||||
announced_to = self.nodes[0]
|
||||
for i in range(1, peer_count):
|
||||
node = self.nodes[i]
|
||||
kad_peer = announced_to.protocol.peer_manager.get_kademlia_peer(
|
||||
node.protocol.node_id, node.protocol.external_ip, node.protocol.udp_port
|
||||
)
|
||||
await announced_to.protocol._add_peer(kad_peer)
|
||||
peer = node.protocol.get_rpc_peer(
|
||||
node.protocol.peer_manager.get_kademlia_peer(
|
||||
announced_to.protocol.node_id,
|
||||
announced_to.protocol.external_ip,
|
||||
announced_to.protocol.udp_port
|
||||
)
|
||||
)
|
||||
response = await peer.store(blob_hash)
|
||||
self.assertEqual(response, b'OK')
|
||||
peers_for_blob = await peer.find_value(blob_hash, 0)
|
||||
if i == 1:
|
||||
self.assertTrue(blob_hash not in peers_for_blob)
|
||||
self.assertEqual(peers_for_blob[b'p'], 0)
|
||||
else:
|
||||
self.assertEqual(len(peers_for_blob[blob_hash]), min(i - 1, constants.k))
|
||||
self.assertEqual(len(announced_to.protocol.data_store.get_peers_for_blob(blob_hash)), i)
|
||||
if i - 1 > constants.k:
|
||||
self.assertEqual(len(peers_for_blob[b'contacts']), constants.k)
|
||||
self.assertEqual(peers_for_blob[b'p'], ((i - 1) // (constants.k + 1)) + 1)
|
||||
seen = set(peers_for_blob[blob_hash])
|
||||
self.assertEqual(len(seen), constants.k)
|
||||
self.assertEqual(len(peers_for_blob[blob_hash]), len(seen))
|
||||
|
||||
for pg in range(1, peers_for_blob[b'p']):
|
||||
page_x = await peer.find_value(blob_hash, pg)
|
||||
self.assertNotIn(b'contacts', page_x)
|
||||
page_x_set = set(page_x[blob_hash])
|
||||
self.assertEqual(len(page_x[blob_hash]), len(page_x_set))
|
||||
self.assertTrue(len(page_x_set) > 0)
|
||||
self.assertSetEqual(seen.intersection(page_x_set), set())
|
||||
seen.intersection_update(page_x_set)
|
||||
total_seen.update(page_x_set)
|
||||
else:
|
||||
self.assertEqual(len(peers_for_blob[b'contacts']), i - 1)
|
||||
self.assertEqual(len(total_seen), peer_count - 2)
|
||||
@@ -41,8 +41,8 @@ class TestStreamAssembler(AsyncioTestCase):
|
||||
|
||||
self.stream = await self.stream_manager.create_stream(file_path)
|
||||
|
||||
async def test_reflect_stream(self):
|
||||
reflector = ReflectorServer(self.server_blob_manager)
|
||||
async def _test_reflect_stream(self, response_chunk_size):
|
||||
reflector = ReflectorServer(self.server_blob_manager, response_chunk_size=response_chunk_size)
|
||||
reflector.start_server(5566, '127.0.0.1')
|
||||
await reflector.started_listening.wait()
|
||||
self.addCleanup(reflector.stop_server)
|
||||
@@ -63,6 +63,12 @@ class TestStreamAssembler(AsyncioTestCase):
|
||||
sent = await self.stream.upload_to_reflector('127.0.0.1', 5566)
|
||||
self.assertListEqual(sent, [])
|
||||
|
||||
async def test_reflect_stream(self):
|
||||
return await asyncio.wait_for(self._test_reflect_stream(response_chunk_size=50), 3, loop=self.loop)
|
||||
|
||||
async def test_reflect_stream_small_response_chunks(self):
|
||||
return await asyncio.wait_for(self._test_reflect_stream(response_chunk_size=30), 3, loop=self.loop)
|
||||
|
||||
async def test_announces(self):
|
||||
to_announce = await self.storage.get_blobs_to_announce()
|
||||
self.assertIn(self.stream.sd_hash, to_announce, "sd blob not set to announce")
|
||||
|
||||
Reference in new issue
Block a user