Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e2634974e7 | ||
|
|
9ed985d6ec | ||
|
|
2d898f802d | ||
|
|
99c64021eb | ||
|
|
366b0d590c | ||
|
|
53e683d307 | ||
|
|
411698f306 |
No files matched your search
@@ -1,4 +1,4 @@
|
||||
FROM ubuntu:20.04
|
||||
FROM ubuntu:19.10
|
||||
|
||||
ARG user=lbry
|
||||
ARG db_dir=/database
|
||||
|
||||
+1
-1
@@ -1,2 +1,2 @@
|
||||
__version__ = "0.88.0"
|
||||
__version__ = "0.87.0"
|
||||
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
|
||||
@@ -2,7 +2,7 @@ import os
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
from lbry.utils import LRUCacheWithMetrics
|
||||
from lbry.utils import LRUCache
|
||||
from lbry.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
|
||||
from lbry.stream.descriptor import StreamDescriptor
|
||||
from lbry.connection_manager import ConnectionManager
|
||||
@@ -32,7 +32,7 @@ class BlobManager:
|
||||
else self._node_data_store.completed_blobs
|
||||
self.blobs: typing.Dict[str, AbstractBlob] = {}
|
||||
self.config = config
|
||||
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCacheWithMetrics(
|
||||
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCache(
|
||||
self.config.blob_lru_cache_size)
|
||||
self.connection_manager = ConnectionManager(loop)
|
||||
|
||||
|
||||
+40
-7
@@ -3,6 +3,7 @@ import asyncio
|
||||
import typing
|
||||
import binascii
|
||||
import socket
|
||||
import time
|
||||
from lbry.utils import resolve_host
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.peer import make_kademlia_peer
|
||||
@@ -17,6 +18,16 @@ if typing.TYPE_CHECKING:
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class NodeState:
|
||||
def __init__(self,
|
||||
routing_table_peers: typing.List[typing.Tuple[bytes, str, int, int]],
|
||||
datastore: typing.List[typing.Tuple[bytes, str, int, int, bytes, float]]):
|
||||
# List[Tuple[node_id, address, udp_port, tcp_port]]
|
||||
self.routing_table_peers = routing_table_peers
|
||||
# List[Tuple[node_id, address, udp_port, tcp_port, blob_hash, added_at]]
|
||||
self.datastore = datastore
|
||||
|
||||
|
||||
class Node:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int,
|
||||
internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: float = constants.RPC_TIMEOUT,
|
||||
@@ -31,6 +42,7 @@ class Node:
|
||||
self._join_task: asyncio.Task = None
|
||||
self._refresh_task: asyncio.Task = None
|
||||
self._storage = storage
|
||||
self.started_listening = asyncio.Event()
|
||||
|
||||
async def refresh_node(self, force_once=False):
|
||||
while True:
|
||||
@@ -103,6 +115,7 @@ class Node:
|
||||
return stored_to
|
||||
|
||||
def stop(self) -> None:
|
||||
self.started_listening.clear()
|
||||
if self.joined.is_set():
|
||||
self.joined.clear()
|
||||
if self._join_task:
|
||||
@@ -118,18 +131,35 @@ class Node:
|
||||
self.listening_port = None
|
||||
log.info("Stopped DHT node")
|
||||
|
||||
def get_state(self) -> NodeState:
|
||||
return NodeState(
|
||||
routing_table_peers=[(p.node_id, p.address, p.udp_port, p.tcp_port)
|
||||
for p in self.protocol.routing_table.get_peers()],
|
||||
datastore=self.protocol.data_store.dump()
|
||||
)
|
||||
|
||||
def load_state(self, state: NodeState):
|
||||
now = self.loop.time()
|
||||
for node_id, address, udp_port, tcp_port, blob_hash, added_at in state.datastore:
|
||||
if added_at + constants.DATA_EXPIRATION < now:
|
||||
continue
|
||||
p = make_kademlia_peer(node_id, address, udp_port, tcp_port)
|
||||
self.protocol.data_store.add_peer_to_blob(p, blob_hash)
|
||||
|
||||
async def start_listening(self, interface: str = '0.0.0.0') -> None:
|
||||
if not self.listening_port:
|
||||
self.listening_port, _ = await self.loop.create_datagram_endpoint(
|
||||
lambda: self.protocol, (interface, self.internal_udp_port)
|
||||
)
|
||||
self.started_listening.set()
|
||||
log.info("DHT node listening on UDP %s:%i", interface, self.internal_udp_port)
|
||||
self.protocol.start()
|
||||
else:
|
||||
log.warning("Already bound to port %s", self.listening_port)
|
||||
|
||||
async def join_network(self, interface: str = '0.0.0.0',
|
||||
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
|
||||
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
persisted_peers: typing.List[typing.Tuple[bytes, str, int, int]] = []):
|
||||
def peers_from_urls(urls: typing.Optional[typing.List[typing.Tuple[bytes, str, int, int]]]):
|
||||
peer_addresses = []
|
||||
for node_id, address, udp_port, tcp_port in urls:
|
||||
@@ -148,15 +178,15 @@ class Node:
|
||||
if not self.joined.is_set():
|
||||
self.joined.set()
|
||||
log.info(
|
||||
"joined dht, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
|
||||
"%s: joined dht, %i peers known in %i buckets",
|
||||
bytes.fromhex(self.protocol.node_id)[:8],
|
||||
len(self.protocol.routing_table.get_peers()),
|
||||
self.protocol.routing_table.buckets_with_contacts()
|
||||
)
|
||||
else:
|
||||
if self.joined.is_set():
|
||||
self.joined.clear()
|
||||
seed_peers = peers_from_urls(
|
||||
await self._storage.get_persisted_kademlia_peers()
|
||||
) if self._storage else []
|
||||
seed_peers = peers_from_urls(persisted_peers) if persisted_peers else []
|
||||
if not seed_peers:
|
||||
try:
|
||||
seed_peers.extend(peers_from_urls([
|
||||
@@ -173,8 +203,11 @@ class Node:
|
||||
|
||||
await asyncio.sleep(1, loop=self.loop)
|
||||
|
||||
def start(self, interface: str, known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
|
||||
self._join_task = self.loop.create_task(self.join_network(interface, known_node_urls))
|
||||
def start(self, interface: str,
|
||||
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
persisted_peers: typing.List[typing.Tuple[bytes, str, int, int]] = []):
|
||||
|
||||
self._join_task = self.loop.create_task(self.join_network(interface, known_node_urls, persisted_peers))
|
||||
|
||||
def get_iterative_node_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
|
||||
bottom_out_limit: int = constants.BOTTOM_OUT_LIMIT,
|
||||
|
||||
+22
-3
@@ -1,14 +1,14 @@
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
import ipaddress
|
||||
from binascii import hexlify
|
||||
from dataclasses import dataclass, field
|
||||
from functools import lru_cache
|
||||
from lbry.utils import is_valid_public_ipv4 as _is_valid_public_ipv4
|
||||
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
|
||||
|
||||
ALLOW_LOCALHOST = False
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -20,9 +20,28 @@ def make_kademlia_peer(node_id: typing.Optional[bytes], address: typing.Optional
|
||||
return KademliaPeer(address, node_id, udp_port, tcp_port=tcp_port, allow_localhost=allow_localhost)
|
||||
|
||||
|
||||
# the ipaddress module does not show these subnets as reserved
|
||||
CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
|
||||
IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
|
||||
|
||||
ALLOW_LOCALHOST = False
|
||||
|
||||
|
||||
def is_valid_public_ipv4(address, allow_localhost: bool = False):
|
||||
allow_localhost = bool(allow_localhost or ALLOW_LOCALHOST)
|
||||
return _is_valid_public_ipv4(address, allow_localhost)
|
||||
try:
|
||||
parsed_ip = ipaddress.ip_address(address)
|
||||
if parsed_ip.is_loopback and allow_localhost:
|
||||
return True
|
||||
|
||||
if any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local, parsed_ip.is_loopback,
|
||||
parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private, parsed_ip.is_reserved)):
|
||||
return False
|
||||
else:
|
||||
return not any((CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
|
||||
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
|
||||
except (ipaddress.AddressValueError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
class PeerManager:
|
||||
|
||||
@@ -68,3 +68,9 @@ class DictDataStore:
|
||||
for _, stored in self._data_store.items():
|
||||
peers.update(set(map(lambda tup: tup[0], stored)))
|
||||
return list(peers)
|
||||
|
||||
def dump(self) -> typing.List[typing.Tuple[bytes, str, int, int, bytes, float]]:
|
||||
data = []
|
||||
for k, peers in self._data_store.items():
|
||||
data.extend([(p.node_id, p.address, p.udp_port, p.tcp_port, k, added_at) for (p, added_at) in peers])
|
||||
return data
|
||||
@@ -287,6 +287,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
self._to_add: typing.Set['KademliaPeer'] = set()
|
||||
self._wakeup_routing_task = asyncio.Event(loop=self.loop)
|
||||
self.maintaing_routing_task: typing.Optional[asyncio.Task] = None
|
||||
self.event_queue = asyncio.Queue(maxsize=100)
|
||||
|
||||
@functools.lru_cache(128)
|
||||
def get_rpc_peer(self, peer: 'KademliaPeer') -> RemoteKademliaRPC:
|
||||
@@ -428,6 +429,11 @@ class KademliaProtocol(DatagramProtocol):
|
||||
log.debug("%s:%i RECV CALL %s %s:%i", self.external_ip, self.udp_port, message.method.decode(),
|
||||
sender_contact.address, sender_contact.udp_port)
|
||||
|
||||
try:
|
||||
self.event_queue.put_nowait((sender_contact.node_id, sender_contact.address, method, args))
|
||||
except asyncio.QueueFull:
|
||||
pass
|
||||
|
||||
if method == b'ping':
|
||||
result = self.node_rpc.ping()
|
||||
elif method == b'store':
|
||||
|
||||
@@ -286,7 +286,7 @@ class TreeRoutingTable:
|
||||
to_pop = [i for i, bucket in enumerate(self.buckets) if len(bucket) == 0]
|
||||
if not to_pop:
|
||||
return
|
||||
log.info("join buckets %i", len(to_pop))
|
||||
log.debug("%s: join buckets %i", bytes.hex(self._parent_node_id)[:8], len(to_pop))
|
||||
bucket_index_to_pop = to_pop[0]
|
||||
assert len(self.buckets[bucket_index_to_pop]) == 0
|
||||
can_go_lower = bucket_index_to_pop - 1 >= 0
|
||||
|
||||
@@ -132,7 +132,7 @@ class AnalyticsManager:
|
||||
async def run(self):
|
||||
while True:
|
||||
if self.enabled:
|
||||
self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
await self._send_heartbeat()
|
||||
await asyncio.sleep(1800)
|
||||
|
||||
|
||||
@@ -119,14 +119,13 @@ class WalletComponent(Component):
|
||||
async def get_status(self):
|
||||
if self.wallet_manager is None:
|
||||
return
|
||||
is_connected = self.wallet_manager.ledger.network.is_connected
|
||||
sessions = []
|
||||
session_pool = self.wallet_manager.ledger.network.session_pool
|
||||
sessions = session_pool.sessions
|
||||
connected = None
|
||||
if is_connected:
|
||||
addr, port = self.wallet_manager.ledger.network.client.server
|
||||
connected = f"{addr}:{port}"
|
||||
sessions.append(self.wallet_manager.ledger.network.client)
|
||||
|
||||
if self.wallet_manager.ledger.network.client:
|
||||
addr_and_port = self.wallet_manager.ledger.network.client.server_address_and_port
|
||||
if addr_and_port:
|
||||
connected = f"{addr_and_port[0]}:{addr_and_port[1]}"
|
||||
result = {
|
||||
'connected': connected,
|
||||
'connected_features': self.wallet_manager.ledger.network.server_features,
|
||||
@@ -138,8 +137,8 @@ class WalletComponent(Component):
|
||||
'availability': session.available,
|
||||
} for session in sessions
|
||||
],
|
||||
'known_servers': len(self.wallet_manager.ledger.network.config['default_servers']),
|
||||
'available_servers': 1 if is_connected else 0
|
||||
'known_servers': len(sessions),
|
||||
'available_servers': len(list(session_pool.available_sessions))
|
||||
}
|
||||
|
||||
if self.wallet_manager.ledger.network.remote_height:
|
||||
@@ -275,7 +274,7 @@ class DHTComponent(Component):
|
||||
external_ip = upnp_component.external_ip
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
if not external_ip:
|
||||
external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
external_ip = await utils.get_external_ip()
|
||||
if not external_ip:
|
||||
log.warning("failed to get external ip")
|
||||
|
||||
@@ -329,7 +328,7 @@ class HashAnnouncerComponent(Component):
|
||||
|
||||
class FileManagerComponent(Component):
|
||||
component_name = FILE_MANAGER_COMPONENT
|
||||
depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT]
|
||||
depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT, LIBTORRENT_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
@@ -352,10 +351,7 @@ class FileManagerComponent(Component):
|
||||
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
node = self.component_manager.get_component(DHT_COMPONENT) \
|
||||
if self.component_manager.has_component(DHT_COMPONENT) else None
|
||||
try:
|
||||
torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT) if TorrentSession else None
|
||||
except NameError:
|
||||
torrent = None
|
||||
torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT) if TorrentSession else None
|
||||
log.info('Starting the file manager')
|
||||
loop = asyncio.get_event_loop()
|
||||
self.file_manager = FileManager(
|
||||
@@ -364,7 +360,7 @@ class FileManagerComponent(Component):
|
||||
self.file_manager.source_managers['stream'] = StreamManager(
|
||||
loop, self.conf, blob_manager, wallet, storage, node,
|
||||
)
|
||||
if TorrentSession and LIBTORRENT_COMPONENT not in self.conf.components_to_skip:
|
||||
if TorrentSession:
|
||||
self.file_manager.source_managers['torrent'] = TorrentManager(
|
||||
loop, self.conf, torrent, storage, self.component_manager.analytics_manager
|
||||
)
|
||||
@@ -476,7 +472,7 @@ class UPnPComponent(Component):
|
||||
pass
|
||||
if external_ip and not is_valid_public_ipv4(external_ip):
|
||||
log.warning("UPnP returned a private/reserved ip - %s, checking lbry.com fallback", external_ip)
|
||||
external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
external_ip = await utils.get_external_ip()
|
||||
if self.external_ip and self.external_ip != external_ip:
|
||||
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
|
||||
if external_ip:
|
||||
@@ -534,7 +530,7 @@ class UPnPComponent(Component):
|
||||
async def start(self):
|
||||
log.info("detecting external ip")
|
||||
if not self.use_upnp:
|
||||
self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
return
|
||||
success = False
|
||||
await self._maintain_redirects()
|
||||
@@ -549,9 +545,9 @@ class UPnPComponent(Component):
|
||||
else:
|
||||
log.error("failed to setup upnp")
|
||||
if not self.external_ip:
|
||||
self.external_ip, probed_url = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
if self.external_ip:
|
||||
log.info("detected external ip using %s fallback", probed_url)
|
||||
log.info("detected external ip using lbry.com fallback")
|
||||
if self.component_manager.analytics_manager:
|
||||
self.component_manager.loop.create_task(
|
||||
self.component_manager.analytics_manager.send_upnp_setup_success_fail(
|
||||
|
||||
@@ -2808,12 +2808,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
for channel certificates, defaults to all accounts.
|
||||
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
|
||||
|
||||
Returns:
|
||||
(dict) Signature if successfully made, (None) or an error otherwise
|
||||
{
|
||||
"signature": (str) The signature of the comment,
|
||||
"signing_ts": (str) The timestamp used to sign the comment,
|
||||
}
|
||||
Returns: {}
|
||||
"""
|
||||
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
|
||||
assert not wallet.is_locked, "Cannot spend funds with locked wallet, unlock first."
|
||||
@@ -4347,8 +4342,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
@staticmethod
|
||||
def _constrain_txo_from_kwargs(
|
||||
constraints, type=None, txid=None, # pylint: disable=redefined-builtin
|
||||
claim_id=None, channel_id=None, not_channel_id=None,
|
||||
name=None, reposted_claim_id=None,
|
||||
claim_id=None, channel_id=None, name=None, reposted_claim_id=None,
|
||||
is_spent=False, is_not_spent=False,
|
||||
is_my_input_or_output=None, exclude_internal_transfers=False,
|
||||
is_my_output=None, is_not_my_output=None,
|
||||
@@ -4371,7 +4365,6 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
constraints['is_my_output'] = False
|
||||
database.constrain_single_or_list(constraints, 'txo_type', type, lambda x: TXO_TYPES[x])
|
||||
database.constrain_single_or_list(constraints, 'channel_id', channel_id)
|
||||
database.constrain_single_or_list(constraints, 'channel_id', not_channel_id, negate=True)
|
||||
database.constrain_single_or_list(constraints, 'claim_id', claim_id)
|
||||
database.constrain_single_or_list(constraints, 'claim_name', name)
|
||||
database.constrain_single_or_list(constraints, 'txid', txid)
|
||||
@@ -4386,9 +4379,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
List my transaction outputs.
|
||||
|
||||
Usage:
|
||||
txo_list [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...] [--claim_id=<claim_id>...]
|
||||
[--channel_id=<channel_id>...] [--not_channel_id=<not_channel_id>...]
|
||||
[--name=<name>...] [--is_spent | --is_not_spent]
|
||||
txo_list [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...]
|
||||
[--claim_id=<claim_id>...] [--channel_id=<channel_id>...] [--name=<name>...]
|
||||
[--is_spent | --is_not_spent]
|
||||
[--is_my_input_or_output |
|
||||
[[--is_my_output | --is_not_my_output] [--is_my_input | --is_not_my_input]]
|
||||
]
|
||||
@@ -4402,7 +4395,6 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--txid=<txid> : (str or list) transaction id of outputs
|
||||
--claim_id=<claim_id> : (str or list) claim id
|
||||
--channel_id=<channel_id> : (str or list) claims in this channel
|
||||
--not_channel_id=<not_channel_id>: (str or list) claims not in this channel
|
||||
--name=<name> : (str or list) claim name
|
||||
--is_spent : (bool) only show spent txos
|
||||
--is_not_spent : (bool) only show not spent txos
|
||||
@@ -4462,9 +4454,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
Spend transaction outputs, batching into multiple transactions as necessary.
|
||||
|
||||
Usage:
|
||||
txo_spend [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...] [--claim_id=<claim_id>...]
|
||||
[--channel_id=<channel_id>...] [--not_channel_id=<not_channel_id>...]
|
||||
[--name=<name>...] [--is_my_input | --is_not_my_input]
|
||||
txo_spend [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...]
|
||||
[--claim_id=<claim_id>...] [--channel_id=<channel_id>...] [--name=<name>...]
|
||||
[--is_my_input | --is_not_my_input]
|
||||
[--exclude_internal_transfers] [--wallet_id=<wallet_id>]
|
||||
[--preview] [--blocking] [--batch_size=<batch_size>] [--include_full_tx]
|
||||
|
||||
@@ -4474,7 +4466,6 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--txid=<txid> : (str or list) transaction id of outputs
|
||||
--claim_id=<claim_id> : (str or list) claim id
|
||||
--channel_id=<channel_id> : (str or list) claims in this channel
|
||||
--not_channel_id=<not_channel_id>: (str or list) claims not in this channel
|
||||
--name=<name> : (str or list) claim name
|
||||
--is_my_input : (bool) show outputs created by you
|
||||
--is_not_my_input : (bool) show outputs not created by you
|
||||
@@ -4519,7 +4510,6 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
Usage:
|
||||
txo_list [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...]
|
||||
[--channel_id=<channel_id>...] [--not_channel_id=<not_channel_id>...]
|
||||
[--claim_id=<claim_id>...] [--name=<name>...]
|
||||
[--is_spent] [--is_not_spent]
|
||||
[--is_my_input_or_output |
|
||||
@@ -4533,8 +4523,6 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--txid=<txid> : (str or list) transaction id of outputs
|
||||
--claim_id=<claim_id> : (str or list) claim id
|
||||
--name=<name> : (str or list) claim name
|
||||
--channel_id=<channel_id> : (str or list) claims in this channel
|
||||
--not_channel_id=<not_channel_id>: (str or list) claims not in this channel
|
||||
--is_spent : (bool) only show spent txos
|
||||
--is_not_spent : (bool) only show not spent txos
|
||||
--is_my_input_or_output : (bool) txos which have your inputs or your outputs,
|
||||
@@ -4569,7 +4557,6 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
Usage:
|
||||
txo_plot [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...]
|
||||
[--claim_id=<claim_id>...] [--name=<name>...] [--is_spent] [--is_not_spent]
|
||||
[--channel_id=<channel_id>...] [--not_channel_id=<not_channel_id>...]
|
||||
[--is_my_input_or_output |
|
||||
[[--is_my_output | --is_not_my_output] [--is_my_input | --is_not_my_input]]
|
||||
]
|
||||
@@ -4584,8 +4571,6 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--txid=<txid> : (str or list) transaction id of outputs
|
||||
--claim_id=<claim_id> : (str or list) claim id
|
||||
--name=<name> : (str or list) claim name
|
||||
--channel_id=<channel_id> : (str or list) claims in this channel
|
||||
--not_channel_id=<not_channel_id>: (str or list) claims not in this channel
|
||||
--is_spent : (bool) only show spent txos
|
||||
--is_not_spent : (bool) only show not spent txos
|
||||
--is_my_input_or_output : (bool) txos which have your inputs or your outputs,
|
||||
|
||||
@@ -85,11 +85,8 @@ class FileManager:
|
||||
raise ResolveError("cannot download a channel claim, specify a /path")
|
||||
try:
|
||||
resolved_result = await asyncio.wait_for(
|
||||
self.wallet_manager.ledger.resolve(
|
||||
wallet.accounts, [uri],
|
||||
include_purchase_receipt=True,
|
||||
include_is_my_output=True
|
||||
), resolve_timeout
|
||||
self.wallet_manager.ledger.resolve(wallet.accounts, [uri], include_purchase_receipt=True),
|
||||
resolve_timeout
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
raise ResolveTimeoutError(uri)
|
||||
@@ -173,14 +170,7 @@ class FileManager:
|
||||
# pay fee
|
||||
####################
|
||||
|
||||
needs_purchasing = (
|
||||
not to_replace and
|
||||
not txo.is_my_output and
|
||||
txo.has_price and
|
||||
not txo.purchase_receipt
|
||||
)
|
||||
|
||||
if needs_purchasing:
|
||||
if not to_replace and txo.has_price and not txo.purchase_receipt:
|
||||
payment = await self.wallet_manager.create_purchase_transaction(
|
||||
wallet.accounts, txo, exchange_rate_manager
|
||||
)
|
||||
|
||||
@@ -46,8 +46,8 @@ types_map = {
|
||||
'.ksh': ('text/plain', 'document'),
|
||||
'.latex': ('application/x-latex', 'binary'),
|
||||
'.m1v': ('video/mpeg', 'video'),
|
||||
'.m3u': ('application/x-mpegurl', 'audio'),
|
||||
'.m3u8': ('application/x-mpegurl', 'video'),
|
||||
'.m3u': ('application/vnd.apple.mpegurl', 'audio'),
|
||||
'.m3u8': ('application/vnd.apple.mpegurl', 'video'),
|
||||
'.man': ('application/x-troff-man', 'document'),
|
||||
'.markdown': ('text/markdown', 'document'),
|
||||
'.md': ('text/markdown', 'document'),
|
||||
@@ -66,7 +66,6 @@ types_map = {
|
||||
'.mpeg': ('video/mpeg', 'video'),
|
||||
'.mpg': ('video/mpeg', 'video'),
|
||||
'.ms': ('application/x-troff-ms', 'binary'),
|
||||
'.m4s': ('video/iso.segment', 'binary'),
|
||||
'.nc': ('application/x-netcdf', 'binary'),
|
||||
'.nws': ('message/rfc822', 'document'),
|
||||
'.o': ('application/octet-stream', 'binary'),
|
||||
@@ -120,7 +119,6 @@ types_map = {
|
||||
'.tif': ('image/tiff', 'image'),
|
||||
'.tiff': ('image/tiff', 'image'),
|
||||
'.tr': ('application/x-troff', 'binary'),
|
||||
'.ts': ('video/mp2t', 'video'),
|
||||
'.tsv': ('text/tab-separated-values', 'document'),
|
||||
'.txt': ('text/plain', 'document'),
|
||||
'.ustar': ('application/x-ustar', 'binary'),
|
||||
|
||||
@@ -5,8 +5,10 @@ from binascii import hexlify
|
||||
from itertools import chain
|
||||
|
||||
from lbry.error import ResolveCensoredError
|
||||
from lbry.schema.types.v2.result_pb2 import Outputs as OutputsMessage
|
||||
from lbry.schema.types.v2.result_pb2 import Error as ErrorMessage
|
||||
from lbry.schema.types.v2.result_pb2 import \
|
||||
Outputs as OutputsMessage, \
|
||||
Output as OutputMessage, \
|
||||
Error as ErrorMessage
|
||||
|
||||
INVALID = ErrorMessage.Code.Name(ErrorMessage.INVALID)
|
||||
NOT_FOUND = ErrorMessage.Code.Name(ErrorMessage.NOT_FOUND)
|
||||
@@ -70,7 +72,7 @@ class Outputs:
|
||||
|
||||
__slots__ = 'txos', 'extra_txos', 'txs', 'offset', 'total', 'blocked', 'blocked_total'
|
||||
|
||||
def __init__(self, txos: List, extra_txos: List, txs: set,
|
||||
def __init__(self, txos: List[OutputMessage], extra_txos: List, txs: set,
|
||||
offset: int, total: int, blocked: List, blocked_total: int):
|
||||
self.txos = txos
|
||||
self.txs = txs
|
||||
|
||||
+2
-6
@@ -23,9 +23,8 @@ from lbry.wallet.orchstr8.node import BlockchainNode, WalletNode
|
||||
from lbry.extras.daemon.daemon import Daemon, jsonrpc_dumps_pretty
|
||||
from lbry.extras.daemon.components import Component, WalletComponent
|
||||
from lbry.extras.daemon.components import (
|
||||
DHT_COMPONENT,
|
||||
HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, LIBTORRENT_COMPONENT
|
||||
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
)
|
||||
from lbry.extras.daemon.componentmanager import ComponentManager
|
||||
from lbry.extras.daemon.exchange_rate_manager import (
|
||||
@@ -321,7 +320,6 @@ class CommandTestCase(IntegrationTestCase):
|
||||
self.server_blob_manager = None
|
||||
self.server = None
|
||||
self.reflector = None
|
||||
self.skip_libtorrent = True
|
||||
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
@@ -397,8 +395,6 @@ class CommandTestCase(IntegrationTestCase):
|
||||
DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
|
||||
PEER_PROTOCOL_SERVER_COMPONENT
|
||||
]
|
||||
if self.skip_libtorrent:
|
||||
conf.components_to_skip.append(LIBTORRENT_COMPONENT)
|
||||
wallet_node.manager.config = conf
|
||||
|
||||
def wallet_maker(component_manager):
|
||||
|
||||
+7
-126
@@ -192,8 +192,6 @@ def cache_concurrent(async_fn):
|
||||
async def resolve_host(url: str, port: int, proto: str) -> str:
|
||||
if proto not in ['udp', 'tcp']:
|
||||
raise Exception("invalid protocol")
|
||||
if url.lower() == 'localhost':
|
||||
return '127.0.0.1'
|
||||
try:
|
||||
if ipaddress.ip_address(url):
|
||||
return url
|
||||
@@ -208,7 +206,7 @@ async def resolve_host(url: str, port: int, proto: str) -> str:
|
||||
))[0][4][0]
|
||||
|
||||
|
||||
class LRUCacheWithMetrics:
|
||||
class LRUCache:
|
||||
__slots__ = [
|
||||
'capacity',
|
||||
'cache',
|
||||
@@ -233,7 +231,7 @@ class LRUCacheWithMetrics:
|
||||
f"{metric_name}_cache_miss_count", "Number of cache misses", namespace=namespace
|
||||
)
|
||||
except ValueError as err:
|
||||
log.debug("failed to set up prometheus %s_cache_miss_count metric: %s", metric_name, err)
|
||||
log.warning("failed to set up prometheus %s_cache_miss_count metric: %s", metric_name, err)
|
||||
self._track_metrics = False
|
||||
self.hits = self.misses = None
|
||||
|
||||
@@ -288,63 +286,12 @@ class LRUCacheWithMetrics:
|
||||
pass
|
||||
|
||||
|
||||
class LRUCache:
|
||||
__slots__ = [
|
||||
'capacity',
|
||||
'cache'
|
||||
]
|
||||
|
||||
def __init__(self, capacity: int):
|
||||
self.capacity = capacity
|
||||
self.cache = collections.OrderedDict()
|
||||
|
||||
def get(self, key, default=None):
|
||||
try:
|
||||
value = self.cache.pop(key)
|
||||
except KeyError:
|
||||
return default
|
||||
self.cache[key] = value
|
||||
return value
|
||||
|
||||
def set(self, key, value):
|
||||
try:
|
||||
self.cache.pop(key)
|
||||
except KeyError:
|
||||
if len(self.cache) >= self.capacity:
|
||||
self.cache.popitem(last=False)
|
||||
self.cache[key] = value
|
||||
|
||||
def clear(self):
|
||||
self.cache.clear()
|
||||
|
||||
def pop(self, key):
|
||||
return self.cache.pop(key)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
return self.set(key, value)
|
||||
|
||||
def __getitem__(self, item):
|
||||
return self.get(item)
|
||||
|
||||
def __contains__(self, item) -> bool:
|
||||
return item in self.cache
|
||||
|
||||
def __len__(self):
|
||||
return len(self.cache)
|
||||
|
||||
def __delitem__(self, key):
|
||||
self.cache.pop(key)
|
||||
|
||||
def __del__(self):
|
||||
self.clear()
|
||||
|
||||
|
||||
def lru_cache_concurrent(cache_size: typing.Optional[int] = None,
|
||||
override_lru_cache: typing.Optional[LRUCacheWithMetrics] = None):
|
||||
override_lru_cache: typing.Optional[LRUCache] = None):
|
||||
if not cache_size and override_lru_cache is None:
|
||||
raise ValueError("invalid cache size")
|
||||
concurrent_cache = {}
|
||||
lru_cache = override_lru_cache if override_lru_cache is not None else LRUCacheWithMetrics(cache_size)
|
||||
lru_cache = override_lru_cache if override_lru_cache is not None else LRUCache(cache_size)
|
||||
|
||||
def wrapper(async_fn):
|
||||
|
||||
@@ -379,80 +326,14 @@ async def aiohttp_request(method, url, **kwargs) -> typing.AsyncContextManager[a
|
||||
yield response
|
||||
|
||||
|
||||
# the ipaddress module does not show these subnets as reserved
|
||||
CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
|
||||
IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
|
||||
|
||||
|
||||
def is_valid_public_ipv4(address, allow_localhost: bool = False):
|
||||
try:
|
||||
parsed_ip = ipaddress.ip_address(address)
|
||||
if parsed_ip.is_loopback and allow_localhost:
|
||||
return True
|
||||
if any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local, parsed_ip.is_loopback,
|
||||
parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private, parsed_ip.is_reserved)):
|
||||
return False
|
||||
else:
|
||||
return not any((CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
|
||||
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
|
||||
except (ipaddress.AddressValueError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
async def fallback_get_external_ip(): # used if spv servers can't be used for ip detection
|
||||
async def get_external_ip() -> typing.Optional[str]: # used if upnp is disabled or non-functioning
|
||||
try:
|
||||
async with aiohttp_request("get", "https://api.lbry.com/ip") as resp:
|
||||
response = await resp.json()
|
||||
if response['success']:
|
||||
return response['data']['ip'], None
|
||||
return response['data']['ip']
|
||||
except Exception:
|
||||
return None, None
|
||||
|
||||
|
||||
async def _get_external_ip(default_servers) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]:
|
||||
# used if upnp is disabled or non-functioning
|
||||
from lbry.wallet.server.udp import SPVStatusClientProtocol # pylint: disable=C0415
|
||||
|
||||
hostname_to_ip = {}
|
||||
ip_to_hostnames = collections.defaultdict(list)
|
||||
|
||||
async def resolve_spv(server, port):
|
||||
try:
|
||||
server_addr = await resolve_host(server, port, 'udp')
|
||||
hostname_to_ip[server] = (server_addr, port)
|
||||
ip_to_hostnames[(server_addr, port)].append(server)
|
||||
except Exception:
|
||||
log.exception("error looking up dns for spv servers")
|
||||
|
||||
# accumulate the dns results
|
||||
await asyncio.gather(*(resolve_spv(server, port) for (server, port) in default_servers))
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
pong_responses = asyncio.Queue()
|
||||
connection = SPVStatusClientProtocol(pong_responses)
|
||||
try:
|
||||
await loop.create_datagram_endpoint(lambda: connection, ('0.0.0.0', 0))
|
||||
# could raise OSError if it cant bind
|
||||
randomized_servers = list(ip_to_hostnames.keys())
|
||||
random.shuffle(randomized_servers)
|
||||
for server in randomized_servers:
|
||||
connection.ping(server)
|
||||
try:
|
||||
_, pong = await asyncio.wait_for(pong_responses.get(), 1)
|
||||
if is_valid_public_ipv4(pong.ip_address):
|
||||
return pong.ip_address, ip_to_hostnames[server][0]
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
return None, None
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
async def get_external_ip(default_servers) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]:
|
||||
ip_from_spv_servers = await _get_external_ip(default_servers)
|
||||
if not ip_from_spv_servers[1]:
|
||||
return await fallback_get_external_ip()
|
||||
return ip_from_spv_servers
|
||||
return
|
||||
|
||||
|
||||
def is_running_from_bundle():
|
||||
|
||||
+3
-23
@@ -387,31 +387,14 @@ def interpolate(sql, values):
|
||||
return sql
|
||||
|
||||
|
||||
def constrain_single_or_list(constraints, column, value, convert=lambda x: x, negate=False):
|
||||
def constrain_single_or_list(constraints, column, value, convert=lambda x: x):
|
||||
if value is not None:
|
||||
if isinstance(value, list):
|
||||
value = [convert(v) for v in value]
|
||||
if len(value) == 1:
|
||||
if negate:
|
||||
constraints[f"{column}__or"] = {
|
||||
f"{column}__is_null": True,
|
||||
f"{column}__not": value[0]
|
||||
}
|
||||
else:
|
||||
constraints[column] = value[0]
|
||||
constraints[column] = value[0]
|
||||
elif len(value) > 1:
|
||||
if negate:
|
||||
constraints[f"{column}__or"] = {
|
||||
f"{column}__is_null": True,
|
||||
f"{column}__not_in": value
|
||||
}
|
||||
else:
|
||||
constraints[f"{column}__in"] = value
|
||||
elif negate:
|
||||
constraints[f"{column}__or"] = {
|
||||
f"{column}__is_null": True,
|
||||
f"{column}__not": convert(value)
|
||||
}
|
||||
constraints[f"{column}__in"] = value
|
||||
else:
|
||||
constraints[column] = convert(value)
|
||||
return constraints
|
||||
@@ -704,9 +687,6 @@ class Database(SQLiteMixin):
|
||||
row['txo_type'] = TXO_TYPES['stream']
|
||||
elif txo.is_support:
|
||||
row['txo_type'] = TXO_TYPES['support']
|
||||
support = txo.can_decode_support
|
||||
if support and support.is_signed:
|
||||
row['channel_id'] = support.signing_channel_id
|
||||
elif txo.purchase is not None:
|
||||
row['txo_type'] = TXO_TYPES['purchase']
|
||||
row['claim_id'] = txo.purchased_claim_id
|
||||
|
||||
+3
-25
@@ -14,7 +14,7 @@ from lbry.schema.result import Outputs, INVALID, NOT_FOUND
|
||||
from lbry.schema.url import URL
|
||||
from lbry.crypto.hash import hash160, double_sha256, sha256
|
||||
from lbry.crypto.base58 import Base58
|
||||
from lbry.utils import LRUCacheWithMetrics
|
||||
from lbry.utils import LRUCache
|
||||
|
||||
from .tasks import TaskGroup
|
||||
from .database import Database
|
||||
@@ -155,7 +155,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
self._on_ready_controller = StreamController()
|
||||
self.on_ready = self._on_ready_controller.stream
|
||||
|
||||
self._tx_cache = LRUCacheWithMetrics(self.config.get("tx_cache_size", 1024), metric_name='tx')
|
||||
self._tx_cache = LRUCache(self.config.get("tx_cache_size", 1024), metric_name='tx')
|
||||
self._update_tasks = TaskGroup()
|
||||
self._other_tasks = TaskGroup() # that we dont need to start
|
||||
self._utxo_reservation_lock = asyncio.Lock()
|
||||
@@ -167,7 +167,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
self._known_addresses_out_of_sync = set()
|
||||
|
||||
self.fee_per_name_char = self.config.get('fee_per_name_char', self.default_fee_per_name_char)
|
||||
self._balance_cache = LRUCacheWithMetrics(2 ** 15)
|
||||
self._balance_cache = LRUCache(2 ** 15)
|
||||
|
||||
@classmethod
|
||||
def get_id(cls):
|
||||
@@ -937,11 +937,6 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
return self.db.get_purchase_count(**constraints)
|
||||
|
||||
async def _resolve_for_local_results(self, accounts, txos):
|
||||
txos = await self._resolve_for_local_claim_results(accounts, txos)
|
||||
txos = await self._resolve_for_local_support_results(accounts, txos)
|
||||
return txos
|
||||
|
||||
async def _resolve_for_local_claim_results(self, accounts, txos):
|
||||
results = []
|
||||
response = await self.resolve(
|
||||
accounts, [txo.permanent_url for txo in txos if txo.can_decode_claim]
|
||||
@@ -957,23 +952,6 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
results.append(txo)
|
||||
return results
|
||||
|
||||
async def _resolve_for_local_support_results(self, accounts, txos):
|
||||
channel_ids = set()
|
||||
signed_support_txos = []
|
||||
for txo in txos:
|
||||
support = txo.can_decode_support
|
||||
if support and support.signing_channel_id:
|
||||
channel_ids.add(support.signing_channel_id)
|
||||
signed_support_txos.append(txo)
|
||||
if channel_ids:
|
||||
channels = {
|
||||
channel.claim_id: channel for channel in
|
||||
(await self.claim_search(accounts, claim_ids=list(channel_ids)))[0]
|
||||
}
|
||||
for txo in signed_support_txos:
|
||||
txo.channel = channels.get(txo.support.signing_channel_id)
|
||||
return txos
|
||||
|
||||
async def get_claims(self, resolve=False, **constraints):
|
||||
claims = await self.db.get_claims(**constraints)
|
||||
if resolve:
|
||||
|
||||
+173
-181
@@ -1,27 +1,26 @@
|
||||
import logging
|
||||
import asyncio
|
||||
import json
|
||||
import socket
|
||||
from time import perf_counter
|
||||
from collections import defaultdict
|
||||
from operator import itemgetter
|
||||
from typing import Dict, Optional, Tuple
|
||||
import aiohttp
|
||||
|
||||
from lbry import __version__
|
||||
from lbry.utils import resolve_host
|
||||
from lbry.error import IncompatibleWalletServerError
|
||||
from lbry.wallet.rpc import RPCSession as BaseClientSession, Connector, RPCError, ProtocolError
|
||||
from lbry.wallet.stream import StreamController
|
||||
from lbry.wallet.server.udp import SPVStatusClientProtocol, SPVPong
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ClientSession(BaseClientSession):
|
||||
def __init__(self, *args, network: 'Network', server, timeout=30, **kwargs):
|
||||
def __init__(self, *args, network, server, timeout=30, on_connect_callback=None, **kwargs):
|
||||
self.network = network
|
||||
self.server = server
|
||||
super().__init__(*args, **kwargs)
|
||||
self._on_disconnect_controller = StreamController()
|
||||
self.on_disconnected = self._on_disconnect_controller.stream
|
||||
self.framer.max_size = self.max_errors = 1 << 32
|
||||
self.timeout = timeout
|
||||
self.max_seconds_idle = timeout * 2
|
||||
@@ -29,6 +28,8 @@ class ClientSession(BaseClientSession):
|
||||
self.connection_latency: Optional[float] = None
|
||||
self._response_samples = 0
|
||||
self.pending_amount = 0
|
||||
self._on_connect_cb = on_connect_callback or (lambda: None)
|
||||
self.trigger_urgent_reconnect = asyncio.Event()
|
||||
|
||||
@property
|
||||
def available(self):
|
||||
@@ -55,7 +56,7 @@ class ClientSession(BaseClientSession):
|
||||
|
||||
async def send_request(self, method, args=()):
|
||||
self.pending_amount += 1
|
||||
log.debug("send %s%s to %s:%i (%i timeout)", method, tuple(args), self.server[0], self.server[1], self.timeout)
|
||||
log.debug("send %s%s to %s:%i", method, tuple(args), *self.server)
|
||||
try:
|
||||
if method == 'server.version':
|
||||
return await self.send_timed_server_version_request(args, self.timeout)
|
||||
@@ -66,7 +67,7 @@ class ClientSession(BaseClientSession):
|
||||
log.debug("Time since last packet: %s", perf_counter() - self.last_packet_received)
|
||||
if (perf_counter() - self.last_packet_received) < self.timeout:
|
||||
continue
|
||||
log.warning("timeout sending %s to %s:%i", method, *self.server)
|
||||
log.info("timeout sending %s to %s:%i", method, *self.server)
|
||||
raise asyncio.TimeoutError
|
||||
if done:
|
||||
try:
|
||||
@@ -86,12 +87,44 @@ class ClientSession(BaseClientSession):
|
||||
self.synchronous_close()
|
||||
raise
|
||||
except asyncio.CancelledError:
|
||||
log.warning("cancelled sending %s to %s:%i", method, *self.server)
|
||||
log.info("cancelled sending %s to %s:%i", method, *self.server)
|
||||
# self.synchronous_close()
|
||||
raise
|
||||
finally:
|
||||
self.pending_amount -= 1
|
||||
|
||||
async def ensure_session(self):
|
||||
# Handles reconnecting and maintaining a session alive
|
||||
# TODO: change to 'ping' on newer protocol (above 1.2)
|
||||
retry_delay = default_delay = 1.0
|
||||
while True:
|
||||
try:
|
||||
if self.is_closing():
|
||||
await self.create_connection(self.timeout)
|
||||
await self.ensure_server_version()
|
||||
self._on_connect_cb()
|
||||
if (perf_counter() - self.last_send) > self.max_seconds_idle or self.response_time is None:
|
||||
await self.ensure_server_version()
|
||||
retry_delay = default_delay
|
||||
except RPCError as e:
|
||||
await self.close()
|
||||
log.debug("Server error, ignoring for 1h: %s:%d -- %s", *self.server, e.message)
|
||||
retry_delay = 60 * 60
|
||||
except IncompatibleWalletServerError:
|
||||
await self.close()
|
||||
retry_delay = 60 * 60
|
||||
log.debug("Wallet server has an incompatible version, retrying in 1h: %s:%d", *self.server)
|
||||
except (asyncio.TimeoutError, OSError):
|
||||
await self.close()
|
||||
retry_delay = min(60, retry_delay * 2)
|
||||
log.debug("Wallet server timeout (retry in %s seconds): %s:%d", retry_delay, *self.server)
|
||||
try:
|
||||
await asyncio.wait_for(self.trigger_urgent_reconnect.wait(), timeout=retry_delay)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
finally:
|
||||
self.trigger_urgent_reconnect.clear()
|
||||
|
||||
async def ensure_server_version(self, required=None, timeout=3):
|
||||
required = required or self.network.PROTOCOL_VERSION
|
||||
response = await asyncio.wait_for(
|
||||
@@ -101,25 +134,6 @@ class ClientSession(BaseClientSession):
|
||||
raise IncompatibleWalletServerError(*self.server)
|
||||
return response
|
||||
|
||||
async def keepalive_loop(self, timeout=3, max_idle=60):
|
||||
try:
|
||||
while True:
|
||||
now = perf_counter()
|
||||
if min(self.last_send, self.last_packet_received) + max_idle < now:
|
||||
await asyncio.wait_for(
|
||||
self.send_request('server.ping', []), timeout=timeout
|
||||
)
|
||||
else:
|
||||
await asyncio.sleep(max(0, max_idle - (now - self.last_send)))
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError):
|
||||
log.warning("closing connection to %s:%i", *self.server)
|
||||
else:
|
||||
log.exception("lost connection to spv")
|
||||
finally:
|
||||
if not self.is_closing():
|
||||
self._close()
|
||||
|
||||
async def create_connection(self, timeout=6):
|
||||
connector = Connector(lambda: self, *self.server)
|
||||
start = perf_counter()
|
||||
@@ -131,14 +145,12 @@ class ClientSession(BaseClientSession):
|
||||
controller.add(request.args)
|
||||
|
||||
def connection_lost(self, exc):
|
||||
log.warning("Connection lost: %s:%d", *self.server)
|
||||
log.debug("Connection lost: %s:%d", *self.server)
|
||||
super().connection_lost(exc)
|
||||
self.response_time = None
|
||||
self.connection_latency = None
|
||||
self._response_samples = 0
|
||||
# self._on_disconnect_controller.add(True)
|
||||
if self.network:
|
||||
self.network.disconnect()
|
||||
self._on_disconnect_controller.add(True)
|
||||
|
||||
|
||||
class Network:
|
||||
@@ -148,9 +160,10 @@ class Network:
|
||||
|
||||
def __init__(self, ledger):
|
||||
self.ledger = ledger
|
||||
self.session_pool = SessionPool(network=self, timeout=self.config.get('connect_timeout', 6))
|
||||
self.client: Optional[ClientSession] = None
|
||||
self.server_features = None
|
||||
# self._switch_task: Optional[asyncio.Task] = None
|
||||
self._switch_task: Optional[asyncio.Task] = None
|
||||
self.running = False
|
||||
self.remote_height: int = 0
|
||||
self._concurrency = asyncio.Semaphore(16)
|
||||
@@ -170,170 +183,58 @@ class Network:
|
||||
}
|
||||
|
||||
self.aiohttp_session: Optional[aiohttp.ClientSession] = None
|
||||
self._urgent_need_reconnect = asyncio.Event()
|
||||
self._loop_task: Optional[asyncio.Task] = None
|
||||
self._keepalive_task: Optional[asyncio.Task] = None
|
||||
|
||||
@property
|
||||
def config(self):
|
||||
return self.ledger.config
|
||||
|
||||
def disconnect(self):
|
||||
if self._keepalive_task and not self._keepalive_task.done():
|
||||
self._keepalive_task.cancel()
|
||||
self._keepalive_task = None
|
||||
|
||||
async def start(self):
|
||||
if not self.running:
|
||||
self.running = True
|
||||
self.aiohttp_session = aiohttp.ClientSession()
|
||||
self.on_header.listen(self._update_remote_height)
|
||||
self._loop_task = asyncio.create_task(self.network_loop())
|
||||
self._urgent_need_reconnect.set()
|
||||
|
||||
def loop_task_done_callback(f):
|
||||
try:
|
||||
f.result()
|
||||
except Exception:
|
||||
if self.running:
|
||||
log.exception("wallet server connection loop crashed")
|
||||
|
||||
self._loop_task.add_done_callback(loop_task_done_callback)
|
||||
|
||||
async def resolve_spv_dns(self):
|
||||
hostname_to_ip = {}
|
||||
ip_to_hostnames = defaultdict(list)
|
||||
|
||||
async def resolve_spv(server, port):
|
||||
try:
|
||||
server_addr = await resolve_host(server, port, 'udp')
|
||||
hostname_to_ip[server] = (server_addr, port)
|
||||
ip_to_hostnames[(server_addr, port)].append(server)
|
||||
except socket.error:
|
||||
log.warning("error looking up dns for spv server %s:%i", server, port)
|
||||
except Exception:
|
||||
log.exception("error looking up dns for spv server %s:%i", server, port)
|
||||
|
||||
# accumulate the dns results
|
||||
await asyncio.gather(*(resolve_spv(server, port) for (server, port) in self.config['default_servers']))
|
||||
return hostname_to_ip, ip_to_hostnames
|
||||
|
||||
async def get_n_fastest_spvs(self, n=5, timeout=3.0) -> Dict[Tuple[str, int], SPVPong]:
|
||||
loop = asyncio.get_event_loop()
|
||||
pong_responses = asyncio.Queue()
|
||||
connection = SPVStatusClientProtocol(pong_responses)
|
||||
sent_ping_timestamps = {}
|
||||
_, ip_to_hostnames = await self.resolve_spv_dns()
|
||||
log.info("%i possible spv servers to try (%i urls in config)", len(ip_to_hostnames),
|
||||
len(self.config['default_servers']))
|
||||
pongs = {}
|
||||
try:
|
||||
await loop.create_datagram_endpoint(lambda: connection, ('0.0.0.0', 0))
|
||||
# could raise OSError if it cant bind
|
||||
start = perf_counter()
|
||||
for server in ip_to_hostnames:
|
||||
connection.ping(server)
|
||||
sent_ping_timestamps[server] = perf_counter()
|
||||
while len(pongs) < n:
|
||||
(remote, ts), pong = await asyncio.wait_for(pong_responses.get(), timeout - (perf_counter() - start))
|
||||
latency = ts - start
|
||||
log.info("%s:%i has latency of %sms (available: %s, height: %i)",
|
||||
'/'.join(ip_to_hostnames[remote]), remote[1], round(latency * 1000, 2),
|
||||
pong.available, pong.height)
|
||||
|
||||
if pong.available:
|
||||
pongs[remote] = pong
|
||||
return pongs
|
||||
except asyncio.TimeoutError:
|
||||
if pongs:
|
||||
log.info("%i/%i probed spv servers are accepting connections", len(pongs), len(ip_to_hostnames))
|
||||
else:
|
||||
log.warning("%i spv status probes failed, retrying later. servers tried: %s",
|
||||
len(sent_ping_timestamps),
|
||||
', '.join('/'.join(hosts) + f' ({ip})' for ip, hosts in ip_to_hostnames.items()))
|
||||
return pongs
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
async def connect_to_fastest(self) -> Optional[ClientSession]:
|
||||
fastest_spvs = await self.get_n_fastest_spvs()
|
||||
for (host, port) in fastest_spvs:
|
||||
|
||||
client = ClientSession(network=self, server=(host, port))
|
||||
try:
|
||||
await client.create_connection()
|
||||
log.warning("Connected to spv server %s:%i", host, port)
|
||||
await client.ensure_server_version()
|
||||
return client
|
||||
except (asyncio.TimeoutError, ConnectionError, OSError, IncompatibleWalletServerError, RPCError):
|
||||
log.warning("Connecting to %s:%d failed", host, port)
|
||||
client._close()
|
||||
return
|
||||
|
||||
async def network_loop(self):
|
||||
sleep_delay = 30
|
||||
async def switch_forever(self):
|
||||
while self.running:
|
||||
await asyncio.wait(
|
||||
[asyncio.sleep(30), self._urgent_need_reconnect.wait()], return_when=asyncio.FIRST_COMPLETED
|
||||
)
|
||||
if self._urgent_need_reconnect.is_set():
|
||||
sleep_delay = 30
|
||||
self._urgent_need_reconnect.clear()
|
||||
if not self.is_connected:
|
||||
client = await self.connect_to_fastest()
|
||||
if not client:
|
||||
log.warning("failed to connect to any spv servers, retrying later")
|
||||
sleep_delay *= 2
|
||||
sleep_delay = min(sleep_delay, 300)
|
||||
continue
|
||||
log.debug("get spv server features %s:%i", *client.server)
|
||||
features = await client.send_request('server.features', [])
|
||||
self.client, self.server_features = client, features
|
||||
log.info("subscribe to headers %s:%i", *client.server)
|
||||
if self.is_connected:
|
||||
await self.client.on_disconnected.first
|
||||
self.server_features = None
|
||||
self.client = None
|
||||
continue
|
||||
self.client = await self.session_pool.wait_for_fastest_session()
|
||||
log.info("Switching to SPV wallet server: %s:%d", *self.client.server)
|
||||
try:
|
||||
self.server_features = await self.get_server_features()
|
||||
self._update_remote_height((await self.subscribe_headers(),))
|
||||
self._on_connected_controller.add(True)
|
||||
server_str = "%s:%i" % client.server
|
||||
log.info("maintaining connection to spv server %s", server_str)
|
||||
self._keepalive_task = asyncio.create_task(self.client.keepalive_loop())
|
||||
try:
|
||||
await asyncio.wait(
|
||||
[self._keepalive_task, self._urgent_need_reconnect.wait()],
|
||||
return_when=asyncio.FIRST_COMPLETED
|
||||
)
|
||||
if self._urgent_need_reconnect.is_set():
|
||||
log.warning("urgent reconnect needed")
|
||||
self._urgent_need_reconnect.clear()
|
||||
if self._keepalive_task and not self._keepalive_task.done():
|
||||
self._keepalive_task.cancel()
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
finally:
|
||||
self._keepalive_task = None
|
||||
self.client = None
|
||||
self.server_features = None
|
||||
log.warning("connection lost to %s", server_str)
|
||||
log.info("network loop finished")
|
||||
log.info("Subscribed to headers: %s:%d", *self.client.server)
|
||||
except (asyncio.TimeoutError, ConnectionError):
|
||||
log.info("Switching to %s:%d timed out, closing and retrying.", *self.client.server)
|
||||
self.client.synchronous_close()
|
||||
self.server_features = None
|
||||
self.client = None
|
||||
|
||||
async def start(self):
|
||||
self.running = True
|
||||
self.aiohttp_session = aiohttp.ClientSession()
|
||||
self._switch_task = asyncio.ensure_future(self.switch_forever())
|
||||
# this may become unnecessary when there are no more bugs found,
|
||||
# but for now it helps understanding log reports
|
||||
self._switch_task.add_done_callback(lambda _: log.info("Wallet client switching task stopped."))
|
||||
self.session_pool.start(self.config['default_servers'])
|
||||
self.on_header.listen(self._update_remote_height)
|
||||
|
||||
async def stop(self):
|
||||
self.running = False
|
||||
self.disconnect()
|
||||
if self._loop_task and not self._loop_task.done():
|
||||
self._loop_task.cancel()
|
||||
self._loop_task = None
|
||||
if self.aiohttp_session:
|
||||
if self.running:
|
||||
self.running = False
|
||||
await self.aiohttp_session.close()
|
||||
self.aiohttp_session = None
|
||||
self._switch_task.cancel()
|
||||
self.session_pool.stop()
|
||||
|
||||
@property
|
||||
def is_connected(self):
|
||||
return self.client and not self.client.is_closing()
|
||||
|
||||
def rpc(self, list_or_method, args, restricted=True, session: Optional[ClientSession] = None):
|
||||
if session or self.is_connected:
|
||||
session = session or self.client
|
||||
def rpc(self, list_or_method, args, restricted=True, session=None):
|
||||
session = session or (self.client if restricted else self.session_pool.fastest_session)
|
||||
if session and not session.is_closing():
|
||||
return session.send_request(list_or_method, args)
|
||||
else:
|
||||
self._urgent_need_reconnect.set()
|
||||
self.session_pool.trigger_nodelay_connect()
|
||||
raise ConnectionError("Attempting to send rpc request when connection is not available.")
|
||||
|
||||
async def retriable_call(self, function, *args, **kwargs):
|
||||
@@ -341,15 +242,14 @@ class Network:
|
||||
while self.running:
|
||||
if not self.is_connected:
|
||||
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
|
||||
self._urgent_need_reconnect.set()
|
||||
await self.on_connected.first
|
||||
await self.session_pool.wait_for_fastest_session()
|
||||
try:
|
||||
return await function(*args, **kwargs)
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("Wallet server call timed out, retrying.")
|
||||
except ConnectionError:
|
||||
log.warning("connection error")
|
||||
|
||||
pass
|
||||
raise asyncio.CancelledError() # if we got here, we are shutting down
|
||||
|
||||
def _update_remote_height(self, header_args):
|
||||
@@ -439,3 +339,95 @@ class Network:
|
||||
async with self.aiohttp_session.post(server, json=message) as r:
|
||||
result = await r.json()
|
||||
return result['result']
|
||||
|
||||
|
||||
class SessionPool:
|
||||
|
||||
def __init__(self, network: Network, timeout: float):
|
||||
self.network = network
|
||||
self.sessions: Dict[ClientSession, Optional[asyncio.Task]] = dict()
|
||||
self.timeout = timeout
|
||||
self.new_connection_event = asyncio.Event()
|
||||
|
||||
@property
|
||||
def online(self):
|
||||
return any(not session.is_closing() for session in self.sessions)
|
||||
|
||||
@property
|
||||
def available_sessions(self):
|
||||
return (session for session in self.sessions if session.available)
|
||||
|
||||
@property
|
||||
def fastest_session(self):
|
||||
if not self.online:
|
||||
return None
|
||||
return min(
|
||||
[((session.response_time + session.connection_latency) * (session.pending_amount + 1), session)
|
||||
for session in self.available_sessions] or [(0, None)],
|
||||
key=itemgetter(0)
|
||||
)[1]
|
||||
|
||||
def _get_session_connect_callback(self, session: ClientSession):
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def callback():
|
||||
duplicate_connections = [
|
||||
s for s in self.sessions
|
||||
if s is not session and s.server_address_and_port == session.server_address_and_port
|
||||
]
|
||||
already_connected = None if not duplicate_connections else duplicate_connections[0]
|
||||
if already_connected:
|
||||
self.sessions.pop(session).cancel()
|
||||
session.synchronous_close()
|
||||
log.debug("wallet server %s resolves to the same server as %s, rechecking in an hour",
|
||||
session.server[0], already_connected.server[0])
|
||||
loop.call_later(3600, self._connect_session, session.server)
|
||||
return
|
||||
self.new_connection_event.set()
|
||||
log.info("connected to %s:%i", *session.server)
|
||||
|
||||
return callback
|
||||
|
||||
def _connect_session(self, server: Tuple[str, int]):
|
||||
session = None
|
||||
for s in self.sessions:
|
||||
if s.server == server:
|
||||
session = s
|
||||
break
|
||||
if not session:
|
||||
session = ClientSession(
|
||||
network=self.network, server=server
|
||||
)
|
||||
session._on_connect_cb = self._get_session_connect_callback(session)
|
||||
task = self.sessions.get(session, None)
|
||||
if not task or task.done():
|
||||
task = asyncio.create_task(session.ensure_session())
|
||||
task.add_done_callback(lambda _: self.ensure_connections())
|
||||
self.sessions[session] = task
|
||||
|
||||
def start(self, default_servers):
|
||||
for server in default_servers:
|
||||
self._connect_session(server)
|
||||
|
||||
def stop(self):
|
||||
for session, task in self.sessions.items():
|
||||
task.cancel()
|
||||
session.synchronous_close()
|
||||
self.sessions.clear()
|
||||
|
||||
def ensure_connections(self):
|
||||
for session in self.sessions:
|
||||
self._connect_session(session.server)
|
||||
|
||||
def trigger_nodelay_connect(self):
|
||||
# used when other parts of the system sees we might have internet back
|
||||
# bypasses the retry interval
|
||||
for session in self.sessions:
|
||||
session.trigger_urgent_reconnect.set()
|
||||
|
||||
async def wait_for_fastest_session(self):
|
||||
while not self.fastest_session:
|
||||
self.trigger_nodelay_connect()
|
||||
self.new_connection_event.clear()
|
||||
await self.new_connection_event.wait()
|
||||
return self.fastest_session
|
||||
@@ -11,7 +11,6 @@ from lbry.wallet.server.daemon import DaemonError
|
||||
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
|
||||
from lbry.wallet.server.util import chunks, class_logger
|
||||
from lbry.wallet.server.leveldb import FlushData
|
||||
from lbry.wallet.server.udp import StatusServer
|
||||
|
||||
|
||||
class Prefetcher:
|
||||
@@ -186,7 +185,6 @@ class BlockProcessor:
|
||||
|
||||
self.search_cache = {}
|
||||
self.history_cache = {}
|
||||
self.status_server = StatusServer()
|
||||
|
||||
async def run_in_thread_with_lock(self, func, *args):
|
||||
# Run in a thread to prevent blocking. Shielded so that
|
||||
@@ -223,7 +221,6 @@ class BlockProcessor:
|
||||
processed_time = time.perf_counter() - start
|
||||
self.block_count_metric.set(self.height)
|
||||
self.block_update_time_metric.observe(processed_time)
|
||||
self.status_server.set_height(self.db.fs_height, self.db.db_tip)
|
||||
if not self.db.first_sync:
|
||||
s = '' if len(blocks) == 1 else 's'
|
||||
self.logger.info('processed {:,d} block{} in {:.1f}s'.format(len(blocks), s, processed_time))
|
||||
@@ -685,11 +682,9 @@ class BlockProcessor:
|
||||
disk before exiting, as otherwise a significant amount of work
|
||||
could be lost.
|
||||
"""
|
||||
|
||||
self._caught_up_event = caught_up_event
|
||||
try:
|
||||
await self._first_open_dbs()
|
||||
self.status_server.set_height(self.db.fs_height, self.db.db_tip)
|
||||
await asyncio.wait([
|
||||
self.prefetcher.main_loop(self.height),
|
||||
self._process_prefetched_blocks()
|
||||
@@ -700,7 +695,6 @@ class BlockProcessor:
|
||||
self.logger.exception("Block processing failed!")
|
||||
raise
|
||||
finally:
|
||||
self.status_server.stop()
|
||||
# Shut down block processing
|
||||
self.logger.info('flushing to DB for a clean shutdown...')
|
||||
await self.flush(True)
|
||||
@@ -720,6 +714,7 @@ class BlockProcessor:
|
||||
|
||||
|
||||
class Timer:
|
||||
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
self.total = 0
|
||||
|
||||
@@ -6,7 +6,7 @@ from functools import wraps
|
||||
|
||||
import aiohttp
|
||||
from prometheus_client import Gauge, Histogram
|
||||
from lbry.utils import LRUCacheWithMetrics
|
||||
from lbry.utils import LRUCache
|
||||
from lbry.wallet.rpc.jsonrpc import RPCError
|
||||
from lbry.wallet.server.util import hex_to_bytes, class_logger
|
||||
from lbry.wallet.rpc import JSONRPC
|
||||
@@ -54,8 +54,8 @@ class Daemon:
|
||||
self._height = None
|
||||
self.available_rpcs = {}
|
||||
self.connector = aiohttp.TCPConnector()
|
||||
self._block_hash_cache = LRUCacheWithMetrics(100000)
|
||||
self._block_cache = LRUCacheWithMetrics(2 ** 16, metric_name='block', namespace=NAMESPACE)
|
||||
self._block_hash_cache = LRUCache(100000)
|
||||
self._block_cache = LRUCache(2**16, metric_name='block', namespace=NAMESPACE)
|
||||
|
||||
async def close(self):
|
||||
if self.connector:
|
||||
|
||||
@@ -24,7 +24,7 @@ from glob import glob
|
||||
from struct import pack, unpack
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
import attr
|
||||
from lbry.utils import LRUCacheWithMetrics
|
||||
from lbry.utils import LRUCache
|
||||
from lbry.wallet.server import util
|
||||
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
|
||||
from lbry.wallet.server.merkle import Merkle, MerkleCache
|
||||
@@ -93,7 +93,7 @@ class LevelDB:
|
||||
self.headers_db = None
|
||||
self.tx_db = None
|
||||
|
||||
self._tx_and_merkle_cache = LRUCacheWithMetrics(2 ** 17, metric_name='tx_and_merkle', namespace="wallet_server")
|
||||
self._tx_and_merkle_cache = LRUCache(2**17, metric_name='tx_and_merkle', namespace="wallet_server")
|
||||
self.total_transactions = None
|
||||
|
||||
async def _read_tx_counts(self):
|
||||
|
||||
@@ -344,7 +344,7 @@ class MemPool:
|
||||
value += sum(v for h168, v in tx.out_pairs if h168 == hashX)
|
||||
return value
|
||||
|
||||
def compact_fee_histogram(self):
|
||||
async def compact_fee_histogram(self):
|
||||
"""Return a compact fee histogram of the current mempool."""
|
||||
return self.cached_compact_histogram
|
||||
|
||||
|
||||
@@ -110,14 +110,11 @@ class Server:
|
||||
self.cancellable_tasks.append(asyncio.ensure_future(run(*args, _flag)))
|
||||
return _flag.wait()
|
||||
|
||||
await self.start_prometheus()
|
||||
await self.bp.status_server.start(0, bytes.fromhex(self.bp.coin.GENESIS_HASH)[::-1]
|
||||
, self.env.host, self.env.tcp_port)
|
||||
await _start_cancellable(self.bp.fetch_and_process_blocks)
|
||||
|
||||
await self.db.populate_header_merkle_cache()
|
||||
await _start_cancellable(self.mempool.keep_synchronized)
|
||||
await _start_cancellable(self.session_mgr.serve, self.notifications)
|
||||
await self.start_prometheus()
|
||||
|
||||
async def stop(self):
|
||||
for task in reversed(self.cancellable_tasks):
|
||||
|
||||
@@ -21,7 +21,7 @@ from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
|
||||
from prometheus_client import Counter, Info, Histogram, Gauge
|
||||
|
||||
import lbry
|
||||
from lbry.utils import LRUCacheWithMetrics
|
||||
from lbry.utils import LRUCache
|
||||
from lbry.build_info import BUILD, COMMIT_HASH, DOCKER_TAG
|
||||
from lbry.wallet.server.block_processor import LBRYBlockProcessor
|
||||
from lbry.wallet.server.db.writer import LBRYLevelDB
|
||||
@@ -247,12 +247,11 @@ class SessionManager:
|
||||
async def _manage_servers(self):
|
||||
paused = False
|
||||
max_sessions = self.env.max_sessions
|
||||
low_watermark = int(max_sessions * 0.95)
|
||||
low_watermark = max_sessions * 19 // 20
|
||||
while True:
|
||||
await self.session_event.wait()
|
||||
self.session_event.clear()
|
||||
if not paused and len(self.sessions) >= max_sessions:
|
||||
self.bp.status_server.set_unavailable()
|
||||
self.logger.info(f'maximum sessions {max_sessions:,d} '
|
||||
f'reached, stopping new connections until '
|
||||
f'count drops to {low_watermark:,d}')
|
||||
@@ -261,7 +260,6 @@ class SessionManager:
|
||||
# Start listening for incoming connections if paused and
|
||||
# session count has fallen
|
||||
if paused and len(self.sessions) <= low_watermark:
|
||||
self.bp.status_server.set_available()
|
||||
self.logger.info('resuming listening for incoming connections')
|
||||
await self._start_external_servers()
|
||||
paused = False
|
||||
@@ -574,7 +572,6 @@ class SessionManager:
|
||||
await self.start_other()
|
||||
await self._start_external_servers()
|
||||
server_listening_event.set()
|
||||
self.bp.status_server.set_available()
|
||||
# Peer discovery should start after the external servers
|
||||
# because we connect to ourself
|
||||
await asyncio.wait([
|
||||
@@ -585,7 +582,6 @@ class SessionManager:
|
||||
])
|
||||
finally:
|
||||
await self._close_servers(list(self.servers.keys()))
|
||||
log.warning("disconnect %i sessions", len(self.sessions))
|
||||
if self.sessions:
|
||||
await asyncio.wait([
|
||||
session.close(force_after=1) for session in self.sessions.values()
|
||||
@@ -814,8 +810,8 @@ class LBRYSessionManager(SessionManager):
|
||||
if self.env.websocket_host is not None and self.env.websocket_port is not None:
|
||||
self.websocket = AdminWebSocket(self)
|
||||
self.search_cache = self.bp.search_cache
|
||||
self.search_cache['search'] = LRUCacheWithMetrics(2 ** 14, metric_name='search', namespace=NAMESPACE)
|
||||
self.search_cache['resolve'] = LRUCacheWithMetrics(2 ** 16, metric_name='resolve', namespace=NAMESPACE)
|
||||
self.search_cache['search'] = LRUCache(2**14, metric_name='search', namespace=NAMESPACE)
|
||||
self.search_cache['resolve'] = LRUCache(2**16, metric_name='resolve', namespace=NAMESPACE)
|
||||
|
||||
async def process_metrics(self):
|
||||
while self.running:
|
||||
@@ -868,7 +864,6 @@ class LBRYElectrumX(SessionBase):
|
||||
max_errors = math.inf # don't disconnect people for errors! let them happen...
|
||||
session_mgr: LBRYSessionManager
|
||||
version = lbry.__version__
|
||||
cached_server_features = {}
|
||||
|
||||
@classmethod
|
||||
def initialize_request_handlers(cls):
|
||||
@@ -915,15 +910,13 @@ class LBRYElectrumX(SessionBase):
|
||||
super().__init__(*args, **kwargs)
|
||||
if not LBRYElectrumX.request_handlers:
|
||||
LBRYElectrumX.initialize_request_handlers()
|
||||
if not LBRYElectrumX.cached_server_features:
|
||||
LBRYElectrumX.set_server_features(self.env)
|
||||
self.subscribe_headers = False
|
||||
self.subscribe_headers_raw = False
|
||||
self.connection.max_response_size = self.env.max_send
|
||||
self.hashX_subs = {}
|
||||
self.sv_seen = False
|
||||
self.protocol_tuple = self.PROTOCOL_MIN
|
||||
self.protocol_string = None
|
||||
|
||||
self.daemon = self.session_mgr.daemon
|
||||
self.bp: LBRYBlockProcessor = self.session_mgr.bp
|
||||
self.db: LBRYLevelDB = self.bp.db
|
||||
@@ -934,10 +927,10 @@ class LBRYElectrumX(SessionBase):
|
||||
for ver in (cls.PROTOCOL_MIN, cls.PROTOCOL_MAX)]
|
||||
|
||||
@classmethod
|
||||
def set_server_features(cls, env):
|
||||
def server_features(cls, env):
|
||||
"""Return the server features dictionary."""
|
||||
min_str, max_str = cls.protocol_min_max_strings()
|
||||
cls.cached_server_features.update({
|
||||
return {
|
||||
'hosts': env.hosts_dict(),
|
||||
'pruning': None,
|
||||
'server_version': cls.version,
|
||||
@@ -950,10 +943,10 @@ class LBRYElectrumX(SessionBase):
|
||||
'daily_fee': env.daily_fee,
|
||||
'hash_function': 'sha256',
|
||||
'trending_algorithm': env.trending_algorithms[0]
|
||||
})
|
||||
}
|
||||
|
||||
async def server_features_async(self):
|
||||
return self.cached_server_features
|
||||
return self.server_features(self.env)
|
||||
|
||||
@classmethod
|
||||
def server_version_args(cls):
|
||||
@@ -1278,7 +1271,7 @@ class LBRYElectrumX(SessionBase):
|
||||
hashXes = [
|
||||
(self.address_to_hashX(address), address) for address in addresses
|
||||
]
|
||||
return [await self.hashX_subscribe(*args) for args in hashXes]
|
||||
return await asyncio.gather(*(self.hashX_subscribe(*args) for args in hashXes))
|
||||
|
||||
async def address_unsubscribe(self, address):
|
||||
"""Unsubscribe an address.
|
||||
@@ -1474,8 +1467,7 @@ class LBRYElectrumX(SessionBase):
|
||||
client_name: a string identifying the client
|
||||
protocol_version: the protocol version spoken by the client
|
||||
"""
|
||||
if self.protocol_string is not None:
|
||||
return self.version, self.protocol_string
|
||||
|
||||
if self.sv_seen and self.protocol_tuple >= (1, 4):
|
||||
raise RPCError(BAD_REQUEST, f'server.version already sent')
|
||||
self.sv_seen = True
|
||||
@@ -1485,7 +1477,8 @@ class LBRYElectrumX(SessionBase):
|
||||
if self.env.drop_client is not None and \
|
||||
self.env.drop_client.match(client_name):
|
||||
self.close_after_send = True
|
||||
raise RPCError(BAD_REQUEST, f'unsupported client: {client_name}')
|
||||
raise RPCError(BAD_REQUEST,
|
||||
f'unsupported client: {client_name}')
|
||||
if self.client_version != client_name[:17]:
|
||||
self.session_mgr.session_count_metric.labels(version=self.client_version).dec()
|
||||
self.client_version = client_name[:17]
|
||||
@@ -1494,16 +1487,19 @@ class LBRYElectrumX(SessionBase):
|
||||
|
||||
# Find the highest common protocol version. Disconnect if
|
||||
# that protocol version in unsupported.
|
||||
ptuple, client_min = util.protocol_version(protocol_version, self.PROTOCOL_MIN, self.PROTOCOL_MAX)
|
||||
ptuple, client_min = util.protocol_version(
|
||||
protocol_version, self.PROTOCOL_MIN, self.PROTOCOL_MAX)
|
||||
if ptuple is None:
|
||||
ptuple, client_min = util.protocol_version(protocol_version, (1, 1, 0), (1, 4, 0))
|
||||
if ptuple is None:
|
||||
self.close_after_send = True
|
||||
raise RPCError(BAD_REQUEST, f'unsupported protocol version: {protocol_version}')
|
||||
|
||||
# FIXME: this fills the logs
|
||||
# if client_min > self.PROTOCOL_MIN:
|
||||
# self.logger.info(f'client requested future protocol version '
|
||||
# f'{util.version_string(client_min)} '
|
||||
# f'- is your software out of date?')
|
||||
self.close_after_send = True
|
||||
raise RPCError(BAD_REQUEST,
|
||||
f'unsupported protocol version: {protocol_version}')
|
||||
self.protocol_tuple = ptuple
|
||||
self.protocol_string = util.version_string(ptuple)
|
||||
return self.version, self.protocol_string
|
||||
return self.version, self.protocol_version_string()
|
||||
|
||||
async def transaction_broadcast(self, raw_tx):
|
||||
"""Broadcast a raw transaction to the network.
|
||||
|
||||
@@ -1,192 +0,0 @@
|
||||
import asyncio
|
||||
import struct
|
||||
from time import perf_counter
|
||||
import logging
|
||||
from typing import Optional, Tuple, NamedTuple
|
||||
from lbry.utils import LRUCache
|
||||
# from prometheus_client import Counter
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
_MAGIC = 1446058291 # genesis blocktime (which is actually wrong)
|
||||
# ping_count_metric = Counter("ping_count", "Number of pings received", namespace='wallet_server_status')
|
||||
_PAD_BYTES = b'\x00' * 64
|
||||
|
||||
|
||||
class SPVPing(NamedTuple):
|
||||
magic: int
|
||||
protocol_version: int
|
||||
pad_bytes: bytes
|
||||
|
||||
def encode(self):
|
||||
return struct.pack(b'!lB64s', *self)
|
||||
|
||||
@staticmethod
|
||||
def make(protocol_version=1) -> bytes:
|
||||
return SPVPing(_MAGIC, protocol_version, _PAD_BYTES).encode()
|
||||
|
||||
@classmethod
|
||||
def decode(cls, packet: bytes):
|
||||
decoded = cls(*struct.unpack(b'!lB64s', packet[:69]))
|
||||
if decoded.magic != _MAGIC:
|
||||
raise ValueError("invalid magic bytes")
|
||||
return decoded
|
||||
|
||||
|
||||
class SPVPong(NamedTuple):
|
||||
protocol_version: int
|
||||
flags: int
|
||||
height: int
|
||||
tip: bytes
|
||||
source_address_raw: bytes
|
||||
|
||||
def encode(self):
|
||||
return struct.pack(b'!BBl32s4s', *self)
|
||||
|
||||
@staticmethod
|
||||
def make(height: int, tip: bytes, flags: int, protocol_version: int = 1) -> bytes:
|
||||
# note: drops the last 4 bytes so the result can be cached and have addresses added to it as needed
|
||||
return SPVPong(protocol_version, flags, height, tip, b'\x00\x00\x00\x00').encode()[:38]
|
||||
|
||||
@classmethod
|
||||
def decode(cls, packet: bytes):
|
||||
return cls(*struct.unpack(b'!BBl32s4s', packet[:42]))
|
||||
|
||||
@property
|
||||
def available(self) -> bool:
|
||||
return (self.flags & 0b00000001) > 0
|
||||
|
||||
@property
|
||||
def ip_address(self) -> str:
|
||||
return ".".join(map(str, self.source_address_raw))
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"SPVPong(external_ip={self.ip_address}, version={self.protocol_version}, " \
|
||||
f"available={'True' if self.flags & 1 > 0 else 'False'}," \
|
||||
f" height={self.height}, tip={self.tip[::-1].hex()})"
|
||||
|
||||
|
||||
class SPVServerStatusProtocol(asyncio.DatagramProtocol):
|
||||
PROTOCOL_VERSION = 1
|
||||
|
||||
def __init__(self, height: int, tip: bytes, throttle_cache_size: int = 1024, throttle_reqs_per_sec: int = 10):
|
||||
super().__init__()
|
||||
self.transport: Optional[asyncio.transports.DatagramTransport] = None
|
||||
self._height = height
|
||||
self._tip = tip
|
||||
self._flags = 0
|
||||
self._cached_response = None
|
||||
self.update_cached_response()
|
||||
self._throttle = LRUCache(throttle_cache_size)
|
||||
self._should_log = LRUCache(throttle_cache_size)
|
||||
self._min_delay = 1 / throttle_reqs_per_sec
|
||||
|
||||
def update_cached_response(self):
|
||||
self._cached_response = SPVPong.make(self._height, self._tip, self._flags, self.PROTOCOL_VERSION)
|
||||
|
||||
def set_unavailable(self):
|
||||
self._flags &= 0b11111110
|
||||
self.update_cached_response()
|
||||
|
||||
def set_available(self):
|
||||
self._flags |= 0b00000001
|
||||
self.update_cached_response()
|
||||
|
||||
def set_height(self, height: int, tip: bytes):
|
||||
self._height, self._tip = height, tip
|
||||
self.update_cached_response()
|
||||
|
||||
def should_throttle(self, host: str):
|
||||
now = perf_counter()
|
||||
last_requested = self._throttle.get(host, default=0)
|
||||
self._throttle[host] = now
|
||||
if now - last_requested < self._min_delay:
|
||||
log_cnt = self._should_log.get(host, default=0) + 1
|
||||
if log_cnt % 100 == 0:
|
||||
log.warning("throttle spv status to %s", host)
|
||||
self._should_log[host] = log_cnt
|
||||
return True
|
||||
return False
|
||||
|
||||
def make_pong(self, host):
|
||||
return self._cached_response + bytes(int(b) for b in host.split("."))
|
||||
|
||||
def datagram_received(self, data: bytes, addr: Tuple[str, int]):
|
||||
if self.should_throttle(addr[0]):
|
||||
return
|
||||
try:
|
||||
SPVPing.decode(data)
|
||||
except (ValueError, struct.error, AttributeError, TypeError):
|
||||
# log.exception("derp")
|
||||
return
|
||||
self.transport.sendto(self.make_pong(addr[0]), addr)
|
||||
# ping_count_metric.inc()
|
||||
|
||||
def connection_made(self, transport) -> None:
|
||||
self.transport = transport
|
||||
|
||||
def connection_lost(self, exc: Optional[Exception]) -> None:
|
||||
self.transport = None
|
||||
|
||||
def close(self):
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
|
||||
|
||||
class StatusServer:
|
||||
def __init__(self):
|
||||
self._protocol: Optional[SPVServerStatusProtocol] = None
|
||||
|
||||
async def start(self, height: int, tip: bytes, interface: str, port: int):
|
||||
if self._protocol:
|
||||
return
|
||||
loop = asyncio.get_event_loop()
|
||||
self._protocol = SPVServerStatusProtocol(height, tip)
|
||||
interface = interface if interface.lower() != 'localhost' else '127.0.0.1'
|
||||
await loop.create_datagram_endpoint(lambda: self._protocol, (interface, port))
|
||||
log.info("started udp status server on %s:%i", interface, port)
|
||||
|
||||
def stop(self):
|
||||
if self._protocol:
|
||||
self._protocol.close()
|
||||
self._protocol = None
|
||||
|
||||
def set_unavailable(self):
|
||||
self._protocol.set_unavailable()
|
||||
|
||||
def set_available(self):
|
||||
self._protocol.set_available()
|
||||
|
||||
def set_height(self, height: int, tip: bytes):
|
||||
self._protocol.set_height(height, tip)
|
||||
|
||||
|
||||
class SPVStatusClientProtocol(asyncio.DatagramProtocol):
|
||||
PROTOCOL_VERSION = 1
|
||||
|
||||
def __init__(self, responses: asyncio.Queue):
|
||||
super().__init__()
|
||||
self.transport: Optional[asyncio.transports.DatagramTransport] = None
|
||||
self.responses = responses
|
||||
self._ping_packet = SPVPing.make(self.PROTOCOL_VERSION)
|
||||
|
||||
def datagram_received(self, data: bytes, addr: Tuple[str, int]):
|
||||
try:
|
||||
self.responses.put_nowait(((addr, perf_counter()), SPVPong.decode(data)))
|
||||
except (ValueError, struct.error, AttributeError, TypeError, RuntimeError):
|
||||
return
|
||||
|
||||
def connection_made(self, transport) -> None:
|
||||
self.transport = transport
|
||||
|
||||
def connection_lost(self, exc: Optional[Exception]) -> None:
|
||||
self.transport = None
|
||||
log.info("closed udp spv server selection client")
|
||||
|
||||
def ping(self, server: Tuple[str, int]):
|
||||
self.transport.sendto(self._ping_packet, server)
|
||||
|
||||
def close(self):
|
||||
# log.info("close udp client")
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
Executable
+273
@@ -0,0 +1,273 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import pickle
|
||||
import signal
|
||||
import sqlite3
|
||||
import time
|
||||
from os import path
|
||||
from pprint import pprint
|
||||
from urllib.parse import unquote
|
||||
|
||||
from aioupnp import upnp, fault as upnpfault
|
||||
from aiohttp import web
|
||||
|
||||
from lbry.wallet.network import ClientSession
|
||||
from lbry.schema.result import Outputs
|
||||
from lbry.wallet.transaction import Transaction
|
||||
from binascii import hexlify, unhexlify
|
||||
from lbry.dht import node, peer
|
||||
|
||||
log = logging.getLogger("lbry")
|
||||
log.addHandler(logging.StreamHandler())
|
||||
log.setLevel(logging.INFO)
|
||||
|
||||
|
||||
async def main():
|
||||
data_dir = "/home/grin/code/lbry/sdk"
|
||||
state_dir = data_dir + '/nodestate/'
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
try:
|
||||
loop.add_signal_handler(signal.SIGINT, shutdown)
|
||||
loop.add_signal_handler(signal.SIGTERM, shutdown)
|
||||
except NotImplementedError:
|
||||
pass # Not implemented on Windows
|
||||
|
||||
db = sqlite3.connect(data_dir + "/tracker.sqlite3")
|
||||
db.execute('CREATE TABLE IF NOT EXISTS announce (local_id TEXT, hash TEXT, node_id TEXT, ip TEXT, port INT, timestamp INT)')
|
||||
db.execute('CREATE UNIQUE INDEX IF NOT EXISTS node_id_hash_idx ON announce (node_id, hash)')
|
||||
|
||||
spv_host = 'spv13.lbry.com'
|
||||
wallet_client = ClientSession(network=None, server=(spv_host, 50001))
|
||||
await wallet_client.create_connection()
|
||||
|
||||
asyncio.create_task(run_web_api(loop, db, wallet_client))
|
||||
|
||||
num_nodes = 128
|
||||
u = await upnp.UPnP.discover()
|
||||
external_ip = await u.get_external_ip()
|
||||
|
||||
try:
|
||||
nodes = await start_nodes(loop, num_nodes, external_ip, state_dir)
|
||||
|
||||
await asyncio.gather(*map(lambda n: n.started_listening.wait(), nodes), loop=loop)
|
||||
print("joined")
|
||||
|
||||
queue = asyncio.Queue(maxsize=100*num_nodes)
|
||||
for n in nodes:
|
||||
asyncio.create_task(drain_events(n, queue))
|
||||
|
||||
while True:
|
||||
(n, node_id, ip, method, args) = await queue.get()
|
||||
local_node_id = bytes.hex(n.protocol.node_id)
|
||||
if method != b'store':
|
||||
# print(f"{local_node_id[:8]}: {method} from {bytes.hex(node_id)} ({ip})")
|
||||
continue
|
||||
|
||||
if len(args)< 5:
|
||||
print(f'malformed args to Store')
|
||||
pprint(args)
|
||||
continue
|
||||
|
||||
blob_hash, token, port, original_publisher_id, age = args[:5]
|
||||
print(f"STORE to {local_node_id[:8]} from {bytes.hex(node_id)[:8]} ({ip}) for blob {bytes.hex(blob_hash)[:8]}")
|
||||
|
||||
try:
|
||||
cur = db.cursor()
|
||||
cur.execute(
|
||||
'''
|
||||
INSERT INTO announce (local_id, hash, node_id, ip, port, timestamp) VALUES (?,?,?,?,?,?)
|
||||
ON CONFLICT (node_id, hash) DO UPDATE SET
|
||||
local_id=excluded.local_id, ip=excluded.ip, port=excluded.port, timestamp=excluded.timestamp
|
||||
''',
|
||||
(local_node_id, bytes.hex(blob_hash), bytes.hex(node_id), ip, port, int(time.time()))
|
||||
)
|
||||
db.commit()
|
||||
cur.close()
|
||||
except sqlite3.Error as err:
|
||||
print("failed sqlite insert", err)
|
||||
finally:
|
||||
print("shutting down")
|
||||
for n in nodes:
|
||||
node_id = bytes.hex(n.protocol.node_id)
|
||||
n.stop()
|
||||
# print(f'deleting upnp port mapping {n.protocol.udp_port}')
|
||||
try:
|
||||
await u.delete_port_mapping(n.protocol.udp_port, "UDP")
|
||||
except upnpfault.UPnPError:
|
||||
pass
|
||||
|
||||
state = n.get_state()
|
||||
# keep existing rt if there is one
|
||||
if len(state.routing_table_peers) == 0 and path.exists(state_dir + node_id):
|
||||
with open(state_dir + node_id, 'rb') as f:
|
||||
existing_state = pickle.load(f)
|
||||
if len(existing_state.routing_table_peers) > 0:
|
||||
state.routing_table_peers = existing_state.routing_table_peers
|
||||
print(f'rt empty on save, but old rt was recovered ({len(state.routing_table_peers)} peers)')
|
||||
with open(state_dir + node_id, 'wb') as f:
|
||||
# pprint(state.routing_table_peers)
|
||||
# pprint(state.datastore)
|
||||
print(f'{node_id[:8]}: saved {len(state.routing_table_peers)} rt peers, {len(state.datastore)} in store')
|
||||
pickle.dump(state, f)
|
||||
await wallet_client.close()
|
||||
db.close()
|
||||
|
||||
|
||||
class ShutdownErr(BaseException):
|
||||
pass
|
||||
|
||||
|
||||
def shutdown():
|
||||
print("got interrupt signal...")
|
||||
raise ShutdownErr()
|
||||
|
||||
|
||||
def make_node_id(i: int, n: int) -> str:
|
||||
"""
|
||||
split dht address space into N chunks and return the first id of the i'th chunk
|
||||
make_node_id(0,n) returns 000...000 for any n
|
||||
"""
|
||||
if not 0 <= i < n:
|
||||
raise ValueError("i must be between 0 (inclusive) and n (exclusive)")
|
||||
bytes_in_id = 48
|
||||
return "{0:0{1}x}".format(i * ((2**8)**bytes_in_id // n), bytes_in_id*2)
|
||||
|
||||
|
||||
async def start_nodes(loop, num_nodes, external_ip, state_dir):
|
||||
start_port = 4445
|
||||
known_node_urls = [("lbrynet1.lbry.com", 4444), ("lbrynet2.lbry.com", 4444), ("lbrynet3.lbry.com", 4444)]
|
||||
peer_manager = peer.PeerManager(loop)
|
||||
|
||||
nodes = []
|
||||
for i in range(num_nodes):
|
||||
node_id = make_node_id(i, num_nodes)
|
||||
# pprint(node_id)
|
||||
|
||||
port = start_port + i
|
||||
# await u.get_next_mapping(port, "UDP", "lbry dht tracker") # not necessary, i just opened ports in router
|
||||
|
||||
n = node.Node(loop, peer_manager, node_id=bytes.fromhex(node_id), external_ip=external_ip,
|
||||
udp_port=port, internal_udp_port=port, peer_port=3333)
|
||||
|
||||
persisted_peers = []
|
||||
if path.exists(state_dir + node_id):
|
||||
with open(state_dir + node_id, 'rb') as f:
|
||||
state = pickle.load(f)
|
||||
# pprint(state.routing_table_peers)
|
||||
# pprint(state.datastore)
|
||||
print(f'{node_id[:8]}: loaded {len(state.routing_table_peers)} rt peers, {len(state.datastore)} in store')
|
||||
n.load_state(state)
|
||||
persisted_peers = state.routing_table_peers
|
||||
# if len(persisted_peers) == 0 and len(state.datastore) > 0:
|
||||
# peers_to_import = map(lambda p: (p[0], p[1], p[2], p[3]), n.get_state().datastore)
|
||||
# persisted_peers.extend(peers_to_import)
|
||||
# print(f'{node_id[:8]}: rt is empty but we recovered {len(state.datastore)} '
|
||||
# f'peers from the datastore. {len(peers_to_import)} of those were recent enough to import')
|
||||
|
||||
n.start("0.0.0.0", known_node_urls, persisted_peers)
|
||||
nodes.append(n)
|
||||
return nodes
|
||||
|
||||
|
||||
async def drain_events(n, q):
|
||||
# print(f'drain started on {bytes.hex(n.protocol.node_id)[:8]}')
|
||||
while True:
|
||||
(node_id, ip, method, args) = await n.protocol.event_queue.get()
|
||||
try:
|
||||
q.put_nowait((n, node_id, ip, method, args))
|
||||
except asyncio.QueueFull:
|
||||
pass
|
||||
|
||||
|
||||
async def run_web_api(loop, db, wallet_client):
|
||||
try:
|
||||
app = web.Application(loop=loop)
|
||||
app['db'] = db
|
||||
app['wallet_client'] = wallet_client
|
||||
app.add_routes([
|
||||
web.get('/', api_handler),
|
||||
web.get('/seeds/hash/{hash}', seeds_handler),
|
||||
web.get('/seeds/url/{url}', url_handler),
|
||||
])
|
||||
# server = web.Server(api_handler, loop=loop)
|
||||
# runner = web.ServerRunner(server)
|
||||
runner = web.AppRunner(app)
|
||||
await runner.setup()
|
||||
site = web.TCPSite(runner, 'localhost', 8080)
|
||||
await site.start()
|
||||
except Exception as err:
|
||||
pprint(err)
|
||||
await runner.cleanup()
|
||||
|
||||
|
||||
async def seeds_handler(request):
|
||||
blobhash = request.match_info['hash']
|
||||
db = request.app['db']
|
||||
try:
|
||||
cur = db.cursor()
|
||||
c = cur.execute("""
|
||||
select count(distinct(node_id)) from announce where hash = ? and timestamp > strftime('%s','now','-1 day')
|
||||
""", (blobhash,)).fetchone()[0]
|
||||
cur.close()
|
||||
return web.Response(text=json.dumps({'seeds': c})+"\n")
|
||||
except Exception as err:
|
||||
return web.Response(text=json.dumps({'error': err})+"\n")
|
||||
|
||||
|
||||
async def url_handler(request):
|
||||
url = unquote(request.match_info['url'])
|
||||
log.warning(url)
|
||||
db = request.app['db']
|
||||
wallet_client = request.app['wallet_client']
|
||||
|
||||
try:
|
||||
sd_hash = await get_sd_hash(wallet_client, url)
|
||||
if sd_hash is None:
|
||||
return web.Response(text=json.dumps({'error': 'Could not get sd hash for url', 'url': url})+"\n")
|
||||
seeds = get_seeds(db, sd_hash)
|
||||
return web.Response(text=json.dumps({'url': url, 'sd_hash': sd_hash, 'seeds': seeds})+"\n")
|
||||
except Exception as err:
|
||||
return web.Response(text=json.dumps({'error': err})+"\n")
|
||||
|
||||
|
||||
def get_seeds(db, blobhash):
|
||||
cur = db.cursor()
|
||||
c = cur.execute(
|
||||
"select count(distinct(node_id)) from announce where hash = ? and timestamp > strftime('%s','now','-1 day')",
|
||||
(blobhash,)
|
||||
).fetchone()[0]
|
||||
cur.close()
|
||||
return c
|
||||
|
||||
|
||||
async def get_sd_hash(wallet_client, url):
|
||||
try:
|
||||
resolved_txos = Outputs.from_base64(await wallet_client.send_request('blockchain.claimtrie.resolve', [url]))
|
||||
if not resolved_txos.txos:
|
||||
return None
|
||||
raw_txs = await wallet_client.send_request('blockchain.transaction.get_batch', [txid for (txid, height) in resolved_txos.txs])
|
||||
txo_proto = resolved_txos.txos[0]
|
||||
txid = txo_proto.tx_hash[::-1].hex()
|
||||
raw_tx_hex, _ = raw_txs[txid]
|
||||
txo = Transaction(bytes.fromhex(raw_tx_hex)).outputs[txo_proto.nout]
|
||||
return txo.claim.stream.source.sd_hash
|
||||
except Exception as err: # claim is not a stream, stream has no source, protobuf err, etc
|
||||
if isinstance(err, asyncio.CancelledError):
|
||||
raise err
|
||||
log.exception("failed to get sd_hash")
|
||||
return None
|
||||
|
||||
|
||||
async def api_handler(request):
|
||||
return web.Response(text="tracker OK")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
asyncio.run(main())
|
||||
except ShutdownErr:
|
||||
pass
|
||||
@@ -461,47 +461,6 @@ class TransactionCommands(ClaimTestCase):
|
||||
|
||||
class TransactionOutputCommands(ClaimTestCase):
|
||||
|
||||
async def test_txo_list_resolve_supports(self):
|
||||
channel = self.get_claim_id(await self.channel_create('@identity'))
|
||||
stream = self.get_claim_id(await self.stream_create())
|
||||
support = await self.support_create(stream, channel_id=channel)
|
||||
r, = await self.txo_list(type='support')
|
||||
self.assertEqual(r['txid'], support['txid'])
|
||||
self.assertNotIn('name', r['signing_channel'])
|
||||
r, = await self.txo_list(type='support', resolve=True)
|
||||
self.assertIn('name', r['signing_channel'])
|
||||
self.assertEqual(r['signing_channel']['name'], '@identity')
|
||||
|
||||
async def test_txo_list_by_channel_filtering(self):
|
||||
channel_foo = self.get_claim_id(await self.channel_create('@foo'))
|
||||
channel_bar = self.get_claim_id(await self.channel_create('@bar'))
|
||||
stream_a = self.get_claim_id(await self.stream_create('a', channel_id=channel_foo))
|
||||
stream_b = self.get_claim_id(await self.stream_create('b', channel_id=channel_foo))
|
||||
stream_c = self.get_claim_id(await self.stream_create('c', channel_id=channel_bar))
|
||||
stream_d = self.get_claim_id(await self.stream_create('d'))
|
||||
support_c = await self.support_create(stream_c, '0.3', channel_id=channel_foo)
|
||||
support_d = await self.support_create(stream_d, '0.3', channel_id=channel_bar)
|
||||
|
||||
r = await self.txo_list(type='stream')
|
||||
self.assertEqual({stream_a, stream_b, stream_c, stream_d}, {c['claim_id'] for c in r})
|
||||
|
||||
r = await self.txo_list(type='stream', channel_id=channel_foo)
|
||||
self.assertEqual({stream_a, stream_b}, {c['claim_id'] for c in r})
|
||||
|
||||
r = await self.txo_list(type='support', channel_id=channel_foo)
|
||||
self.assertEqual({support_c['txid']}, {s['txid'] for s in r})
|
||||
r = await self.txo_list(type='support', channel_id=channel_bar)
|
||||
self.assertEqual({support_d['txid']}, {s['txid'] for s in r})
|
||||
|
||||
r = await self.txo_list(type='stream', channel_id=[channel_foo, channel_bar])
|
||||
self.assertEqual({stream_a, stream_b, stream_c}, {c['claim_id'] for c in r})
|
||||
|
||||
r = await self.txo_list(type='stream', not_channel_id=channel_foo)
|
||||
self.assertEqual({stream_c, stream_d}, {c['claim_id'] for c in r})
|
||||
|
||||
r = await self.txo_list(type='stream', not_channel_id=[channel_foo, channel_bar])
|
||||
self.assertEqual({stream_d}, {c['claim_id'] for c in r})
|
||||
|
||||
async def test_txo_list_and_sum_filtering(self):
|
||||
channel_id = self.get_claim_id(await self.channel_create())
|
||||
self.assertEqual('1.0', lbc(await self.txo_sum(type='channel', is_not_spent=True)))
|
||||
|
||||
@@ -6,7 +6,6 @@ from unittest.mock import Mock
|
||||
from lbry.wallet.network import Network
|
||||
from lbry.wallet.orchstr8.node import SPVNode
|
||||
from lbry.wallet.rpc import RPCSession
|
||||
from lbry.wallet.server.udp import StatusServer
|
||||
from lbry.testcase import IntegrationTestCase, AsyncioTestCase
|
||||
|
||||
|
||||
@@ -33,17 +32,18 @@ class NetworkTests(IntegrationTestCase):
|
||||
'server_version': lbry.__version__,
|
||||
'trending_algorithm': 'zscore',
|
||||
}, await self.ledger.network.get_server_features())
|
||||
# await self.conductor.spv_node.stop()
|
||||
await self.conductor.spv_node.stop()
|
||||
payment_address, donation_address = await self.account.get_addresses(limit=2)
|
||||
self.conductor.spv_node.server.env.payment_address = payment_address
|
||||
self.conductor.spv_node.server.env.donation_address = donation_address
|
||||
self.conductor.spv_node.server.env.description = 'Fastest server in the west.'
|
||||
self.conductor.spv_node.server.env.daily_fee = '42'
|
||||
|
||||
from lbry.wallet.server.session import LBRYElectrumX
|
||||
LBRYElectrumX.set_server_features(self.conductor.spv_node.server.env)
|
||||
|
||||
# await self.ledger.network.on_connected.first
|
||||
await self.conductor.spv_node.start(
|
||||
self.conductor.blockchain_node,
|
||||
extraconf={
|
||||
'DESCRIPTION': 'Fastest server in the west.',
|
||||
'PAYMENT_ADDRESS': payment_address,
|
||||
'DONATION_ADDRESS': donation_address,
|
||||
'DAILY_FEE': '42'
|
||||
}
|
||||
)
|
||||
await self.ledger.network.on_connected.first
|
||||
self.assertDictEqual({
|
||||
'genesis_hash': self.conductor.spv_node.coin_class.GENESIS_HASH,
|
||||
'hash_function': 'sha256',
|
||||
@@ -65,21 +65,22 @@ class ReconnectTests(IntegrationTestCase):
|
||||
async def test_multiple_servers(self):
|
||||
# we have a secondary node that connects later, so
|
||||
node2 = SPVNode(self.conductor.spv_module, node_number=2)
|
||||
await node2.start(self.blockchain)
|
||||
|
||||
self.ledger.network.config['default_servers'].append((node2.hostname, node2.port))
|
||||
self.ledger.network.config['default_servers'].reverse()
|
||||
self.assertEqual(50002, self.ledger.network.client.server[1])
|
||||
await self.ledger.stop()
|
||||
await self.ledger.start()
|
||||
|
||||
await asyncio.wait_for(self.ledger.stop(), timeout=1)
|
||||
await asyncio.wait_for(self.ledger.start(), timeout=1)
|
||||
self.ledger.network.session_pool.new_connection_event.clear()
|
||||
await node2.start(self.blockchain)
|
||||
# this is only to speed up the test as retrying would take 4+ seconds
|
||||
for session in self.ledger.network.session_pool.sessions:
|
||||
session.trigger_urgent_reconnect.set()
|
||||
await asyncio.wait_for(self.ledger.network.session_pool.new_connection_event.wait(), timeout=1)
|
||||
self.assertEqual(2, len(list(self.ledger.network.session_pool.available_sessions)))
|
||||
self.assertTrue(self.ledger.network.is_connected)
|
||||
self.assertEqual(50003, self.ledger.network.client.server[1])
|
||||
await node2.stop(True)
|
||||
self.assertFalse(self.ledger.network.is_connected)
|
||||
await self.ledger.resolve([], ['derp'])
|
||||
self.assertEqual(50002, self.ledger.network.client.server[1])
|
||||
switch_event = self.ledger.network.on_connected.first
|
||||
await node2.stop(True)
|
||||
# secondary down, but primary is ok, do not switch! (switches trigger new on_connected events)
|
||||
with self.assertRaises(asyncio.TimeoutError):
|
||||
await asyncio.wait_for(switch_event, timeout=1)
|
||||
|
||||
async def test_direct_sync(self):
|
||||
await self.ledger.stop()
|
||||
@@ -97,13 +98,10 @@ class ReconnectTests(IntegrationTestCase):
|
||||
async def test_connection_drop_still_receives_events_after_reconnected(self):
|
||||
address1 = await self.account.receiving.get_or_create_usable_address()
|
||||
# disconnect and send a new tx, should reconnect and get it
|
||||
self.ledger.network.client.transport.close()
|
||||
self.ledger.network.client.connection_lost(Exception())
|
||||
self.assertFalse(self.ledger.network.is_connected)
|
||||
await self.ledger.resolve([], 'derp')
|
||||
sendtxid = await self.blockchain.send_to_address(address1, 1.1337)
|
||||
# await self.ledger.resolve([], 'derp')
|
||||
# self.assertTrue(self.ledger.network.is_connected)
|
||||
await asyncio.wait_for(self.on_transaction_id(sendtxid), 10.0) # mempool
|
||||
await asyncio.wait_for(self.on_transaction_id(sendtxid), 1.0) # mempool
|
||||
await self.blockchain.generate(1)
|
||||
await self.on_transaction_id(sendtxid) # confirmed
|
||||
self.assertLess(self.ledger.network.client.response_time, 1) # response time properly set lower, we are fine
|
||||
@@ -125,7 +123,7 @@ class ReconnectTests(IntegrationTestCase):
|
||||
await self.blockchain.generate(1)
|
||||
# (this is just so the test doesn't hang forever if it doesn't reconnect)
|
||||
if not self.ledger.network.is_connected:
|
||||
await asyncio.wait_for(self.ledger.network.on_connected.first, timeout=10.0)
|
||||
await asyncio.wait_for(self.ledger.network.on_connected.first, timeout=1.0)
|
||||
# omg, the burned cable still works! torba is fire proof!
|
||||
await self.ledger.network.get_transaction(sendtxid)
|
||||
|
||||
@@ -138,19 +136,15 @@ class ReconnectTests(IntegrationTestCase):
|
||||
await self.ledger.network.on_connected.first
|
||||
self.assertTrue(self.ledger.network.is_connected)
|
||||
|
||||
# async def test_online_but_still_unavailable(self):
|
||||
# # Edge case. See issue #2445 for context
|
||||
# self.assertIsNotNone(self.ledger.network.session_pool.fastest_session)
|
||||
# for session in self.ledger.network.session_pool.sessions:
|
||||
# session.response_time = None
|
||||
# self.assertIsNone(self.ledger.network.session_pool.fastest_session)
|
||||
async def test_online_but_still_unavailable(self):
|
||||
# Edge case. See issue #2445 for context
|
||||
self.assertIsNotNone(self.ledger.network.session_pool.fastest_session)
|
||||
for session in self.ledger.network.session_pool.sessions:
|
||||
session.response_time = None
|
||||
self.assertIsNone(self.ledger.network.session_pool.fastest_session)
|
||||
|
||||
|
||||
class ServerPickingTestCase(AsyncioTestCase):
|
||||
async def _make_udp_server(self, port):
|
||||
s = StatusServer()
|
||||
await s.start(0, b'\x00' * 32, '127.0.0.1', port)
|
||||
self.addCleanup(s.stop)
|
||||
|
||||
async def _make_fake_server(self, latency=1.0, port=1):
|
||||
# local fake server with artificial latency
|
||||
@@ -162,7 +156,6 @@ class ServerPickingTestCase(AsyncioTestCase):
|
||||
return {'height': 1}
|
||||
server = await self.loop.create_server(lambda: FakeSession(), host='127.0.0.1', port=port)
|
||||
self.addCleanup(server.close)
|
||||
await self._make_udp_server(port)
|
||||
return '127.0.0.1', port
|
||||
|
||||
async def _make_bad_server(self, port=42420):
|
||||
@@ -171,10 +164,9 @@ class ServerPickingTestCase(AsyncioTestCase):
|
||||
writer.write(await reader.read())
|
||||
server = await asyncio.start_server(echo, host='127.0.0.1', port=port)
|
||||
self.addCleanup(server.close)
|
||||
await self._make_udp_server(port)
|
||||
return '127.0.0.1', port
|
||||
|
||||
async def _test_pick_fastest(self):
|
||||
async def test_pick_fastest(self):
|
||||
ledger = Mock(config={
|
||||
'default_servers': [
|
||||
# fast but unhealthy, should be discarded
|
||||
@@ -190,8 +182,8 @@ class ServerPickingTestCase(AsyncioTestCase):
|
||||
|
||||
network = Network(ledger)
|
||||
self.addCleanup(network.stop)
|
||||
await network.start()
|
||||
await asyncio.wait_for(network.on_connected.first, timeout=10)
|
||||
asyncio.ensure_future(network.start())
|
||||
await asyncio.wait_for(network.on_connected.first, timeout=1)
|
||||
self.assertTrue(network.is_connected)
|
||||
self.assertTupleEqual(network.client.server, ('127.0.0.1', 1337))
|
||||
self.assertTrue(all([not session.is_closing() for session in network.session_pool.available_sessions]))
|
||||
|
||||
@@ -11,9 +11,7 @@ class PurchaseCommandTests(CommandTestCase):
|
||||
await super().asyncSetUp()
|
||||
self.merchant_address = await self.blockchain.get_raw_change_address()
|
||||
|
||||
async def priced_stream(
|
||||
self, name='stream', price: Optional[str] = '2.0', currency='LBC', mine=False
|
||||
) -> Transaction:
|
||||
async def priced_stream(self, name='stream', price: Optional[str] = '2.0', currency='LBC') -> Transaction:
|
||||
kwargs = {}
|
||||
if price and currency:
|
||||
kwargs = {
|
||||
@@ -21,8 +19,6 @@ class PurchaseCommandTests(CommandTestCase):
|
||||
'fee_currency': currency,
|
||||
'fee_address': self.merchant_address
|
||||
}
|
||||
if not mine:
|
||||
kwargs['claim_address'] = self.merchant_address
|
||||
file_path = self.create_upload_file(data=b'high value content')
|
||||
tx = await self.daemon.jsonrpc_stream_create(
|
||||
name, '0.01', file_path=file_path, **kwargs
|
||||
@@ -186,12 +182,3 @@ class PurchaseCommandTests(CommandTestCase):
|
||||
await self.ledger.wait(spend)
|
||||
await self.assertBalance(self.account, '0.487695')
|
||||
self.assertItemCount(await self.daemon.jsonrpc_utxo_list(), 1)
|
||||
|
||||
async def test_owner_not_required_purchase_own_content(self):
|
||||
await self.priced_stream(mine=True)
|
||||
# check that `get` doesn't purchase own claim
|
||||
balance = await self.account.get_balance()
|
||||
response = await self.daemon.jsonrpc_get('lbry://stream')
|
||||
self.assertIsNone(response.content_fee)
|
||||
self.assertEqual(await self.account.get_balance(), balance)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_purchase_list(), 0)
|
||||
@@ -43,14 +43,12 @@ class WalletCommands(CommandTestCase):
|
||||
)
|
||||
|
||||
async def test_wallet_reconnect(self):
|
||||
status = await self.daemon.jsonrpc_status()
|
||||
self.assertEqual(len(status['wallet']['servers']), 1)
|
||||
self.assertEqual(status['wallet']['servers'][0]['port'], 50002)
|
||||
await self.conductor.spv_node.stop(True)
|
||||
self.conductor.spv_node.port = 54320
|
||||
await self.conductor.spv_node.start(self.conductor.blockchain_node)
|
||||
status = await self.daemon.jsonrpc_status()
|
||||
self.assertEqual(len(status['wallet']['servers']), 0)
|
||||
self.assertEqual(len(status['wallet']['servers']), 1)
|
||||
self.assertEqual(status['wallet']['servers'][0]['port'], 50002)
|
||||
self.daemon.jsonrpc_settings_set('lbryum_servers', ['localhost:54320'])
|
||||
await self.daemon.jsonrpc_wallet_reconnect()
|
||||
status = await self.daemon.jsonrpc_status()
|
||||
|
||||
@@ -4,7 +4,6 @@ import lbry
|
||||
import lbry.wallet
|
||||
from lbry.error import ServerPaymentFeeAboveMaxAllowedError
|
||||
from lbry.wallet.network import ClientSession
|
||||
from lbry.wallet.server.session import LBRYElectrumX
|
||||
from lbry.testcase import IntegrationTestCase, CommandTestCase
|
||||
from lbry.wallet.orchstr8.node import SPVNode
|
||||
|
||||
@@ -47,7 +46,7 @@ class TestSessions(IntegrationTestCase):
|
||||
|
||||
|
||||
class TestUsagePayment(CommandTestCase):
|
||||
async def _test_single_server_payment(self):
|
||||
async def test_single_server_payment(self):
|
||||
wallet_pay_service = self.daemon.component_manager.get_component('wallet_server_payments')
|
||||
wallet_pay_service.payment_period = 1
|
||||
# only starts with a positive max key fee
|
||||
@@ -69,21 +68,21 @@ class TestUsagePayment(CommandTestCase):
|
||||
self.addCleanup(node.stop)
|
||||
self.daemon.jsonrpc_settings_set('lbryum_servers', [f"{node.hostname}:{node.port}"])
|
||||
await self.daemon.jsonrpc_wallet_reconnect()
|
||||
LBRYElectrumX.set_server_features(node.server.env)
|
||||
|
||||
features = await self.ledger.network.get_server_features()
|
||||
self.assertEqual(features["payment_address"], address)
|
||||
self.assertEqual(features["daily_fee"], "1.1")
|
||||
with self.assertRaises(ServerPaymentFeeAboveMaxAllowedError):
|
||||
await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=30)
|
||||
node.server.env.daily_fee = "1.0"
|
||||
node.server.env.payment_address = address
|
||||
LBRYElectrumX.set_server_features(node.server.env)
|
||||
# self.daemon.jsonrpc_settings_set('lbryum_servers', [f"{node.hostname}:{node.port}"])
|
||||
await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=8)
|
||||
|
||||
await node.stop(False)
|
||||
await node.start(self.blockchain, extraconf={"PAYMENT_ADDRESS": address, "DAILY_FEE": "1.0"})
|
||||
self.daemon.jsonrpc_settings_set('lbryum_servers', [f"{node.hostname}:{node.port}"])
|
||||
await self.daemon.jsonrpc_wallet_reconnect()
|
||||
features = await self.ledger.network.get_server_features()
|
||||
self.assertEqual(features["payment_address"], address)
|
||||
self.assertEqual(features["daily_fee"], "1.0")
|
||||
tx = await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=30)
|
||||
tx = await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=8)
|
||||
self.assertIsNotNone(await self.blockchain.get_raw_transaction(tx.id)) # verify its broadcasted
|
||||
self.assertEqual(tx.outputs[0].amount, 100000000)
|
||||
self.assertEqual(tx.outputs[0].get_address(self.ledger), address)
|
||||
@@ -9,10 +9,6 @@ from lbry.wallet import Transaction
|
||||
|
||||
|
||||
class FileCommands(CommandTestCase):
|
||||
def __init__(self, *a, **kw):
|
||||
super().__init__(*a, **kw)
|
||||
self.skip_libtorrent = False
|
||||
|
||||
async def initialize_torrent(self, tx_to_update=None):
|
||||
if not hasattr(self, 'seeder_session'):
|
||||
self.seeder_session = TorrentSession(self.loop, None)
|
||||
@@ -358,8 +354,7 @@ class FileCommands(CommandTestCase):
|
||||
# FAIL: beyond available balance
|
||||
await self.stream_create(
|
||||
'expensive', '0.01', data=b'pay me if you can',
|
||||
fee_currency='LBC', fee_amount='11.0',
|
||||
fee_address=target_address, claim_address=target_address
|
||||
fee_currency='LBC', fee_amount='11.0', fee_address=target_address
|
||||
)
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='expensive')
|
||||
response = await self.out(self.daemon.jsonrpc_get('lbry://expensive'))
|
||||
@@ -369,8 +364,7 @@ class FileCommands(CommandTestCase):
|
||||
# FAIL: beyond maximum key fee
|
||||
await self.stream_create(
|
||||
'maxkey', '0.01', data=b'no pay me, no',
|
||||
fee_currency='LBC', fee_amount='111.0',
|
||||
fee_address=target_address, claim_address=target_address
|
||||
fee_currency='LBC', fee_amount='111.0', fee_address=target_address
|
||||
)
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='maxkey')
|
||||
response = await self.out(self.daemon.jsonrpc_get('lbry://maxkey'))
|
||||
@@ -382,8 +376,7 @@ class FileCommands(CommandTestCase):
|
||||
# PASS: purchase is successful
|
||||
await self.stream_create(
|
||||
'icanpay', '0.01', data=b'I got the power!',
|
||||
fee_currency='LBC', fee_amount='1.0',
|
||||
fee_address=target_address, claim_address=target_address
|
||||
fee_currency='LBC', fee_amount='1.0', fee_address=target_address
|
||||
)
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='icanpay')
|
||||
await self.assertBalance(self.account, '9.925679')
|
||||
@@ -434,8 +427,7 @@ class FileCommands(CommandTestCase):
|
||||
|
||||
await self.stream_create(
|
||||
'somename', '0.5', data=b'Yes, please',
|
||||
fee_currency='LBC', fee_amount='1.0',
|
||||
fee_address=target_address, claim_address=target_address
|
||||
fee_currency='LBC', fee_amount='1.0', fee_address=target_address
|
||||
)
|
||||
self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='somename'))
|
||||
# Assert the fee and bid are subtracted
|
||||
|
||||
@@ -7,8 +7,7 @@ from lbry.extras import cli
|
||||
from lbry.extras.daemon.components import (
|
||||
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
|
||||
HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT,
|
||||
LIBTORRENT_COMPONENT
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT
|
||||
)
|
||||
from lbry.extras.daemon.daemon import Daemon
|
||||
|
||||
@@ -23,8 +22,7 @@ class CLIIntegrationTest(AsyncioTestCase):
|
||||
conf.components_to_skip = (
|
||||
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
|
||||
HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT,
|
||||
LIBTORRENT_COMPONENT
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT
|
||||
)
|
||||
Daemon.component_attributes = {}
|
||||
self.daemon = Daemon(conf)
|
||||
|
||||
Reference in new issue
Block a user