Compare commits
58
Commits
fix_order2
...
v0.88.0
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3b27cb3671 | ||
|
|
3fe0db4a7d | ||
|
|
8b55814ab2 | ||
|
|
575e471553 | ||
|
|
0f5f1aebed | ||
|
|
50e17eb1ab | ||
|
|
158cc2f660 | ||
|
|
1066a31acd | ||
|
|
1f9d0f4582 | ||
|
|
a6d65233f1 | ||
|
|
eff2fe7a1b | ||
|
|
20efdc70b3 | ||
|
|
f0d8fb8f1a | ||
|
|
f7a380e9b7 | ||
|
|
e9c7cf6f63 | ||
|
|
68f1661452 | ||
|
|
36fd1b91ae | ||
|
|
a4ec430ac0 | ||
|
|
519614b2fd | ||
|
|
bf0118c8ef | ||
|
|
a4db0820bc | ||
|
|
ee7528413e | ||
|
|
7952fc8324 | ||
|
|
652773d2cf | ||
|
|
2a17787242 | ||
|
|
0a53ad5721 | ||
|
|
6da6bdc863 | ||
|
|
42ad2bb83f | ||
|
|
f309a65cb4 | ||
|
|
77e19ab1a4 | ||
|
|
1a996b6ef3 | ||
|
|
b882f1a010 | ||
|
|
82a030e6ff | ||
|
|
0758b85179 | ||
|
|
ab3d9bd080 | ||
|
|
8ff813f689 | ||
|
|
88ae72c0d3 | ||
|
|
312aa4be26 | ||
|
|
cbb06fce9d | ||
|
|
f259e497c4 | ||
|
|
dd4172ac66 | ||
|
|
66029e60d3 | ||
|
|
364f484f04 | ||
|
|
9dd5159414 | ||
|
|
13e38d6fd8 | ||
|
|
10dcb64715 | ||
|
|
7551b51e7d | ||
|
|
adb418aafc | ||
|
|
270da80d64 | ||
|
|
b2027cfd66 | ||
|
|
7f1f4eeac6 | ||
|
|
7a7446c8bd | ||
|
|
ddbae294e6 | ||
|
|
8c71b744f3 | ||
|
|
479b5d31a9 | ||
|
|
4cbf4230e8 | ||
|
|
6a610187e0 | ||
|
|
eb2a4aebba |
No files matched your search
+8
-6
@@ -57,11 +57,11 @@ test:other-integration:
|
||||
- pip install tox-travis
|
||||
- tox -e other
|
||||
|
||||
test:json-api:
|
||||
stage: test
|
||||
script:
|
||||
- make install tools
|
||||
- HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
|
||||
#test:json-api:
|
||||
# stage: test
|
||||
# script:
|
||||
# - make install tools
|
||||
# - HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
|
||||
|
||||
|
||||
|
||||
@@ -121,9 +121,11 @@ build:windows:
|
||||
GIT_DEPTH: 5
|
||||
before_script:
|
||||
- ./docker/install_choco.ps1
|
||||
- choco install -y --x86 python3 7zip checksum
|
||||
- choco install -y --x64 python --version=3.7.9
|
||||
- choco install -y 7zip checksum
|
||||
- python --version # | findstr /B "Python 3\.7\." # dont upgrade python on every run. just make sure we're on the right Python
|
||||
- pip --version # | findstr /E '\(python 3\.7\)'
|
||||
- python -c "import sys;print(f'{str(64 if sys.maxsize > 2**32 else 32)} bit python\n{sys.platform}')"
|
||||
- pip install virtualenv pywin32
|
||||
- virtualenv venv
|
||||
- venv/Scripts/activate.ps1
|
||||
|
||||
+1
-1
@@ -1,2 +1,2 @@
|
||||
__version__ = "0.86.1"
|
||||
__version__ = "0.88.0"
|
||||
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
|
||||
@@ -2,7 +2,7 @@ import os
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
from lbry.utils import LRUCache
|
||||
from lbry.utils import LRUCacheWithMetrics
|
||||
from lbry.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
|
||||
from lbry.stream.descriptor import StreamDescriptor
|
||||
from lbry.connection_manager import ConnectionManager
|
||||
@@ -32,7 +32,7 @@ class BlobManager:
|
||||
else self._node_data_store.completed_blobs
|
||||
self.blobs: typing.Dict[str, AbstractBlob] = {}
|
||||
self.config = config
|
||||
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCache(
|
||||
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCacheWithMetrics(
|
||||
self.config.blob_lru_cache_size)
|
||||
self.connection_manager = ConnectionManager(loop)
|
||||
|
||||
|
||||
+1
-1
@@ -638,7 +638,7 @@ class Config(CLIConfig):
|
||||
"Strategy to use when selecting UTXOs for a transaction",
|
||||
STRATEGIES, "standard")
|
||||
|
||||
transaction_cache_size = Integer("Transaction cache size", 100_000)
|
||||
transaction_cache_size = Integer("Transaction cache size", 2 ** 17)
|
||||
save_resolved_claims = Toggle(
|
||||
"Save content claims to the database when they are resolved to keep file_list up to date, "
|
||||
"only disable this if file_x commands are not needed", True
|
||||
|
||||
+3
-22
@@ -1,14 +1,14 @@
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
import ipaddress
|
||||
from binascii import hexlify
|
||||
from dataclasses import dataclass, field
|
||||
from functools import lru_cache
|
||||
|
||||
from lbry.utils import is_valid_public_ipv4 as _is_valid_public_ipv4
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
|
||||
|
||||
ALLOW_LOCALHOST = False
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -20,28 +20,9 @@ def make_kademlia_peer(node_id: typing.Optional[bytes], address: typing.Optional
|
||||
return KademliaPeer(address, node_id, udp_port, tcp_port=tcp_port, allow_localhost=allow_localhost)
|
||||
|
||||
|
||||
# the ipaddress module does not show these subnets as reserved
|
||||
CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
|
||||
IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
|
||||
|
||||
ALLOW_LOCALHOST = False
|
||||
|
||||
|
||||
def is_valid_public_ipv4(address, allow_localhost: bool = False):
|
||||
allow_localhost = bool(allow_localhost or ALLOW_LOCALHOST)
|
||||
try:
|
||||
parsed_ip = ipaddress.ip_address(address)
|
||||
if parsed_ip.is_loopback and allow_localhost:
|
||||
return True
|
||||
|
||||
if any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local, parsed_ip.is_loopback,
|
||||
parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private, parsed_ip.is_reserved)):
|
||||
return False
|
||||
else:
|
||||
return not any((CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
|
||||
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
|
||||
except (ipaddress.AddressValueError, ValueError):
|
||||
return False
|
||||
return _is_valid_public_ipv4(address, allow_localhost)
|
||||
|
||||
|
||||
class PeerManager:
|
||||
|
||||
@@ -132,7 +132,7 @@ class AnalyticsManager:
|
||||
async def run(self):
|
||||
while True:
|
||||
if self.enabled:
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
await self._send_heartbeat()
|
||||
await asyncio.sleep(1800)
|
||||
|
||||
|
||||
@@ -26,42 +26,45 @@ def is_comment_signed_by_channel(comment: dict, channel: Output, sign_comment_id
|
||||
if isinstance(channel, Output):
|
||||
try:
|
||||
signing_field = comment['comment_id'] if sign_comment_id else comment['comment']
|
||||
pieces = [
|
||||
comment['signing_ts'].encode(),
|
||||
cid2hash(comment['channel_id']),
|
||||
signing_field.encode()
|
||||
]
|
||||
return Output.is_signature_valid(
|
||||
get_encoded_signature(comment['signature']),
|
||||
sha256(b''.join(pieces)),
|
||||
channel.claim.channel.public_key_bytes
|
||||
)
|
||||
return verify(channel, signing_field.encode(), comment, cid2hash(comment['channel_id']))
|
||||
except KeyError:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def verify(channel, data, signature, channel_hash=None):
|
||||
pieces = [
|
||||
signature['signing_ts'].encode(),
|
||||
channel_hash or channel.claim_hash,
|
||||
data
|
||||
]
|
||||
return Output.is_signature_valid(
|
||||
get_encoded_signature(signature['signature']),
|
||||
sha256(b''.join(pieces)),
|
||||
channel.claim.channel.public_key_bytes
|
||||
)
|
||||
|
||||
|
||||
def sign_comment(comment: dict, channel: Output, sign_comment_id=False):
|
||||
timestamp = str(int(time.time()))
|
||||
signing_field = comment['comment_id'] if sign_comment_id else comment['comment']
|
||||
pieces = [timestamp.encode(), channel.claim_hash, signing_field.encode()]
|
||||
comment.update(sign(channel, signing_field.encode()))
|
||||
|
||||
|
||||
def sign(channel, data):
|
||||
timestamp = str(int(time.time()))
|
||||
pieces = [timestamp.encode(), channel.claim_hash, data]
|
||||
digest = sha256(b''.join(pieces))
|
||||
signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
|
||||
comment.update({
|
||||
return {
|
||||
'signature': binascii.hexlify(signature).decode(),
|
||||
'signing_ts': timestamp
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def sign_reaction(reaction: dict, channel: Output):
|
||||
timestamp = str(int(time.time()))
|
||||
signing_field = reaction['channel_name']
|
||||
pieces = [timestamp.encode(), channel.claim_hash, signing_field.encode()]
|
||||
digest = sha256(b''.join(pieces))
|
||||
signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
|
||||
reaction.update({
|
||||
'signature': binascii.hexlify(signature).decode(),
|
||||
'signing_ts': timestamp
|
||||
})
|
||||
reaction.update(sign(channel, signing_field.encode()))
|
||||
|
||||
|
||||
async def jsonrpc_post(url: str, method: str, params: dict = None, **kwargs) -> any:
|
||||
params = params or {}
|
||||
|
||||
@@ -119,13 +119,14 @@ class WalletComponent(Component):
|
||||
async def get_status(self):
|
||||
if self.wallet_manager is None:
|
||||
return
|
||||
session_pool = self.wallet_manager.ledger.network.session_pool
|
||||
sessions = session_pool.sessions
|
||||
is_connected = self.wallet_manager.ledger.network.is_connected
|
||||
sessions = []
|
||||
connected = None
|
||||
if self.wallet_manager.ledger.network.client:
|
||||
addr_and_port = self.wallet_manager.ledger.network.client.server_address_and_port
|
||||
if addr_and_port:
|
||||
connected = f"{addr_and_port[0]}:{addr_and_port[1]}"
|
||||
if is_connected:
|
||||
addr, port = self.wallet_manager.ledger.network.client.server
|
||||
connected = f"{addr}:{port}"
|
||||
sessions.append(self.wallet_manager.ledger.network.client)
|
||||
|
||||
result = {
|
||||
'connected': connected,
|
||||
'connected_features': self.wallet_manager.ledger.network.server_features,
|
||||
@@ -137,8 +138,8 @@ class WalletComponent(Component):
|
||||
'availability': session.available,
|
||||
} for session in sessions
|
||||
],
|
||||
'known_servers': len(sessions),
|
||||
'available_servers': len(list(session_pool.available_sessions))
|
||||
'known_servers': len(self.wallet_manager.ledger.network.config['default_servers']),
|
||||
'available_servers': 1 if is_connected else 0
|
||||
}
|
||||
|
||||
if self.wallet_manager.ledger.network.remote_height:
|
||||
@@ -274,7 +275,7 @@ class DHTComponent(Component):
|
||||
external_ip = upnp_component.external_ip
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
if not external_ip:
|
||||
external_ip = await utils.get_external_ip()
|
||||
external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
if not external_ip:
|
||||
log.warning("failed to get external ip")
|
||||
|
||||
@@ -328,7 +329,7 @@ class HashAnnouncerComponent(Component):
|
||||
|
||||
class FileManagerComponent(Component):
|
||||
component_name = FILE_MANAGER_COMPONENT
|
||||
depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT, LIBTORRENT_COMPONENT]
|
||||
depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
@@ -351,7 +352,10 @@ class FileManagerComponent(Component):
|
||||
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
node = self.component_manager.get_component(DHT_COMPONENT) \
|
||||
if self.component_manager.has_component(DHT_COMPONENT) else None
|
||||
torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT) if TorrentSession else None
|
||||
try:
|
||||
torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT) if TorrentSession else None
|
||||
except NameError:
|
||||
torrent = None
|
||||
log.info('Starting the file manager')
|
||||
loop = asyncio.get_event_loop()
|
||||
self.file_manager = FileManager(
|
||||
@@ -360,7 +364,7 @@ class FileManagerComponent(Component):
|
||||
self.file_manager.source_managers['stream'] = StreamManager(
|
||||
loop, self.conf, blob_manager, wallet, storage, node,
|
||||
)
|
||||
if TorrentSession:
|
||||
if TorrentSession and LIBTORRENT_COMPONENT not in self.conf.components_to_skip:
|
||||
self.file_manager.source_managers['torrent'] = TorrentManager(
|
||||
loop, self.conf, torrent, storage, self.component_manager.analytics_manager
|
||||
)
|
||||
@@ -472,7 +476,7 @@ class UPnPComponent(Component):
|
||||
pass
|
||||
if external_ip and not is_valid_public_ipv4(external_ip):
|
||||
log.warning("UPnP returned a private/reserved ip - %s, checking lbry.com fallback", external_ip)
|
||||
external_ip = await utils.get_external_ip()
|
||||
external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
if self.external_ip and self.external_ip != external_ip:
|
||||
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
|
||||
if external_ip:
|
||||
@@ -530,7 +534,7 @@ class UPnPComponent(Component):
|
||||
async def start(self):
|
||||
log.info("detecting external ip")
|
||||
if not self.use_upnp:
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
return
|
||||
success = False
|
||||
await self._maintain_redirects()
|
||||
@@ -545,9 +549,9 @@ class UPnPComponent(Component):
|
||||
else:
|
||||
log.error("failed to setup upnp")
|
||||
if not self.external_ip:
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
self.external_ip, probed_url = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
if self.external_ip:
|
||||
log.info("detected external ip using lbry.com fallback")
|
||||
log.info("detected external ip using %s fallback", probed_url)
|
||||
if self.component_manager.analytics_manager:
|
||||
self.component_manager.loop.create_task(
|
||||
self.component_manager.analytics_manager.send_upnp_setup_success_fail(
|
||||
|
||||
@@ -21,6 +21,7 @@ import base58
|
||||
from aiohttp import web
|
||||
from prometheus_client import generate_latest as prom_generate_latest, Gauge, Histogram, Counter
|
||||
from google.protobuf.message import DecodeError
|
||||
|
||||
from lbry.wallet import (
|
||||
Wallet, ENCRYPT_ON_DISK, SingleKey, HierarchicalDeterministic,
|
||||
Transaction, Output, Input, Account, database
|
||||
@@ -2788,6 +2789,34 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
return tx
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
async def jsonrpc_channel_sign(
|
||||
self, channel_name=None, channel_id=None, hexdata=None, channel_account_id=None, wallet_id=None):
|
||||
"""
|
||||
Signs data using the specified channel signing key.
|
||||
|
||||
Usage:
|
||||
channel_sign [<channel_name> | --channel_name=<channel_name>]
|
||||
[<channel_id> | --channel_id=<channel_id>] [<hexdata> | --hexdata=<hexdata>]
|
||||
[--channel_account_id=<channel_account_id>...] [--wallet_id=<wallet_id>]
|
||||
|
||||
Options:
|
||||
--channel_name=<channel_name> : (str) name of channel used to sign (or use channel id)
|
||||
--channel_id=<channel_id> : (str) claim id of channel used to sign (or use channel name)
|
||||
--hexdata=<hexdata> : (str) data to sign, encoded as hexadecimal
|
||||
--channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in
|
||||
for channel certificates, defaults to all accounts.
|
||||
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
|
||||
|
||||
Returns: {}
|
||||
"""
|
||||
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
|
||||
assert not wallet.is_locked, "Cannot spend funds with locked wallet, unlock first."
|
||||
signing_channel = await self.get_channel_or_error(
|
||||
wallet, channel_account_id, channel_id, channel_name, for_signing=True
|
||||
)
|
||||
return comment_client.sign(signing_channel, unhexlify(hexdata))
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
async def jsonrpc_channel_abandon(
|
||||
self, claim_id=None, txid=None, nout=None, account_id=None, wallet_id=None,
|
||||
@@ -4313,7 +4342,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
@staticmethod
|
||||
def _constrain_txo_from_kwargs(
|
||||
constraints, type=None, txid=None, # pylint: disable=redefined-builtin
|
||||
claim_id=None, channel_id=None, name=None, reposted_claim_id=None,
|
||||
claim_id=None, channel_id=None, not_channel_id=None,
|
||||
name=None, reposted_claim_id=None,
|
||||
is_spent=False, is_not_spent=False,
|
||||
is_my_input_or_output=None, exclude_internal_transfers=False,
|
||||
is_my_output=None, is_not_my_output=None,
|
||||
@@ -4336,6 +4366,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
constraints['is_my_output'] = False
|
||||
database.constrain_single_or_list(constraints, 'txo_type', type, lambda x: TXO_TYPES[x])
|
||||
database.constrain_single_or_list(constraints, 'channel_id', channel_id)
|
||||
database.constrain_single_or_list(constraints, 'channel_id', not_channel_id, negate=True)
|
||||
database.constrain_single_or_list(constraints, 'claim_id', claim_id)
|
||||
database.constrain_single_or_list(constraints, 'claim_name', name)
|
||||
database.constrain_single_or_list(constraints, 'txid', txid)
|
||||
@@ -4350,9 +4381,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
List my transaction outputs.
|
||||
|
||||
Usage:
|
||||
txo_list [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...]
|
||||
[--claim_id=<claim_id>...] [--channel_id=<channel_id>...] [--name=<name>...]
|
||||
[--is_spent | --is_not_spent]
|
||||
txo_list [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...] [--claim_id=<claim_id>...]
|
||||
[--channel_id=<channel_id>...] [--not_channel_id=<not_channel_id>...]
|
||||
[--name=<name>...] [--is_spent | --is_not_spent]
|
||||
[--is_my_input_or_output |
|
||||
[[--is_my_output | --is_not_my_output] [--is_my_input | --is_not_my_input]]
|
||||
]
|
||||
@@ -4366,6 +4397,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--txid=<txid> : (str or list) transaction id of outputs
|
||||
--claim_id=<claim_id> : (str or list) claim id
|
||||
--channel_id=<channel_id> : (str or list) claims in this channel
|
||||
--not_channel_id=<not_channel_id>: (str or list) claims not in this channel
|
||||
--name=<name> : (str or list) claim name
|
||||
--is_spent : (bool) only show spent txos
|
||||
--is_not_spent : (bool) only show not spent txos
|
||||
@@ -4425,9 +4457,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
Spend transaction outputs, batching into multiple transactions as necessary.
|
||||
|
||||
Usage:
|
||||
txo_spend [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...]
|
||||
[--claim_id=<claim_id>...] [--channel_id=<channel_id>...] [--name=<name>...]
|
||||
[--is_my_input | --is_not_my_input]
|
||||
txo_spend [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...] [--claim_id=<claim_id>...]
|
||||
[--channel_id=<channel_id>...] [--not_channel_id=<not_channel_id>...]
|
||||
[--name=<name>...] [--is_my_input | --is_not_my_input]
|
||||
[--exclude_internal_transfers] [--wallet_id=<wallet_id>]
|
||||
[--preview] [--blocking] [--batch_size=<batch_size>] [--include_full_tx]
|
||||
|
||||
@@ -4437,6 +4469,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--txid=<txid> : (str or list) transaction id of outputs
|
||||
--claim_id=<claim_id> : (str or list) claim id
|
||||
--channel_id=<channel_id> : (str or list) claims in this channel
|
||||
--not_channel_id=<not_channel_id>: (str or list) claims not in this channel
|
||||
--name=<name> : (str or list) claim name
|
||||
--is_my_input : (bool) show outputs created by you
|
||||
--is_not_my_input : (bool) show outputs not created by you
|
||||
@@ -4481,6 +4514,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
Usage:
|
||||
txo_list [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...]
|
||||
[--channel_id=<channel_id>...] [--not_channel_id=<not_channel_id>...]
|
||||
[--claim_id=<claim_id>...] [--name=<name>...]
|
||||
[--is_spent] [--is_not_spent]
|
||||
[--is_my_input_or_output |
|
||||
@@ -4494,6 +4528,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--txid=<txid> : (str or list) transaction id of outputs
|
||||
--claim_id=<claim_id> : (str or list) claim id
|
||||
--name=<name> : (str or list) claim name
|
||||
--channel_id=<channel_id> : (str or list) claims in this channel
|
||||
--not_channel_id=<not_channel_id>: (str or list) claims not in this channel
|
||||
--is_spent : (bool) only show spent txos
|
||||
--is_not_spent : (bool) only show not spent txos
|
||||
--is_my_input_or_output : (bool) txos which have your inputs or your outputs,
|
||||
@@ -4528,6 +4564,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
Usage:
|
||||
txo_plot [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...]
|
||||
[--claim_id=<claim_id>...] [--name=<name>...] [--is_spent] [--is_not_spent]
|
||||
[--channel_id=<channel_id>...] [--not_channel_id=<not_channel_id>...]
|
||||
[--is_my_input_or_output |
|
||||
[[--is_my_output | --is_not_my_output] [--is_my_input | --is_not_my_input]]
|
||||
]
|
||||
@@ -4542,6 +4579,8 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--txid=<txid> : (str or list) transaction id of outputs
|
||||
--claim_id=<claim_id> : (str or list) claim id
|
||||
--name=<name> : (str or list) claim name
|
||||
--channel_id=<channel_id> : (str or list) claims in this channel
|
||||
--not_channel_id=<not_channel_id>: (str or list) claims not in this channel
|
||||
--is_spent : (bool) only show spent txos
|
||||
--is_not_spent : (bool) only show not spent txos
|
||||
--is_my_input_or_output : (bool) txos which have your inputs or your outputs,
|
||||
|
||||
@@ -85,8 +85,11 @@ class FileManager:
|
||||
raise ResolveError("cannot download a channel claim, specify a /path")
|
||||
try:
|
||||
resolved_result = await asyncio.wait_for(
|
||||
self.wallet_manager.ledger.resolve(wallet.accounts, [uri], include_purchase_receipt=True),
|
||||
resolve_timeout
|
||||
self.wallet_manager.ledger.resolve(
|
||||
wallet.accounts, [uri],
|
||||
include_purchase_receipt=True,
|
||||
include_is_my_output=True
|
||||
), resolve_timeout
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
raise ResolveTimeoutError(uri)
|
||||
@@ -170,7 +173,14 @@ class FileManager:
|
||||
# pay fee
|
||||
####################
|
||||
|
||||
if not to_replace and txo.has_price and not txo.purchase_receipt:
|
||||
needs_purchasing = (
|
||||
not to_replace and
|
||||
not txo.is_my_output and
|
||||
txo.has_price and
|
||||
not txo.purchase_receipt
|
||||
)
|
||||
|
||||
if needs_purchasing:
|
||||
payment = await self.wallet_manager.create_purchase_transaction(
|
||||
wallet.accounts, txo, exchange_rate_manager
|
||||
)
|
||||
|
||||
+37
-1
@@ -1,14 +1,44 @@
|
||||
import time
|
||||
import logging
|
||||
import asyncio
|
||||
import asyncio.tasks
|
||||
from aiohttp import web
|
||||
from prometheus_client import generate_latest as prom_generate_latest
|
||||
from prometheus_client import Counter, Histogram, Gauge
|
||||
|
||||
|
||||
PROBES_IN_FLIGHT = Counter("probes_in_flight", "Number of loop probes in flight", namespace='asyncio')
|
||||
PROBES_FINISHED = Counter("probes_finished", "Number of finished loop probes", namespace='asyncio')
|
||||
PROBE_TIMES = Histogram("probe_times", "Loop probe times", namespace='asyncio')
|
||||
TASK_COUNT = Gauge("running_tasks", "Number of running tasks", namespace='asyncio')
|
||||
|
||||
|
||||
def get_loop_metrics(delay=1):
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def callback(started):
|
||||
PROBE_TIMES.observe(time.perf_counter() - started - delay)
|
||||
PROBES_FINISHED.inc()
|
||||
|
||||
async def monitor_loop_responsiveness():
|
||||
while True:
|
||||
now = time.perf_counter()
|
||||
loop.call_later(delay, callback, now)
|
||||
PROBES_IN_FLIGHT.inc()
|
||||
TASK_COUNT.set(len(asyncio.tasks._all_tasks))
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
return loop.create_task(monitor_loop_responsiveness())
|
||||
|
||||
|
||||
class PrometheusServer:
|
||||
def __init__(self, logger=None):
|
||||
self.runner = None
|
||||
self.logger = logger or logging.getLogger(__name__)
|
||||
self._monitor_loop_task = None
|
||||
|
||||
async def start(self, interface: str, port: int):
|
||||
self.logger.info("start prometheus metrics")
|
||||
prom_app = web.Application()
|
||||
prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
|
||||
self.runner = web.AppRunner(prom_app)
|
||||
@@ -16,7 +46,10 @@ class PrometheusServer:
|
||||
|
||||
metrics_site = web.TCPSite(self.runner, interface, port, shutdown_timeout=.5)
|
||||
await metrics_site.start()
|
||||
self.logger.info('metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2])
|
||||
self.logger.info(
|
||||
'prometheus metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2]
|
||||
)
|
||||
self._monitor_loop_task = get_loop_metrics()
|
||||
|
||||
async def handle_metrics_get_request(self, request: web.Request):
|
||||
try:
|
||||
@@ -29,4 +62,7 @@ class PrometheusServer:
|
||||
raise
|
||||
|
||||
async def stop(self):
|
||||
if self._monitor_loop_task and not self._monitor_loop_task.done():
|
||||
self._monitor_loop_task.cancel()
|
||||
self._monitor_loop_task = None
|
||||
await self.runner.cleanup()
|
||||
@@ -40,7 +40,7 @@ class StreamDownloader:
|
||||
async def cached_read_blob(blob_info: 'BlobInfo') -> bytes:
|
||||
return await self.read_blob(blob_info, 2)
|
||||
|
||||
if self.blob_manager.decrypted_blob_lru_cache:
|
||||
if self.blob_manager.decrypted_blob_lru_cache is not None:
|
||||
cached_read_blob = lru_cache_concurrent(override_lru_cache=self.blob_manager.decrypted_blob_lru_cache)(
|
||||
cached_read_blob
|
||||
)
|
||||
|
||||
@@ -354,20 +354,19 @@ class ManagedStream(ManagedDownloadSource):
|
||||
self.reflector_progress = int((i + 1) / len(we_have) * 100)
|
||||
except (asyncio.TimeoutError, ValueError):
|
||||
return sent
|
||||
except ConnectionRefusedError:
|
||||
except ConnectionError:
|
||||
return sent
|
||||
except (OSError, Exception) as err:
|
||||
if isinstance(err, asyncio.CancelledError):
|
||||
log.warning("stopped uploading %s#%s to reflector", self.claim_name, self.claim_id)
|
||||
raise err
|
||||
if isinstance(err, OSError):
|
||||
elif isinstance(err, OSError):
|
||||
log.warning(
|
||||
"stopped uploading %s#%s to reflector because blobs were deleted or moved", self.claim_name,
|
||||
self.claim_id
|
||||
)
|
||||
else:
|
||||
log.exception("unexpected error reflecting %s#%s", self.claim_name, self.claim_id)
|
||||
raise asyncio.CancelledError()
|
||||
return sent
|
||||
finally:
|
||||
if protocol.transport:
|
||||
protocol.transport.close()
|
||||
|
||||
@@ -160,13 +160,13 @@ class StreamManager(SourceManager):
|
||||
|
||||
async def reflect_streams(self):
|
||||
try:
|
||||
return await self._reflact_streams()
|
||||
return await self._reflect_streams()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
log.exception("reflector task encountered an unexpected error!")
|
||||
|
||||
async def _reflact_streams(self):
|
||||
async def _reflect_streams(self):
|
||||
# todo: those debug statements are temporary for #2987 - remove them if its closed
|
||||
while True:
|
||||
if self.config.reflect_streams and self.config.reflector_servers:
|
||||
|
||||
+6
-2
@@ -23,8 +23,9 @@ from lbry.wallet.orchstr8.node import BlockchainNode, WalletNode
|
||||
from lbry.extras.daemon.daemon import Daemon, jsonrpc_dumps_pretty
|
||||
from lbry.extras.daemon.components import Component, WalletComponent
|
||||
from lbry.extras.daemon.components import (
|
||||
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
DHT_COMPONENT,
|
||||
HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, LIBTORRENT_COMPONENT
|
||||
)
|
||||
from lbry.extras.daemon.componentmanager import ComponentManager
|
||||
from lbry.extras.daemon.exchange_rate_manager import (
|
||||
@@ -320,6 +321,7 @@ class CommandTestCase(IntegrationTestCase):
|
||||
self.server_blob_manager = None
|
||||
self.server = None
|
||||
self.reflector = None
|
||||
self.skip_libtorrent = True
|
||||
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
@@ -395,6 +397,8 @@ class CommandTestCase(IntegrationTestCase):
|
||||
DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
|
||||
PEER_PROTOCOL_SERVER_COMPONENT
|
||||
]
|
||||
if self.skip_libtorrent:
|
||||
conf.components_to_skip.append(LIBTORRENT_COMPONENT)
|
||||
wallet_node.manager.config = conf
|
||||
|
||||
def wallet_maker(component_manager):
|
||||
|
||||
+185
-10
@@ -20,8 +20,11 @@ import pkg_resources
|
||||
|
||||
import certifi
|
||||
import aiohttp
|
||||
from prometheus_client import Counter
|
||||
from prometheus_client.registry import REGISTRY
|
||||
from lbry.schema.claim import Claim
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -189,6 +192,8 @@ def cache_concurrent(async_fn):
|
||||
async def resolve_host(url: str, port: int, proto: str) -> str:
|
||||
if proto not in ['udp', 'tcp']:
|
||||
raise Exception("invalid protocol")
|
||||
if url.lower() == 'localhost':
|
||||
return '127.0.0.1'
|
||||
try:
|
||||
if ipaddress.ip_address(url):
|
||||
return url
|
||||
@@ -203,18 +208,44 @@ async def resolve_host(url: str, port: int, proto: str) -> str:
|
||||
))[0][4][0]
|
||||
|
||||
|
||||
class LRUCache:
|
||||
class LRUCacheWithMetrics:
|
||||
__slots__ = [
|
||||
'capacity',
|
||||
'cache'
|
||||
'cache',
|
||||
'_track_metrics',
|
||||
'hits',
|
||||
'misses'
|
||||
]
|
||||
|
||||
def __init__(self, capacity):
|
||||
def __init__(self, capacity: int, metric_name: typing.Optional[str] = None, namespace: str = "daemon_cache"):
|
||||
self.capacity = capacity
|
||||
self.cache = collections.OrderedDict()
|
||||
if metric_name is None:
|
||||
self._track_metrics = False
|
||||
self.hits = self.misses = None
|
||||
else:
|
||||
self._track_metrics = True
|
||||
try:
|
||||
self.hits = Counter(
|
||||
f"{metric_name}_cache_hit_count", "Number of cache hits", namespace=namespace
|
||||
)
|
||||
self.misses = Counter(
|
||||
f"{metric_name}_cache_miss_count", "Number of cache misses", namespace=namespace
|
||||
)
|
||||
except ValueError as err:
|
||||
log.debug("failed to set up prometheus %s_cache_miss_count metric: %s", metric_name, err)
|
||||
self._track_metrics = False
|
||||
self.hits = self.misses = None
|
||||
|
||||
def get(self, key):
|
||||
value = self.cache.pop(key)
|
||||
def get(self, key, default=None):
|
||||
try:
|
||||
value = self.cache.pop(key)
|
||||
if self._track_metrics:
|
||||
self.hits.inc()
|
||||
except KeyError:
|
||||
if self._track_metrics:
|
||||
self.misses.inc()
|
||||
return default
|
||||
self.cache[key] = value
|
||||
return value
|
||||
|
||||
@@ -226,16 +257,94 @@ class LRUCache:
|
||||
self.cache.popitem(last=False)
|
||||
self.cache[key] = value
|
||||
|
||||
def clear(self):
|
||||
self.cache.clear()
|
||||
|
||||
def pop(self, key):
|
||||
return self.cache.pop(key)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
return self.set(key, value)
|
||||
|
||||
def __getitem__(self, item):
|
||||
return self.get(item)
|
||||
|
||||
def __contains__(self, item) -> bool:
|
||||
return item in self.cache
|
||||
|
||||
def __len__(self):
|
||||
return len(self.cache)
|
||||
|
||||
def __delitem__(self, key):
|
||||
self.cache.pop(key)
|
||||
|
||||
def __del__(self):
|
||||
self.clear()
|
||||
if self._track_metrics: # needed for tests
|
||||
try:
|
||||
REGISTRY.unregister(self.hits)
|
||||
REGISTRY.unregister(self.misses)
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
|
||||
class LRUCache:
|
||||
__slots__ = [
|
||||
'capacity',
|
||||
'cache'
|
||||
]
|
||||
|
||||
def __init__(self, capacity: int):
|
||||
self.capacity = capacity
|
||||
self.cache = collections.OrderedDict()
|
||||
|
||||
def get(self, key, default=None):
|
||||
try:
|
||||
value = self.cache.pop(key)
|
||||
except KeyError:
|
||||
return default
|
||||
self.cache[key] = value
|
||||
return value
|
||||
|
||||
def set(self, key, value):
|
||||
try:
|
||||
self.cache.pop(key)
|
||||
except KeyError:
|
||||
if len(self.cache) >= self.capacity:
|
||||
self.cache.popitem(last=False)
|
||||
self.cache[key] = value
|
||||
|
||||
def clear(self):
|
||||
self.cache.clear()
|
||||
|
||||
def pop(self, key):
|
||||
return self.cache.pop(key)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
return self.set(key, value)
|
||||
|
||||
def __getitem__(self, item):
|
||||
return self.get(item)
|
||||
|
||||
def __contains__(self, item) -> bool:
|
||||
return item in self.cache
|
||||
|
||||
def __len__(self):
|
||||
return len(self.cache)
|
||||
|
||||
def __delitem__(self, key):
|
||||
self.cache.pop(key)
|
||||
|
||||
def __del__(self):
|
||||
self.clear()
|
||||
|
||||
|
||||
def lru_cache_concurrent(cache_size: typing.Optional[int] = None,
|
||||
override_lru_cache: typing.Optional[LRUCache] = None):
|
||||
override_lru_cache: typing.Optional[LRUCacheWithMetrics] = None):
|
||||
if not cache_size and override_lru_cache is None:
|
||||
raise ValueError("invalid cache size")
|
||||
concurrent_cache = {}
|
||||
lru_cache = override_lru_cache or LRUCache(cache_size)
|
||||
lru_cache = override_lru_cache if override_lru_cache is not None else LRUCacheWithMetrics(cache_size)
|
||||
|
||||
def wrapper(async_fn):
|
||||
|
||||
@@ -270,14 +379,80 @@ async def aiohttp_request(method, url, **kwargs) -> typing.AsyncContextManager[a
|
||||
yield response
|
||||
|
||||
|
||||
async def get_external_ip() -> typing.Optional[str]: # used if upnp is disabled or non-functioning
|
||||
# the ipaddress module does not show these subnets as reserved
|
||||
CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
|
||||
IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
|
||||
|
||||
|
||||
def is_valid_public_ipv4(address, allow_localhost: bool = False):
|
||||
try:
|
||||
parsed_ip = ipaddress.ip_address(address)
|
||||
if parsed_ip.is_loopback and allow_localhost:
|
||||
return True
|
||||
if any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local, parsed_ip.is_loopback,
|
||||
parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private, parsed_ip.is_reserved)):
|
||||
return False
|
||||
else:
|
||||
return not any((CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
|
||||
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
|
||||
except (ipaddress.AddressValueError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
async def fallback_get_external_ip(): # used if spv servers can't be used for ip detection
|
||||
try:
|
||||
async with aiohttp_request("get", "https://api.lbry.com/ip") as resp:
|
||||
response = await resp.json()
|
||||
if response['success']:
|
||||
return response['data']['ip']
|
||||
return response['data']['ip'], None
|
||||
except Exception:
|
||||
return
|
||||
return None, None
|
||||
|
||||
|
||||
async def _get_external_ip(default_servers) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]:
|
||||
# used if upnp is disabled or non-functioning
|
||||
from lbry.wallet.server.udp import SPVStatusClientProtocol # pylint: disable=C0415
|
||||
|
||||
hostname_to_ip = {}
|
||||
ip_to_hostnames = collections.defaultdict(list)
|
||||
|
||||
async def resolve_spv(server, port):
|
||||
try:
|
||||
server_addr = await resolve_host(server, port, 'udp')
|
||||
hostname_to_ip[server] = (server_addr, port)
|
||||
ip_to_hostnames[(server_addr, port)].append(server)
|
||||
except Exception:
|
||||
log.exception("error looking up dns for spv servers")
|
||||
|
||||
# accumulate the dns results
|
||||
await asyncio.gather(*(resolve_spv(server, port) for (server, port) in default_servers))
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
pong_responses = asyncio.Queue()
|
||||
connection = SPVStatusClientProtocol(pong_responses)
|
||||
try:
|
||||
await loop.create_datagram_endpoint(lambda: connection, ('0.0.0.0', 0))
|
||||
# could raise OSError if it cant bind
|
||||
randomized_servers = list(ip_to_hostnames.keys())
|
||||
random.shuffle(randomized_servers)
|
||||
for server in randomized_servers:
|
||||
connection.ping(server)
|
||||
try:
|
||||
_, pong = await asyncio.wait_for(pong_responses.get(), 1)
|
||||
if is_valid_public_ipv4(pong.ip_address):
|
||||
return pong.ip_address, ip_to_hostnames[server][0]
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
return None, None
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
async def get_external_ip(default_servers) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]:
|
||||
ip_from_spv_servers = await _get_external_ip(default_servers)
|
||||
if not ip_from_spv_servers[1]:
|
||||
return await fallback_get_external_ip()
|
||||
return ip_from_spv_servers
|
||||
|
||||
|
||||
def is_running_from_bundle():
|
||||
|
||||
+23
-5
@@ -124,6 +124,7 @@ class AIOSQLite:
|
||||
|
||||
def __checkpoint_and_close(conn: sqlite3.Connection):
|
||||
conn.execute("PRAGMA WAL_CHECKPOINT(FULL);")
|
||||
log.info("DB checkpoint finished.")
|
||||
conn.close()
|
||||
await asyncio.get_event_loop().run_in_executor(
|
||||
self.writer_executor, __checkpoint_and_close, self.writer_connection)
|
||||
@@ -386,14 +387,31 @@ def interpolate(sql, values):
|
||||
return sql
|
||||
|
||||
|
||||
def constrain_single_or_list(constraints, column, value, convert=lambda x: x):
|
||||
def constrain_single_or_list(constraints, column, value, convert=lambda x: x, negate=False):
|
||||
if value is not None:
|
||||
if isinstance(value, list):
|
||||
value = [convert(v) for v in value]
|
||||
if len(value) == 1:
|
||||
constraints[column] = value[0]
|
||||
if negate:
|
||||
constraints[f"{column}__or"] = {
|
||||
f"{column}__is_null": True,
|
||||
f"{column}__not": value[0]
|
||||
}
|
||||
else:
|
||||
constraints[column] = value[0]
|
||||
elif len(value) > 1:
|
||||
constraints[f"{column}__in"] = value
|
||||
if negate:
|
||||
constraints[f"{column}__or"] = {
|
||||
f"{column}__is_null": True,
|
||||
f"{column}__not_in": value
|
||||
}
|
||||
else:
|
||||
constraints[f"{column}__in"] = value
|
||||
elif negate:
|
||||
constraints[f"{column}__or"] = {
|
||||
f"{column}__is_null": True,
|
||||
f"{column}__not": convert(value)
|
||||
}
|
||||
else:
|
||||
constraints[column] = convert(value)
|
||||
return constraints
|
||||
@@ -430,7 +448,7 @@ class SQLiteMixin:
|
||||
return
|
||||
await self.db.executescript('\n'.join(
|
||||
f"DROP TABLE {table};" for table in tables
|
||||
))
|
||||
) + '\n' + 'PRAGMA WAL_CHECKPOINT(FULL);' + '\n' + 'VACUUM;')
|
||||
await self.db.execute(self.CREATE_VERSION_TABLE)
|
||||
await self.db.execute("INSERT INTO version VALUES (?)", (self.SCHEMA_VERSION,))
|
||||
await self.db.executescript(self.CREATE_TABLES_QUERY)
|
||||
@@ -574,7 +592,7 @@ def get_and_reserve_spendable_utxos(transaction: sqlite3.Connection, accounts: L
|
||||
|
||||
class Database(SQLiteMixin):
|
||||
|
||||
SCHEMA_VERSION = "1.4"
|
||||
SCHEMA_VERSION = "1.5"
|
||||
|
||||
PRAGMAS = """
|
||||
pragma journal_mode=WAL;
|
||||
|
||||
+53
-67
@@ -10,11 +10,11 @@ from collections import defaultdict
|
||||
from binascii import hexlify, unhexlify
|
||||
from typing import Dict, Tuple, Type, Iterable, List, Optional, DefaultDict, NamedTuple
|
||||
|
||||
import pylru
|
||||
from lbry.schema.result import Outputs, INVALID, NOT_FOUND
|
||||
from lbry.schema.url import URL
|
||||
from lbry.crypto.hash import hash160, double_sha256, sha256
|
||||
from lbry.crypto.base58 import Base58
|
||||
from lbry.utils import LRUCacheWithMetrics
|
||||
|
||||
from .tasks import TaskGroup
|
||||
from .database import Database
|
||||
@@ -155,7 +155,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
self._on_ready_controller = StreamController()
|
||||
self.on_ready = self._on_ready_controller.stream
|
||||
|
||||
self._tx_cache = pylru.lrucache(self.config.get("tx_cache_size", 10_000))
|
||||
self._tx_cache = LRUCacheWithMetrics(self.config.get("tx_cache_size", 1024), metric_name='tx')
|
||||
self._update_tasks = TaskGroup()
|
||||
self._other_tasks = TaskGroup() # that we dont need to start
|
||||
self._utxo_reservation_lock = asyncio.Lock()
|
||||
@@ -167,7 +167,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
self._known_addresses_out_of_sync = set()
|
||||
|
||||
self.fee_per_name_char = self.config.get('fee_per_name_char', self.default_fee_per_name_char)
|
||||
self._balance_cache = pylru.lrucache(100000)
|
||||
self._balance_cache = LRUCacheWithMetrics(2 ** 15)
|
||||
|
||||
@classmethod
|
||||
def get_id(cls):
|
||||
@@ -415,6 +415,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
"Blockchain Reorganization: attempting rewind to height %s from starting height %s",
|
||||
height, height+rewound
|
||||
)
|
||||
self._tx_cache.clear()
|
||||
|
||||
else:
|
||||
raise IndexError(f"headers.connect() returned negative number ({added})")
|
||||
@@ -513,10 +514,8 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
)
|
||||
return True
|
||||
|
||||
synced_txs = []
|
||||
to_request = {}
|
||||
pending_synced_history = {}
|
||||
updated_cached_items = {}
|
||||
already_synced = set()
|
||||
|
||||
already_synced_offset = 0
|
||||
@@ -526,10 +525,6 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
already_synced.add((txid, remote_height))
|
||||
already_synced_offset += 1
|
||||
continue
|
||||
cache_item = self._tx_cache.get(txid)
|
||||
if cache_item is None:
|
||||
cache_item = TransactionCacheItem()
|
||||
self._tx_cache[txid] = cache_item
|
||||
|
||||
tx_indexes = {}
|
||||
|
||||
@@ -537,29 +532,16 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
tx_indexes[txid] = i
|
||||
if (txid, remote_height) in already_synced:
|
||||
continue
|
||||
cache_item = self._tx_cache.get(txid)
|
||||
cache_item.pending_verifications += 1
|
||||
updated_cached_items[txid] = cache_item
|
||||
|
||||
assert cache_item is not None, 'cache item is none'
|
||||
# tx = cache_item.tx
|
||||
# if cache_item.tx is not None and \
|
||||
# cache_item.tx.height >= remote_height and \
|
||||
# (cache_item.tx.is_verified or remote_height < 1):
|
||||
# synced_txs.append(cache_item.tx) # cached tx is already up-to-date
|
||||
# pending_synced_history[i] = f'{tx.id}:{tx.height}:'
|
||||
# continue
|
||||
to_request[i] = (txid, remote_height)
|
||||
|
||||
log.debug(
|
||||
"request %i transactions, %i/%i for %s are already synced", len(to_request), len(synced_txs),
|
||||
"request %i transactions, %i/%i for %s are already synced", len(to_request), len(already_synced),
|
||||
len(remote_history), address
|
||||
)
|
||||
remote_history_txids = set(txid for txid, _ in remote_history)
|
||||
async for tx in self.request_synced_transactions(to_request, remote_history_txids, address):
|
||||
pending_synced_history[tx_indexes[tx.id]] = f"{tx.id}:{tx.height}:"
|
||||
synced_txs.append(tx)
|
||||
if len(synced_txs) >= 100:
|
||||
if len(pending_synced_history) % 100 == 0:
|
||||
log.info("Syncing address %s: %d/%d", address, len(pending_synced_history), len(to_request))
|
||||
log.info("Sync finished for address %s: %d/%d", address, len(pending_synced_history), len(to_request))
|
||||
|
||||
@@ -572,19 +554,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
if f"{txid}:{height}:" != pending_synced_history[i]:
|
||||
log.warning("history mismatch: %s vs %s", remote_history[remote_i], pending_synced_history[i])
|
||||
synced_history += pending_synced_history[i]
|
||||
await self.db.save_transaction_io_batch(
|
||||
synced_txs, address, self.address_to_hash160(address), synced_history
|
||||
)
|
||||
while synced_txs:
|
||||
self._on_transaction_controller.add(TransactionEvent(address, synced_txs.pop()))
|
||||
|
||||
cache_size = self.config.get("tx_cache_size", 10_000)
|
||||
for txid, cache_item in updated_cached_items.items():
|
||||
cache_item.pending_verifications -= 1
|
||||
if cache_item.pending_verifications < 0:
|
||||
log.warning("config value tx cache size %i needs to be increased", cache_size)
|
||||
cache_item.pending_verifications = 0
|
||||
|
||||
await self.db.set_address_history(address, synced_history)
|
||||
|
||||
if address_manager is None:
|
||||
address_manager = await self.get_address_manager_for_address(address)
|
||||
@@ -592,13 +562,6 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
if address_manager is not None:
|
||||
await address_manager.ensure_address_gap()
|
||||
|
||||
for txid, cache_item in updated_cached_items.items():
|
||||
if self._tx_cache.get(txid) is not cache_item:
|
||||
log.warning("tx cache corrupted while syncing %s, reattempt sync=%s", address, reattempt_update)
|
||||
if reattempt_update:
|
||||
return await self.update_history(address, remote_status, address_manager, False)
|
||||
return False
|
||||
|
||||
local_status, local_history = \
|
||||
await self.get_local_status_and_history(address, synced_history)
|
||||
|
||||
@@ -632,12 +595,6 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
|
||||
async def maybe_verify_transaction(self, tx, remote_height, merkle=None):
|
||||
tx.height = remote_height
|
||||
cached = self._tx_cache.get(tx.id)
|
||||
if not cached:
|
||||
# cache txs looked up by transaction_show too
|
||||
cached = TransactionCacheItem()
|
||||
self._tx_cache[tx.id] = cached
|
||||
cached.tx = tx
|
||||
if 0 < remote_height < len(self.headers):
|
||||
# can't be tx.pending_verifications == 1 because we have to handle the transaction_show case
|
||||
if not merkle:
|
||||
@@ -648,39 +605,65 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
header = await self.headers.get(remote_height)
|
||||
tx.position = merkle['pos']
|
||||
tx.is_verified = merkle_root == header['merkle_root']
|
||||
return tx
|
||||
|
||||
async def request_transactions(self, to_request: Tuple[Tuple[str, int], ...]):
|
||||
async def request_transactions(self, to_request: Tuple[Tuple[str, int], ...], cached=False):
|
||||
batches = [[]]
|
||||
remote_heights = {}
|
||||
cache_hits = set()
|
||||
|
||||
for txid, height in sorted(to_request, key=lambda x: x[1]):
|
||||
if cached:
|
||||
cached_tx = self._tx_cache.get(txid)
|
||||
if cached_tx is not None:
|
||||
if cached_tx.tx is not None and cached_tx.tx.is_verified:
|
||||
cache_hits.add(txid)
|
||||
continue
|
||||
else:
|
||||
self._tx_cache[txid] = TransactionCacheItem()
|
||||
remote_heights[txid] = height
|
||||
if len(batches[-1]) == 100:
|
||||
batches.append([])
|
||||
batches[-1].append(txid)
|
||||
if not batches[-1]:
|
||||
batches.pop()
|
||||
if cached and cache_hits:
|
||||
yield {txid: self._tx_cache[txid].tx for txid in cache_hits}
|
||||
|
||||
for batch in batches:
|
||||
async for tx in self._single_batch(batch, remote_heights):
|
||||
yield tx
|
||||
txs = await self._single_batch(batch, remote_heights)
|
||||
if cached:
|
||||
for txid, tx in txs.items():
|
||||
self._tx_cache[txid].tx = tx
|
||||
yield txs
|
||||
|
||||
async def request_synced_transactions(self, to_request, remote_history, address):
|
||||
pending_sync = []
|
||||
async for tx in self.request_transactions(((txid, height) for txid, height in to_request.values())):
|
||||
pending_sync.append(asyncio.ensure_future(self._sync(tx, remote_history)))
|
||||
yield tx
|
||||
await asyncio.gather(*pending_sync)
|
||||
async for txs in self.request_transactions(((txid, height) for txid, height in to_request.values())):
|
||||
for tx in txs.values():
|
||||
yield tx
|
||||
await self._sync_and_save_batch(address, remote_history, txs)
|
||||
|
||||
async def _single_batch(self, batch, remote_heights):
|
||||
batch_result = await self.network.retriable_call(self.network.get_transaction_batch, batch)
|
||||
heights = {remote_heights[txid] for txid in batch}
|
||||
unrestriced = 0 < min(heights) < max(heights) < max(self.headers.checkpoints or [0])
|
||||
batch_result = await self.network.retriable_call(self.network.get_transaction_batch, batch, not unrestriced)
|
||||
txs = {}
|
||||
for txid, (raw, merkle) in batch_result.items():
|
||||
remote_height = remote_heights[txid]
|
||||
tx = Transaction(unhexlify(raw), height=remote_height)
|
||||
txs[tx.id] = tx
|
||||
await self.maybe_verify_transaction(tx, remote_height, merkle)
|
||||
yield tx
|
||||
return txs
|
||||
|
||||
async def _sync(self, tx, remote_history):
|
||||
async def _sync_and_save_batch(self, address, remote_history, pending_txs):
|
||||
await asyncio.gather(*(self._sync(tx, remote_history, pending_txs) for tx in pending_txs.values()))
|
||||
await self.db.save_transaction_io_batch(
|
||||
pending_txs.values(), address, self.address_to_hash160(address), ""
|
||||
)
|
||||
while pending_txs:
|
||||
self._on_transaction_controller.add(TransactionEvent(address, pending_txs.popitem()[1]))
|
||||
|
||||
async def _sync(self, tx, remote_history, pending_txs):
|
||||
check_db_for_txos = {}
|
||||
for txi in tx.inputs:
|
||||
if txi.txo_ref.txo is not None:
|
||||
@@ -688,9 +671,8 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
wanted_txid = txi.txo_ref.tx_ref.id
|
||||
if wanted_txid not in remote_history:
|
||||
continue
|
||||
if wanted_txid in self._tx_cache:
|
||||
await self._tx_cache[wanted_txid].has_tx.wait()
|
||||
txi.txo_ref = self._tx_cache[wanted_txid].tx.outputs[txi.txo_ref.position].ref
|
||||
if wanted_txid in pending_txs:
|
||||
txi.txo_ref = pending_txs[wanted_txid].outputs[txi.txo_ref.position].ref
|
||||
else:
|
||||
check_db_for_txos[txi] = txi.txo_ref.id
|
||||
|
||||
@@ -704,7 +686,11 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
if txi.txo_ref.id in referenced_txos:
|
||||
txi.txo_ref = referenced_txos[txi.txo_ref.id].ref
|
||||
else:
|
||||
log.warning("%s not on db, not on cache, but on remote history!", txi.txo_ref.id)
|
||||
tx_from_db = await self.db.get_transaction(txid=txi.txo_ref.tx_ref.id)
|
||||
if tx_from_db is None:
|
||||
log.warning("%s not on db, not on cache, but on remote history!", txi.txo_ref.id)
|
||||
else:
|
||||
txi.txo_ref = tx_from_db.outputs[txi.txo_ref.position].ref
|
||||
return tx
|
||||
|
||||
async def get_address_manager_for_address(self, address) -> Optional[AddressManager]:
|
||||
@@ -775,8 +761,8 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
outputs = Outputs.from_base64(encoded_outputs or b'') # TODO: why is the server returning None?
|
||||
txs: List[Transaction] = []
|
||||
if len(outputs.txs) > 0:
|
||||
async for tx in self.request_transactions(tuple(outputs.txs)):
|
||||
txs.append(tx)
|
||||
async for tx in self.request_transactions(tuple(outputs.txs), cached=True):
|
||||
txs.extend(tx.values())
|
||||
|
||||
_txos, blocked = outputs.inflate(txs)
|
||||
|
||||
|
||||
+182
-174
@@ -1,26 +1,27 @@
|
||||
import logging
|
||||
import asyncio
|
||||
import json
|
||||
import socket
|
||||
from time import perf_counter
|
||||
from operator import itemgetter
|
||||
from collections import defaultdict
|
||||
from typing import Dict, Optional, Tuple
|
||||
import aiohttp
|
||||
|
||||
from lbry import __version__
|
||||
from lbry.utils import resolve_host
|
||||
from lbry.error import IncompatibleWalletServerError
|
||||
from lbry.wallet.rpc import RPCSession as BaseClientSession, Connector, RPCError, ProtocolError
|
||||
from lbry.wallet.stream import StreamController
|
||||
from lbry.wallet.server.udp import SPVStatusClientProtocol, SPVPong
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ClientSession(BaseClientSession):
|
||||
def __init__(self, *args, network, server, timeout=30, on_connect_callback=None, **kwargs):
|
||||
def __init__(self, *args, network: 'Network', server, timeout=30, **kwargs):
|
||||
self.network = network
|
||||
self.server = server
|
||||
super().__init__(*args, **kwargs)
|
||||
self._on_disconnect_controller = StreamController()
|
||||
self.on_disconnected = self._on_disconnect_controller.stream
|
||||
self.framer.max_size = self.max_errors = 1 << 32
|
||||
self.timeout = timeout
|
||||
self.max_seconds_idle = timeout * 2
|
||||
@@ -28,8 +29,6 @@ class ClientSession(BaseClientSession):
|
||||
self.connection_latency: Optional[float] = None
|
||||
self._response_samples = 0
|
||||
self.pending_amount = 0
|
||||
self._on_connect_cb = on_connect_callback or (lambda: None)
|
||||
self.trigger_urgent_reconnect = asyncio.Event()
|
||||
|
||||
@property
|
||||
def available(self):
|
||||
@@ -56,7 +55,7 @@ class ClientSession(BaseClientSession):
|
||||
|
||||
async def send_request(self, method, args=()):
|
||||
self.pending_amount += 1
|
||||
log.debug("send %s%s to %s:%i", method, tuple(args), *self.server)
|
||||
log.debug("send %s%s to %s:%i (%i timeout)", method, tuple(args), self.server[0], self.server[1], self.timeout)
|
||||
try:
|
||||
if method == 'server.version':
|
||||
return await self.send_timed_server_version_request(args, self.timeout)
|
||||
@@ -67,7 +66,7 @@ class ClientSession(BaseClientSession):
|
||||
log.debug("Time since last packet: %s", perf_counter() - self.last_packet_received)
|
||||
if (perf_counter() - self.last_packet_received) < self.timeout:
|
||||
continue
|
||||
log.info("timeout sending %s to %s:%i", method, *self.server)
|
||||
log.warning("timeout sending %s to %s:%i", method, *self.server)
|
||||
raise asyncio.TimeoutError
|
||||
if done:
|
||||
try:
|
||||
@@ -87,44 +86,12 @@ class ClientSession(BaseClientSession):
|
||||
self.synchronous_close()
|
||||
raise
|
||||
except asyncio.CancelledError:
|
||||
log.info("cancelled sending %s to %s:%i", method, *self.server)
|
||||
log.warning("cancelled sending %s to %s:%i", method, *self.server)
|
||||
# self.synchronous_close()
|
||||
raise
|
||||
finally:
|
||||
self.pending_amount -= 1
|
||||
|
||||
async def ensure_session(self):
|
||||
# Handles reconnecting and maintaining a session alive
|
||||
# TODO: change to 'ping' on newer protocol (above 1.2)
|
||||
retry_delay = default_delay = 1.0
|
||||
while True:
|
||||
try:
|
||||
if self.is_closing():
|
||||
await self.create_connection(self.timeout)
|
||||
await self.ensure_server_version()
|
||||
self._on_connect_cb()
|
||||
if (perf_counter() - self.last_send) > self.max_seconds_idle or self.response_time is None:
|
||||
await self.ensure_server_version()
|
||||
retry_delay = default_delay
|
||||
except RPCError as e:
|
||||
await self.close()
|
||||
log.debug("Server error, ignoring for 1h: %s:%d -- %s", *self.server, e.message)
|
||||
retry_delay = 60 * 60
|
||||
except IncompatibleWalletServerError:
|
||||
await self.close()
|
||||
retry_delay = 60 * 60
|
||||
log.debug("Wallet server has an incompatible version, retrying in 1h: %s:%d", *self.server)
|
||||
except (asyncio.TimeoutError, OSError):
|
||||
await self.close()
|
||||
retry_delay = min(60, retry_delay * 2)
|
||||
log.debug("Wallet server timeout (retry in %s seconds): %s:%d", retry_delay, *self.server)
|
||||
try:
|
||||
await asyncio.wait_for(self.trigger_urgent_reconnect.wait(), timeout=retry_delay)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
finally:
|
||||
self.trigger_urgent_reconnect.clear()
|
||||
|
||||
async def ensure_server_version(self, required=None, timeout=3):
|
||||
required = required or self.network.PROTOCOL_VERSION
|
||||
response = await asyncio.wait_for(
|
||||
@@ -134,6 +101,25 @@ class ClientSession(BaseClientSession):
|
||||
raise IncompatibleWalletServerError(*self.server)
|
||||
return response
|
||||
|
||||
async def keepalive_loop(self, timeout=3, max_idle=60):
|
||||
try:
|
||||
while True:
|
||||
now = perf_counter()
|
||||
if min(self.last_send, self.last_packet_received) + max_idle < now:
|
||||
await asyncio.wait_for(
|
||||
self.send_request('server.ping', []), timeout=timeout
|
||||
)
|
||||
else:
|
||||
await asyncio.sleep(max(0, max_idle - (now - self.last_send)))
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError):
|
||||
log.warning("closing connection to %s:%i", *self.server)
|
||||
else:
|
||||
log.exception("lost connection to spv")
|
||||
finally:
|
||||
if not self.is_closing():
|
||||
self._close()
|
||||
|
||||
async def create_connection(self, timeout=6):
|
||||
connector = Connector(lambda: self, *self.server)
|
||||
start = perf_counter()
|
||||
@@ -145,12 +131,14 @@ class ClientSession(BaseClientSession):
|
||||
controller.add(request.args)
|
||||
|
||||
def connection_lost(self, exc):
|
||||
log.debug("Connection lost: %s:%d", *self.server)
|
||||
log.warning("Connection lost: %s:%d", *self.server)
|
||||
super().connection_lost(exc)
|
||||
self.response_time = None
|
||||
self.connection_latency = None
|
||||
self._response_samples = 0
|
||||
self._on_disconnect_controller.add(True)
|
||||
# self._on_disconnect_controller.add(True)
|
||||
if self.network:
|
||||
self.network.disconnect()
|
||||
|
||||
|
||||
class Network:
|
||||
@@ -160,10 +148,9 @@ class Network:
|
||||
|
||||
def __init__(self, ledger):
|
||||
self.ledger = ledger
|
||||
self.session_pool = SessionPool(network=self, timeout=self.config.get('connect_timeout', 6))
|
||||
self.client: Optional[ClientSession] = None
|
||||
self.server_features = None
|
||||
self._switch_task: Optional[asyncio.Task] = None
|
||||
# self._switch_task: Optional[asyncio.Task] = None
|
||||
self.running = False
|
||||
self.remote_height: int = 0
|
||||
self._concurrency = asyncio.Semaphore(16)
|
||||
@@ -183,58 +170,170 @@ class Network:
|
||||
}
|
||||
|
||||
self.aiohttp_session: Optional[aiohttp.ClientSession] = None
|
||||
self._urgent_need_reconnect = asyncio.Event()
|
||||
self._loop_task: Optional[asyncio.Task] = None
|
||||
self._keepalive_task: Optional[asyncio.Task] = None
|
||||
|
||||
@property
|
||||
def config(self):
|
||||
return self.ledger.config
|
||||
|
||||
async def switch_forever(self):
|
||||
while self.running:
|
||||
if self.is_connected:
|
||||
await self.client.on_disconnected.first
|
||||
self.server_features = None
|
||||
self.client = None
|
||||
continue
|
||||
self.client = await self.session_pool.wait_for_fastest_session()
|
||||
log.info("Switching to SPV wallet server: %s:%d", *self.client.server)
|
||||
try:
|
||||
self.server_features = await self.get_server_features()
|
||||
self._update_remote_height((await self.subscribe_headers(),))
|
||||
self._on_connected_controller.add(True)
|
||||
log.info("Subscribed to headers: %s:%d", *self.client.server)
|
||||
except (asyncio.TimeoutError, ConnectionError):
|
||||
log.info("Switching to %s:%d timed out, closing and retrying.", *self.client.server)
|
||||
self.client.synchronous_close()
|
||||
self.server_features = None
|
||||
self.client = None
|
||||
def disconnect(self):
|
||||
if self._keepalive_task and not self._keepalive_task.done():
|
||||
self._keepalive_task.cancel()
|
||||
self._keepalive_task = None
|
||||
|
||||
async def start(self):
|
||||
self.running = True
|
||||
self.aiohttp_session = aiohttp.ClientSession()
|
||||
self._switch_task = asyncio.ensure_future(self.switch_forever())
|
||||
# this may become unnecessary when there are no more bugs found,
|
||||
# but for now it helps understanding log reports
|
||||
self._switch_task.add_done_callback(lambda _: log.info("Wallet client switching task stopped."))
|
||||
self.session_pool.start(self.config['default_servers'])
|
||||
self.on_header.listen(self._update_remote_height)
|
||||
if not self.running:
|
||||
self.running = True
|
||||
self.aiohttp_session = aiohttp.ClientSession()
|
||||
self.on_header.listen(self._update_remote_height)
|
||||
self._loop_task = asyncio.create_task(self.network_loop())
|
||||
self._urgent_need_reconnect.set()
|
||||
|
||||
def loop_task_done_callback(f):
|
||||
try:
|
||||
f.result()
|
||||
except Exception:
|
||||
if self.running:
|
||||
log.exception("wallet server connection loop crashed")
|
||||
|
||||
self._loop_task.add_done_callback(loop_task_done_callback)
|
||||
|
||||
async def resolve_spv_dns(self):
|
||||
hostname_to_ip = {}
|
||||
ip_to_hostnames = defaultdict(list)
|
||||
|
||||
async def resolve_spv(server, port):
|
||||
try:
|
||||
server_addr = await resolve_host(server, port, 'udp')
|
||||
hostname_to_ip[server] = (server_addr, port)
|
||||
ip_to_hostnames[(server_addr, port)].append(server)
|
||||
except socket.error:
|
||||
log.warning("error looking up dns for spv server %s:%i", server, port)
|
||||
except Exception:
|
||||
log.exception("error looking up dns for spv server %s:%i", server, port)
|
||||
|
||||
# accumulate the dns results
|
||||
await asyncio.gather(*(resolve_spv(server, port) for (server, port) in self.config['default_servers']))
|
||||
return hostname_to_ip, ip_to_hostnames
|
||||
|
||||
async def get_n_fastest_spvs(self, n=5, timeout=3.0) -> Dict[Tuple[str, int], SPVPong]:
|
||||
loop = asyncio.get_event_loop()
|
||||
pong_responses = asyncio.Queue()
|
||||
connection = SPVStatusClientProtocol(pong_responses)
|
||||
sent_ping_timestamps = {}
|
||||
_, ip_to_hostnames = await self.resolve_spv_dns()
|
||||
log.info("%i possible spv servers to try (%i urls in config)", len(ip_to_hostnames),
|
||||
len(self.config['default_servers']))
|
||||
pongs = {}
|
||||
try:
|
||||
await loop.create_datagram_endpoint(lambda: connection, ('0.0.0.0', 0))
|
||||
# could raise OSError if it cant bind
|
||||
start = perf_counter()
|
||||
for server in ip_to_hostnames:
|
||||
connection.ping(server)
|
||||
sent_ping_timestamps[server] = perf_counter()
|
||||
while len(pongs) < n:
|
||||
(remote, ts), pong = await asyncio.wait_for(pong_responses.get(), timeout - (perf_counter() - start))
|
||||
latency = ts - start
|
||||
log.info("%s:%i has latency of %sms (available: %s, height: %i)",
|
||||
'/'.join(ip_to_hostnames[remote]), remote[1], round(latency * 1000, 2),
|
||||
pong.available, pong.height)
|
||||
|
||||
if pong.available:
|
||||
pongs[remote] = pong
|
||||
return pongs
|
||||
except asyncio.TimeoutError:
|
||||
if pongs:
|
||||
log.info("%i/%i probed spv servers are accepting connections", len(pongs), len(ip_to_hostnames))
|
||||
else:
|
||||
log.warning("%i spv status probes failed, retrying later. servers tried: %s",
|
||||
len(sent_ping_timestamps),
|
||||
', '.join('/'.join(hosts) + f' ({ip})' for ip, hosts in ip_to_hostnames.items()))
|
||||
return pongs
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
async def connect_to_fastest(self) -> Optional[ClientSession]:
|
||||
fastest_spvs = await self.get_n_fastest_spvs()
|
||||
for (host, port) in fastest_spvs:
|
||||
|
||||
client = ClientSession(network=self, server=(host, port))
|
||||
try:
|
||||
await client.create_connection()
|
||||
log.warning("Connected to spv server %s:%i", host, port)
|
||||
await client.ensure_server_version()
|
||||
return client
|
||||
except (asyncio.TimeoutError, ConnectionError, OSError, IncompatibleWalletServerError, RPCError):
|
||||
log.warning("Connecting to %s:%d failed", host, port)
|
||||
client._close()
|
||||
return
|
||||
|
||||
async def network_loop(self):
|
||||
sleep_delay = 30
|
||||
while self.running:
|
||||
await asyncio.wait(
|
||||
[asyncio.sleep(30), self._urgent_need_reconnect.wait()], return_when=asyncio.FIRST_COMPLETED
|
||||
)
|
||||
if self._urgent_need_reconnect.is_set():
|
||||
sleep_delay = 30
|
||||
self._urgent_need_reconnect.clear()
|
||||
if not self.is_connected:
|
||||
client = await self.connect_to_fastest()
|
||||
if not client:
|
||||
log.warning("failed to connect to any spv servers, retrying later")
|
||||
sleep_delay *= 2
|
||||
sleep_delay = min(sleep_delay, 300)
|
||||
continue
|
||||
log.debug("get spv server features %s:%i", *client.server)
|
||||
features = await client.send_request('server.features', [])
|
||||
self.client, self.server_features = client, features
|
||||
log.info("subscribe to headers %s:%i", *client.server)
|
||||
self._update_remote_height((await self.subscribe_headers(),))
|
||||
self._on_connected_controller.add(True)
|
||||
server_str = "%s:%i" % client.server
|
||||
log.info("maintaining connection to spv server %s", server_str)
|
||||
self._keepalive_task = asyncio.create_task(self.client.keepalive_loop())
|
||||
try:
|
||||
await asyncio.wait(
|
||||
[self._keepalive_task, self._urgent_need_reconnect.wait()],
|
||||
return_when=asyncio.FIRST_COMPLETED
|
||||
)
|
||||
if self._urgent_need_reconnect.is_set():
|
||||
log.warning("urgent reconnect needed")
|
||||
self._urgent_need_reconnect.clear()
|
||||
if self._keepalive_task and not self._keepalive_task.done():
|
||||
self._keepalive_task.cancel()
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
finally:
|
||||
self._keepalive_task = None
|
||||
self.client = None
|
||||
self.server_features = None
|
||||
log.warning("connection lost to %s", server_str)
|
||||
log.info("network loop finished")
|
||||
|
||||
async def stop(self):
|
||||
if self.running:
|
||||
self.running = False
|
||||
self.running = False
|
||||
self.disconnect()
|
||||
if self._loop_task and not self._loop_task.done():
|
||||
self._loop_task.cancel()
|
||||
self._loop_task = None
|
||||
if self.aiohttp_session:
|
||||
await self.aiohttp_session.close()
|
||||
self._switch_task.cancel()
|
||||
self.session_pool.stop()
|
||||
self.aiohttp_session = None
|
||||
|
||||
@property
|
||||
def is_connected(self):
|
||||
return self.client and not self.client.is_closing()
|
||||
|
||||
def rpc(self, list_or_method, args, restricted=True, session=None):
|
||||
session = session or (self.client if restricted else self.session_pool.fastest_session)
|
||||
if session and not session.is_closing():
|
||||
def rpc(self, list_or_method, args, restricted=True, session: Optional[ClientSession] = None):
|
||||
if session or self.is_connected:
|
||||
session = session or self.client
|
||||
return session.send_request(list_or_method, args)
|
||||
else:
|
||||
self.session_pool.trigger_nodelay_connect()
|
||||
self._urgent_need_reconnect.set()
|
||||
raise ConnectionError("Attempting to send rpc request when connection is not available.")
|
||||
|
||||
async def retriable_call(self, function, *args, **kwargs):
|
||||
@@ -242,14 +341,15 @@ class Network:
|
||||
while self.running:
|
||||
if not self.is_connected:
|
||||
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
|
||||
self._urgent_need_reconnect.set()
|
||||
await self.on_connected.first
|
||||
await self.session_pool.wait_for_fastest_session()
|
||||
try:
|
||||
return await function(*args, **kwargs)
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("Wallet server call timed out, retrying.")
|
||||
except ConnectionError:
|
||||
pass
|
||||
log.warning("connection error")
|
||||
|
||||
raise asyncio.CancelledError() # if we got here, we are shutting down
|
||||
|
||||
def _update_remote_height(self, header_args):
|
||||
@@ -339,95 +439,3 @@ class Network:
|
||||
async with self.aiohttp_session.post(server, json=message) as r:
|
||||
result = await r.json()
|
||||
return result['result']
|
||||
|
||||
|
||||
class SessionPool:
|
||||
|
||||
def __init__(self, network: Network, timeout: float):
|
||||
self.network = network
|
||||
self.sessions: Dict[ClientSession, Optional[asyncio.Task]] = dict()
|
||||
self.timeout = timeout
|
||||
self.new_connection_event = asyncio.Event()
|
||||
|
||||
@property
|
||||
def online(self):
|
||||
return any(not session.is_closing() for session in self.sessions)
|
||||
|
||||
@property
|
||||
def available_sessions(self):
|
||||
return (session for session in self.sessions if session.available)
|
||||
|
||||
@property
|
||||
def fastest_session(self):
|
||||
if not self.online:
|
||||
return None
|
||||
return min(
|
||||
[((session.response_time + session.connection_latency) * (session.pending_amount + 1), session)
|
||||
for session in self.available_sessions] or [(0, None)],
|
||||
key=itemgetter(0)
|
||||
)[1]
|
||||
|
||||
def _get_session_connect_callback(self, session: ClientSession):
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def callback():
|
||||
duplicate_connections = [
|
||||
s for s in self.sessions
|
||||
if s is not session and s.server_address_and_port == session.server_address_and_port
|
||||
]
|
||||
already_connected = None if not duplicate_connections else duplicate_connections[0]
|
||||
if already_connected:
|
||||
self.sessions.pop(session).cancel()
|
||||
session.synchronous_close()
|
||||
log.debug("wallet server %s resolves to the same server as %s, rechecking in an hour",
|
||||
session.server[0], already_connected.server[0])
|
||||
loop.call_later(3600, self._connect_session, session.server)
|
||||
return
|
||||
self.new_connection_event.set()
|
||||
log.info("connected to %s:%i", *session.server)
|
||||
|
||||
return callback
|
||||
|
||||
def _connect_session(self, server: Tuple[str, int]):
|
||||
session = None
|
||||
for s in self.sessions:
|
||||
if s.server == server:
|
||||
session = s
|
||||
break
|
||||
if not session:
|
||||
session = ClientSession(
|
||||
network=self.network, server=server
|
||||
)
|
||||
session._on_connect_cb = self._get_session_connect_callback(session)
|
||||
task = self.sessions.get(session, None)
|
||||
if not task or task.done():
|
||||
task = asyncio.create_task(session.ensure_session())
|
||||
task.add_done_callback(lambda _: self.ensure_connections())
|
||||
self.sessions[session] = task
|
||||
|
||||
def start(self, default_servers):
|
||||
for server in default_servers:
|
||||
self._connect_session(server)
|
||||
|
||||
def stop(self):
|
||||
for session, task in self.sessions.items():
|
||||
task.cancel()
|
||||
session.synchronous_close()
|
||||
self.sessions.clear()
|
||||
|
||||
def ensure_connections(self):
|
||||
for session in self.sessions:
|
||||
self._connect_session(session.server)
|
||||
|
||||
def trigger_nodelay_connect(self):
|
||||
# used when other parts of the system sees we might have internet back
|
||||
# bypasses the retry interval
|
||||
for session in self.sessions:
|
||||
session.trigger_urgent_reconnect.set()
|
||||
|
||||
async def wait_for_fastest_session(self):
|
||||
while not self.fastest_session:
|
||||
self.trigger_nodelay_connect()
|
||||
self.new_connection_event.clear()
|
||||
await self.new_connection_event.wait()
|
||||
return self.fastest_session
|
||||
@@ -11,6 +11,7 @@ from lbry.wallet.server.daemon import DaemonError
|
||||
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
|
||||
from lbry.wallet.server.util import chunks, class_logger
|
||||
from lbry.wallet.server.leveldb import FlushData
|
||||
from lbry.wallet.server.udp import StatusServer
|
||||
|
||||
|
||||
class Prefetcher:
|
||||
@@ -185,6 +186,7 @@ class BlockProcessor:
|
||||
|
||||
self.search_cache = {}
|
||||
self.history_cache = {}
|
||||
self.status_server = StatusServer()
|
||||
|
||||
async def run_in_thread_with_lock(self, func, *args):
|
||||
# Run in a thread to prevent blocking. Shielded so that
|
||||
@@ -221,6 +223,7 @@ class BlockProcessor:
|
||||
processed_time = time.perf_counter() - start
|
||||
self.block_count_metric.set(self.height)
|
||||
self.block_update_time_metric.observe(processed_time)
|
||||
self.status_server.set_height(self.db.fs_height, self.db.db_tip)
|
||||
if not self.db.first_sync:
|
||||
s = '' if len(blocks) == 1 else 's'
|
||||
self.logger.info('processed {:,d} block{} in {:.1f}s'.format(len(blocks), s, processed_time))
|
||||
@@ -682,9 +685,11 @@ class BlockProcessor:
|
||||
disk before exiting, as otherwise a significant amount of work
|
||||
could be lost.
|
||||
"""
|
||||
|
||||
self._caught_up_event = caught_up_event
|
||||
try:
|
||||
await self._first_open_dbs()
|
||||
self.status_server.set_height(self.db.fs_height, self.db.db_tip)
|
||||
await asyncio.wait([
|
||||
self.prefetcher.main_loop(self.height),
|
||||
self._process_prefetched_blocks()
|
||||
@@ -695,6 +700,7 @@ class BlockProcessor:
|
||||
self.logger.exception("Block processing failed!")
|
||||
raise
|
||||
finally:
|
||||
self.status_server.stop()
|
||||
# Shut down block processing
|
||||
self.logger.info('flushing to DB for a clean shutdown...')
|
||||
await self.flush(True)
|
||||
@@ -714,7 +720,6 @@ class BlockProcessor:
|
||||
|
||||
|
||||
class Timer:
|
||||
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
self.total = 0
|
||||
|
||||
@@ -3,11 +3,10 @@ import itertools
|
||||
import json
|
||||
import time
|
||||
from functools import wraps
|
||||
from pylru import lrucache
|
||||
|
||||
import aiohttp
|
||||
from prometheus_client import Gauge, Histogram
|
||||
|
||||
from lbry.utils import LRUCacheWithMetrics
|
||||
from lbry.wallet.rpc.jsonrpc import RPCError
|
||||
from lbry.wallet.server.util import hex_to_bytes, class_logger
|
||||
from lbry.wallet.rpc import JSONRPC
|
||||
@@ -55,8 +54,8 @@ class Daemon:
|
||||
self._height = None
|
||||
self.available_rpcs = {}
|
||||
self.connector = aiohttp.TCPConnector()
|
||||
self._block_hash_cache = lrucache(100000)
|
||||
self._block_cache = lrucache(10000)
|
||||
self._block_hash_cache = LRUCacheWithMetrics(100000)
|
||||
self._block_cache = LRUCacheWithMetrics(2 ** 16, metric_name='block', namespace=NAMESPACE)
|
||||
|
||||
async def close(self):
|
||||
if self.connector:
|
||||
@@ -250,8 +249,10 @@ class Daemon:
|
||||
|
||||
async def deserialised_block(self, hex_hash):
|
||||
"""Return the deserialised block with the given hex hash."""
|
||||
if not self._block_cache.get(hex_hash):
|
||||
self._block_cache[hex_hash] = await self._send_single('getblock', (hex_hash, True))
|
||||
if hex_hash not in self._block_cache:
|
||||
block = await self._send_single('getblock', (hex_hash, True))
|
||||
self._block_cache[hex_hash] = block
|
||||
return block
|
||||
return self._block_cache[hex_hash]
|
||||
|
||||
async def raw_blocks(self, hex_hashes):
|
||||
|
||||
@@ -283,7 +283,7 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
|
||||
channel_ids = constraints.pop('channel_ids')
|
||||
if channel_ids:
|
||||
constraints['claim.channel_hash__in'] = {
|
||||
unhexlify(cid)[::-1] for cid in channel_ids
|
||||
unhexlify(cid)[::-1] for cid in channel_ids if cid
|
||||
}
|
||||
if 'not_channel_ids' in constraints:
|
||||
not_channel_ids = constraints.pop('not_channel_ids')
|
||||
|
||||
@@ -15,7 +15,6 @@ import ast
|
||||
import os
|
||||
import time
|
||||
import zlib
|
||||
import pylru
|
||||
import typing
|
||||
from typing import Optional, List, Tuple, Iterable
|
||||
from asyncio import sleep
|
||||
@@ -25,7 +24,7 @@ from glob import glob
|
||||
from struct import pack, unpack
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
import attr
|
||||
|
||||
from lbry.utils import LRUCacheWithMetrics
|
||||
from lbry.wallet.server import util
|
||||
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
|
||||
from lbry.wallet.server.merkle import Merkle, MerkleCache
|
||||
@@ -94,7 +93,7 @@ class LevelDB:
|
||||
self.headers_db = None
|
||||
self.tx_db = None
|
||||
|
||||
self._tx_and_merkle_cache = pylru.lrucache(100000)
|
||||
self._tx_and_merkle_cache = LRUCacheWithMetrics(2 ** 17, metric_name='tx_and_merkle', namespace="wallet_server")
|
||||
self.total_transactions = None
|
||||
|
||||
async def _read_tx_counts(self):
|
||||
|
||||
@@ -344,7 +344,7 @@ class MemPool:
|
||||
value += sum(v for h168, v in tx.out_pairs if h168 == hashX)
|
||||
return value
|
||||
|
||||
async def compact_fee_histogram(self):
|
||||
def compact_fee_histogram(self):
|
||||
"""Return a compact fee histogram of the current mempool."""
|
||||
return self.cached_compact_histogram
|
||||
|
||||
|
||||
@@ -110,11 +110,14 @@ class Server:
|
||||
self.cancellable_tasks.append(asyncio.ensure_future(run(*args, _flag)))
|
||||
return _flag.wait()
|
||||
|
||||
await self.start_prometheus()
|
||||
await self.bp.status_server.start(0, bytes.fromhex(self.bp.coin.GENESIS_HASH)[::-1]
|
||||
, self.env.host, self.env.tcp_port)
|
||||
await _start_cancellable(self.bp.fetch_and_process_blocks)
|
||||
|
||||
await self.db.populate_header_merkle_cache()
|
||||
await _start_cancellable(self.mempool.keep_synchronized)
|
||||
await _start_cancellable(self.session_mgr.serve, self.notifications)
|
||||
await self.start_prometheus()
|
||||
|
||||
async def stop(self):
|
||||
for task in reversed(self.cancellable_tasks):
|
||||
|
||||
@@ -4,7 +4,6 @@ import math
|
||||
import time
|
||||
import json
|
||||
import zlib
|
||||
import pylru
|
||||
import base64
|
||||
import codecs
|
||||
import typing
|
||||
@@ -18,11 +17,11 @@ from collections import defaultdict
|
||||
from functools import partial
|
||||
|
||||
from binascii import hexlify, unhexlify
|
||||
from pylru import lrucache
|
||||
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
|
||||
from prometheus_client import Counter, Info, Histogram, Gauge
|
||||
|
||||
import lbry
|
||||
from lbry.utils import LRUCacheWithMetrics
|
||||
from lbry.build_info import BUILD, COMMIT_HASH, DOCKER_TAG
|
||||
from lbry.wallet.server.block_processor import LBRYBlockProcessor
|
||||
from lbry.wallet.server.db.writer import LBRYLevelDB
|
||||
@@ -248,11 +247,12 @@ class SessionManager:
|
||||
async def _manage_servers(self):
|
||||
paused = False
|
||||
max_sessions = self.env.max_sessions
|
||||
low_watermark = max_sessions * 19 // 20
|
||||
low_watermark = int(max_sessions * 0.95)
|
||||
while True:
|
||||
await self.session_event.wait()
|
||||
self.session_event.clear()
|
||||
if not paused and len(self.sessions) >= max_sessions:
|
||||
self.bp.status_server.set_unavailable()
|
||||
self.logger.info(f'maximum sessions {max_sessions:,d} '
|
||||
f'reached, stopping new connections until '
|
||||
f'count drops to {low_watermark:,d}')
|
||||
@@ -261,6 +261,7 @@ class SessionManager:
|
||||
# Start listening for incoming connections if paused and
|
||||
# session count has fallen
|
||||
if paused and len(self.sessions) <= low_watermark:
|
||||
self.bp.status_server.set_available()
|
||||
self.logger.info('resuming listening for incoming connections')
|
||||
await self._start_external_servers()
|
||||
paused = False
|
||||
@@ -573,6 +574,7 @@ class SessionManager:
|
||||
await self.start_other()
|
||||
await self._start_external_servers()
|
||||
server_listening_event.set()
|
||||
self.bp.status_server.set_available()
|
||||
# Peer discovery should start after the external servers
|
||||
# because we connect to ourself
|
||||
await asyncio.wait([
|
||||
@@ -583,6 +585,7 @@ class SessionManager:
|
||||
])
|
||||
finally:
|
||||
await self._close_servers(list(self.servers.keys()))
|
||||
log.warning("disconnect %i sessions", len(self.sessions))
|
||||
if self.sessions:
|
||||
await asyncio.wait([
|
||||
session.close(force_after=1) for session in self.sessions.values()
|
||||
@@ -811,8 +814,8 @@ class LBRYSessionManager(SessionManager):
|
||||
if self.env.websocket_host is not None and self.env.websocket_port is not None:
|
||||
self.websocket = AdminWebSocket(self)
|
||||
self.search_cache = self.bp.search_cache
|
||||
self.search_cache['search'] = lrucache(10000)
|
||||
self.search_cache['resolve'] = lrucache(10000)
|
||||
self.search_cache['search'] = LRUCacheWithMetrics(2 ** 14, metric_name='search', namespace=NAMESPACE)
|
||||
self.search_cache['resolve'] = LRUCacheWithMetrics(2 ** 16, metric_name='resolve', namespace=NAMESPACE)
|
||||
|
||||
async def process_metrics(self):
|
||||
while self.running:
|
||||
@@ -865,6 +868,7 @@ class LBRYElectrumX(SessionBase):
|
||||
max_errors = math.inf # don't disconnect people for errors! let them happen...
|
||||
session_mgr: LBRYSessionManager
|
||||
version = lbry.__version__
|
||||
cached_server_features = {}
|
||||
|
||||
@classmethod
|
||||
def initialize_request_handlers(cls):
|
||||
@@ -911,13 +915,15 @@ class LBRYElectrumX(SessionBase):
|
||||
super().__init__(*args, **kwargs)
|
||||
if not LBRYElectrumX.request_handlers:
|
||||
LBRYElectrumX.initialize_request_handlers()
|
||||
if not LBRYElectrumX.cached_server_features:
|
||||
LBRYElectrumX.set_server_features(self.env)
|
||||
self.subscribe_headers = False
|
||||
self.subscribe_headers_raw = False
|
||||
self.connection.max_response_size = self.env.max_send
|
||||
self.hashX_subs = {}
|
||||
self.sv_seen = False
|
||||
self.protocol_tuple = self.PROTOCOL_MIN
|
||||
|
||||
self.protocol_string = None
|
||||
self.daemon = self.session_mgr.daemon
|
||||
self.bp: LBRYBlockProcessor = self.session_mgr.bp
|
||||
self.db: LBRYLevelDB = self.bp.db
|
||||
@@ -928,10 +934,10 @@ class LBRYElectrumX(SessionBase):
|
||||
for ver in (cls.PROTOCOL_MIN, cls.PROTOCOL_MAX)]
|
||||
|
||||
@classmethod
|
||||
def server_features(cls, env):
|
||||
def set_server_features(cls, env):
|
||||
"""Return the server features dictionary."""
|
||||
min_str, max_str = cls.protocol_min_max_strings()
|
||||
return {
|
||||
cls.cached_server_features.update({
|
||||
'hosts': env.hosts_dict(),
|
||||
'pruning': None,
|
||||
'server_version': cls.version,
|
||||
@@ -944,10 +950,10 @@ class LBRYElectrumX(SessionBase):
|
||||
'daily_fee': env.daily_fee,
|
||||
'hash_function': 'sha256',
|
||||
'trending_algorithm': env.trending_algorithms[0]
|
||||
}
|
||||
})
|
||||
|
||||
async def server_features_async(self):
|
||||
return self.server_features(self.env)
|
||||
return self.cached_server_features
|
||||
|
||||
@classmethod
|
||||
def server_version_args(cls):
|
||||
@@ -1272,7 +1278,7 @@ class LBRYElectrumX(SessionBase):
|
||||
hashXes = [
|
||||
(self.address_to_hashX(address), address) for address in addresses
|
||||
]
|
||||
return await asyncio.gather(*(self.hashX_subscribe(*args) for args in hashXes))
|
||||
return [await self.hashX_subscribe(*args) for args in hashXes]
|
||||
|
||||
async def address_unsubscribe(self, address):
|
||||
"""Unsubscribe an address.
|
||||
@@ -1468,7 +1474,8 @@ class LBRYElectrumX(SessionBase):
|
||||
client_name: a string identifying the client
|
||||
protocol_version: the protocol version spoken by the client
|
||||
"""
|
||||
|
||||
if self.protocol_string is not None:
|
||||
return self.version, self.protocol_string
|
||||
if self.sv_seen and self.protocol_tuple >= (1, 4):
|
||||
raise RPCError(BAD_REQUEST, f'server.version already sent')
|
||||
self.sv_seen = True
|
||||
@@ -1478,8 +1485,7 @@ class LBRYElectrumX(SessionBase):
|
||||
if self.env.drop_client is not None and \
|
||||
self.env.drop_client.match(client_name):
|
||||
self.close_after_send = True
|
||||
raise RPCError(BAD_REQUEST,
|
||||
f'unsupported client: {client_name}')
|
||||
raise RPCError(BAD_REQUEST, f'unsupported client: {client_name}')
|
||||
if self.client_version != client_name[:17]:
|
||||
self.session_mgr.session_count_metric.labels(version=self.client_version).dec()
|
||||
self.client_version = client_name[:17]
|
||||
@@ -1488,19 +1494,16 @@ class LBRYElectrumX(SessionBase):
|
||||
|
||||
# Find the highest common protocol version. Disconnect if
|
||||
# that protocol version in unsupported.
|
||||
ptuple, client_min = util.protocol_version(
|
||||
protocol_version, self.PROTOCOL_MIN, self.PROTOCOL_MAX)
|
||||
ptuple, client_min = util.protocol_version(protocol_version, self.PROTOCOL_MIN, self.PROTOCOL_MAX)
|
||||
if ptuple is None:
|
||||
# FIXME: this fills the logs
|
||||
# if client_min > self.PROTOCOL_MIN:
|
||||
# self.logger.info(f'client requested future protocol version '
|
||||
# f'{util.version_string(client_min)} '
|
||||
# f'- is your software out of date?')
|
||||
self.close_after_send = True
|
||||
raise RPCError(BAD_REQUEST,
|
||||
f'unsupported protocol version: {protocol_version}')
|
||||
ptuple, client_min = util.protocol_version(protocol_version, (1, 1, 0), (1, 4, 0))
|
||||
if ptuple is None:
|
||||
self.close_after_send = True
|
||||
raise RPCError(BAD_REQUEST, f'unsupported protocol version: {protocol_version}')
|
||||
|
||||
self.protocol_tuple = ptuple
|
||||
return self.version, self.protocol_version_string()
|
||||
self.protocol_string = util.version_string(ptuple)
|
||||
return self.version, self.protocol_string
|
||||
|
||||
async def transaction_broadcast(self, raw_tx):
|
||||
"""Broadcast a raw transaction to the network.
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
import asyncio
|
||||
import struct
|
||||
from time import perf_counter
|
||||
import logging
|
||||
from typing import Optional, Tuple, NamedTuple
|
||||
from lbry.utils import LRUCache
|
||||
# from prometheus_client import Counter
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
_MAGIC = 1446058291 # genesis blocktime (which is actually wrong)
|
||||
# ping_count_metric = Counter("ping_count", "Number of pings received", namespace='wallet_server_status')
|
||||
_PAD_BYTES = b'\x00' * 64
|
||||
|
||||
|
||||
class SPVPing(NamedTuple):
|
||||
magic: int
|
||||
protocol_version: int
|
||||
pad_bytes: bytes
|
||||
|
||||
def encode(self):
|
||||
return struct.pack(b'!lB64s', *self)
|
||||
|
||||
@staticmethod
|
||||
def make(protocol_version=1) -> bytes:
|
||||
return SPVPing(_MAGIC, protocol_version, _PAD_BYTES).encode()
|
||||
|
||||
@classmethod
|
||||
def decode(cls, packet: bytes):
|
||||
decoded = cls(*struct.unpack(b'!lB64s', packet[:69]))
|
||||
if decoded.magic != _MAGIC:
|
||||
raise ValueError("invalid magic bytes")
|
||||
return decoded
|
||||
|
||||
|
||||
class SPVPong(NamedTuple):
|
||||
protocol_version: int
|
||||
flags: int
|
||||
height: int
|
||||
tip: bytes
|
||||
source_address_raw: bytes
|
||||
|
||||
def encode(self):
|
||||
return struct.pack(b'!BBl32s4s', *self)
|
||||
|
||||
@staticmethod
|
||||
def make(height: int, tip: bytes, flags: int, protocol_version: int = 1) -> bytes:
|
||||
# note: drops the last 4 bytes so the result can be cached and have addresses added to it as needed
|
||||
return SPVPong(protocol_version, flags, height, tip, b'\x00\x00\x00\x00').encode()[:38]
|
||||
|
||||
@classmethod
|
||||
def decode(cls, packet: bytes):
|
||||
return cls(*struct.unpack(b'!BBl32s4s', packet[:42]))
|
||||
|
||||
@property
|
||||
def available(self) -> bool:
|
||||
return (self.flags & 0b00000001) > 0
|
||||
|
||||
@property
|
||||
def ip_address(self) -> str:
|
||||
return ".".join(map(str, self.source_address_raw))
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"SPVPong(external_ip={self.ip_address}, version={self.protocol_version}, " \
|
||||
f"available={'True' if self.flags & 1 > 0 else 'False'}," \
|
||||
f" height={self.height}, tip={self.tip[::-1].hex()})"
|
||||
|
||||
|
||||
class SPVServerStatusProtocol(asyncio.DatagramProtocol):
|
||||
PROTOCOL_VERSION = 1
|
||||
|
||||
def __init__(self, height: int, tip: bytes, throttle_cache_size: int = 1024, throttle_reqs_per_sec: int = 10):
|
||||
super().__init__()
|
||||
self.transport: Optional[asyncio.transports.DatagramTransport] = None
|
||||
self._height = height
|
||||
self._tip = tip
|
||||
self._flags = 0
|
||||
self._cached_response = None
|
||||
self.update_cached_response()
|
||||
self._throttle = LRUCache(throttle_cache_size)
|
||||
self._should_log = LRUCache(throttle_cache_size)
|
||||
self._min_delay = 1 / throttle_reqs_per_sec
|
||||
|
||||
def update_cached_response(self):
|
||||
self._cached_response = SPVPong.make(self._height, self._tip, self._flags, self.PROTOCOL_VERSION)
|
||||
|
||||
def set_unavailable(self):
|
||||
self._flags &= 0b11111110
|
||||
self.update_cached_response()
|
||||
|
||||
def set_available(self):
|
||||
self._flags |= 0b00000001
|
||||
self.update_cached_response()
|
||||
|
||||
def set_height(self, height: int, tip: bytes):
|
||||
self._height, self._tip = height, tip
|
||||
self.update_cached_response()
|
||||
|
||||
def should_throttle(self, host: str):
|
||||
now = perf_counter()
|
||||
last_requested = self._throttle.get(host, default=0)
|
||||
self._throttle[host] = now
|
||||
if now - last_requested < self._min_delay:
|
||||
log_cnt = self._should_log.get(host, default=0) + 1
|
||||
if log_cnt % 100 == 0:
|
||||
log.warning("throttle spv status to %s", host)
|
||||
self._should_log[host] = log_cnt
|
||||
return True
|
||||
return False
|
||||
|
||||
def make_pong(self, host):
|
||||
return self._cached_response + bytes(int(b) for b in host.split("."))
|
||||
|
||||
def datagram_received(self, data: bytes, addr: Tuple[str, int]):
|
||||
if self.should_throttle(addr[0]):
|
||||
return
|
||||
try:
|
||||
SPVPing.decode(data)
|
||||
except (ValueError, struct.error, AttributeError, TypeError):
|
||||
# log.exception("derp")
|
||||
return
|
||||
self.transport.sendto(self.make_pong(addr[0]), addr)
|
||||
# ping_count_metric.inc()
|
||||
|
||||
def connection_made(self, transport) -> None:
|
||||
self.transport = transport
|
||||
|
||||
def connection_lost(self, exc: Optional[Exception]) -> None:
|
||||
self.transport = None
|
||||
|
||||
def close(self):
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
|
||||
|
||||
class StatusServer:
|
||||
def __init__(self):
|
||||
self._protocol: Optional[SPVServerStatusProtocol] = None
|
||||
|
||||
async def start(self, height: int, tip: bytes, interface: str, port: int):
|
||||
if self._protocol:
|
||||
return
|
||||
loop = asyncio.get_event_loop()
|
||||
self._protocol = SPVServerStatusProtocol(height, tip)
|
||||
interface = interface if interface.lower() != 'localhost' else '127.0.0.1'
|
||||
await loop.create_datagram_endpoint(lambda: self._protocol, (interface, port))
|
||||
log.info("started udp status server on %s:%i", interface, port)
|
||||
|
||||
def stop(self):
|
||||
if self._protocol:
|
||||
self._protocol.close()
|
||||
self._protocol = None
|
||||
|
||||
def set_unavailable(self):
|
||||
self._protocol.set_unavailable()
|
||||
|
||||
def set_available(self):
|
||||
self._protocol.set_available()
|
||||
|
||||
def set_height(self, height: int, tip: bytes):
|
||||
self._protocol.set_height(height, tip)
|
||||
|
||||
|
||||
class SPVStatusClientProtocol(asyncio.DatagramProtocol):
|
||||
PROTOCOL_VERSION = 1
|
||||
|
||||
def __init__(self, responses: asyncio.Queue):
|
||||
super().__init__()
|
||||
self.transport: Optional[asyncio.transports.DatagramTransport] = None
|
||||
self.responses = responses
|
||||
self._ping_packet = SPVPing.make(self.PROTOCOL_VERSION)
|
||||
|
||||
def datagram_received(self, data: bytes, addr: Tuple[str, int]):
|
||||
try:
|
||||
self.responses.put_nowait(((addr, perf_counter()), SPVPong.decode(data)))
|
||||
except (ValueError, struct.error, AttributeError, TypeError, RuntimeError):
|
||||
return
|
||||
|
||||
def connection_made(self, transport) -> None:
|
||||
self.transport = transport
|
||||
|
||||
def connection_lost(self, exc: Optional[Exception]) -> None:
|
||||
self.transport = None
|
||||
log.info("closed udp spv server selection client")
|
||||
|
||||
def ping(self, server: Tuple[str, int]):
|
||||
self.transport.sendto(self._ping_packet, server)
|
||||
|
||||
def close(self):
|
||||
# log.info("close udp client")
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
@@ -6,6 +6,7 @@ from binascii import unhexlify
|
||||
from urllib.request import urlopen
|
||||
|
||||
from lbry.error import InsufficientFundsError
|
||||
from lbry.extras.daemon.comment_client import verify
|
||||
|
||||
from lbry.extras.daemon.daemon import DEFAULT_PAGE_SIZE
|
||||
from lbry.testcase import CommandTestCase
|
||||
@@ -460,6 +461,29 @@ class TransactionCommands(ClaimTestCase):
|
||||
|
||||
class TransactionOutputCommands(ClaimTestCase):
|
||||
|
||||
async def test_txo_list_by_channel_filtering(self):
|
||||
channel_foo = self.get_claim_id(await self.channel_create('@foo'))
|
||||
channel_bar = self.get_claim_id(await self.channel_create('@bar'))
|
||||
stream_a = self.get_claim_id(await self.stream_create('a', channel_id=channel_foo))
|
||||
stream_b = self.get_claim_id(await self.stream_create('b', channel_id=channel_foo))
|
||||
stream_c = self.get_claim_id(await self.stream_create('c', channel_id=channel_bar))
|
||||
stream_d = self.get_claim_id(await self.stream_create('d'))
|
||||
|
||||
r = await self.txo_list(type='stream')
|
||||
self.assertEqual({stream_a, stream_b, stream_c, stream_d}, {c['claim_id'] for c in r})
|
||||
|
||||
r = await self.txo_list(type='stream', channel_id=channel_foo)
|
||||
self.assertEqual({stream_a, stream_b}, {c['claim_id'] for c in r})
|
||||
|
||||
r = await self.txo_list(type='stream', channel_id=[channel_foo, channel_bar])
|
||||
self.assertEqual({stream_a, stream_b, stream_c}, {c['claim_id'] for c in r})
|
||||
|
||||
r = await self.txo_list(type='stream', not_channel_id=channel_foo)
|
||||
self.assertEqual({stream_c, stream_d}, {c['claim_id'] for c in r})
|
||||
|
||||
r = await self.txo_list(type='stream', not_channel_id=[channel_foo, channel_bar])
|
||||
self.assertEqual({stream_d}, {c['claim_id'] for c in r})
|
||||
|
||||
async def test_txo_list_and_sum_filtering(self):
|
||||
channel_id = self.get_claim_id(await self.channel_create())
|
||||
self.assertEqual('1.0', lbc(await self.txo_sum(type='channel', is_not_spent=True)))
|
||||
@@ -1004,6 +1028,18 @@ class ChannelCommands(CommandTestCase):
|
||||
self.assertItemCount(await self.daemon.jsonrpc_channel_list(account_id=self.account.id), 2)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_channel_list(account_id=account2_id), 1)
|
||||
|
||||
async def test_sign_hex_encoded_data(self):
|
||||
data_to_sign = "CAFEBABE"
|
||||
# claim new name
|
||||
await self.channel_create('@someotherchan')
|
||||
channel_tx = await self.daemon.jsonrpc_channel_create('@signer', '0.1')
|
||||
await self.confirm_tx(channel_tx.id)
|
||||
channel = channel_tx.outputs[0]
|
||||
signature1 = await self.out(self.daemon.jsonrpc_channel_sign(channel_name='@signer', hexdata=data_to_sign))
|
||||
signature2 = await self.out(self.daemon.jsonrpc_channel_sign(channel_id=channel.claim_id, hexdata=data_to_sign))
|
||||
self.assertTrue(verify(channel, unhexlify(data_to_sign), signature1))
|
||||
self.assertTrue(verify(channel, unhexlify(data_to_sign), signature2))
|
||||
|
||||
async def test_channel_export_import_before_sending_channel(self):
|
||||
# export
|
||||
tx = await self.channel_create('@foo', '1.0')
|
||||
|
||||
@@ -6,6 +6,7 @@ from unittest.mock import Mock
|
||||
from lbry.wallet.network import Network
|
||||
from lbry.wallet.orchstr8.node import SPVNode
|
||||
from lbry.wallet.rpc import RPCSession
|
||||
from lbry.wallet.server.udp import StatusServer
|
||||
from lbry.testcase import IntegrationTestCase, AsyncioTestCase
|
||||
|
||||
|
||||
@@ -32,18 +33,17 @@ class NetworkTests(IntegrationTestCase):
|
||||
'server_version': lbry.__version__,
|
||||
'trending_algorithm': 'zscore',
|
||||
}, await self.ledger.network.get_server_features())
|
||||
await self.conductor.spv_node.stop()
|
||||
# await self.conductor.spv_node.stop()
|
||||
payment_address, donation_address = await self.account.get_addresses(limit=2)
|
||||
await self.conductor.spv_node.start(
|
||||
self.conductor.blockchain_node,
|
||||
extraconf={
|
||||
'DESCRIPTION': 'Fastest server in the west.',
|
||||
'PAYMENT_ADDRESS': payment_address,
|
||||
'DONATION_ADDRESS': donation_address,
|
||||
'DAILY_FEE': '42'
|
||||
}
|
||||
)
|
||||
await self.ledger.network.on_connected.first
|
||||
self.conductor.spv_node.server.env.payment_address = payment_address
|
||||
self.conductor.spv_node.server.env.donation_address = donation_address
|
||||
self.conductor.spv_node.server.env.description = 'Fastest server in the west.'
|
||||
self.conductor.spv_node.server.env.daily_fee = '42'
|
||||
|
||||
from lbry.wallet.server.session import LBRYElectrumX
|
||||
LBRYElectrumX.set_server_features(self.conductor.spv_node.server.env)
|
||||
|
||||
# await self.ledger.network.on_connected.first
|
||||
self.assertDictEqual({
|
||||
'genesis_hash': self.conductor.spv_node.coin_class.GENESIS_HASH,
|
||||
'hash_function': 'sha256',
|
||||
@@ -65,22 +65,21 @@ class ReconnectTests(IntegrationTestCase):
|
||||
async def test_multiple_servers(self):
|
||||
# we have a secondary node that connects later, so
|
||||
node2 = SPVNode(self.conductor.spv_module, node_number=2)
|
||||
self.ledger.network.config['default_servers'].append((node2.hostname, node2.port))
|
||||
await asyncio.wait_for(self.ledger.stop(), timeout=1)
|
||||
await asyncio.wait_for(self.ledger.start(), timeout=1)
|
||||
self.ledger.network.session_pool.new_connection_event.clear()
|
||||
await node2.start(self.blockchain)
|
||||
# this is only to speed up the test as retrying would take 4+ seconds
|
||||
for session in self.ledger.network.session_pool.sessions:
|
||||
session.trigger_urgent_reconnect.set()
|
||||
await asyncio.wait_for(self.ledger.network.session_pool.new_connection_event.wait(), timeout=1)
|
||||
self.assertEqual(2, len(list(self.ledger.network.session_pool.available_sessions)))
|
||||
|
||||
self.ledger.network.config['default_servers'].append((node2.hostname, node2.port))
|
||||
self.ledger.network.config['default_servers'].reverse()
|
||||
self.assertEqual(50002, self.ledger.network.client.server[1])
|
||||
await self.ledger.stop()
|
||||
await self.ledger.start()
|
||||
|
||||
self.assertTrue(self.ledger.network.is_connected)
|
||||
switch_event = self.ledger.network.on_connected.first
|
||||
self.assertEqual(50003, self.ledger.network.client.server[1])
|
||||
await node2.stop(True)
|
||||
self.assertFalse(self.ledger.network.is_connected)
|
||||
await self.ledger.resolve([], ['derp'])
|
||||
self.assertEqual(50002, self.ledger.network.client.server[1])
|
||||
await node2.stop(True)
|
||||
# secondary down, but primary is ok, do not switch! (switches trigger new on_connected events)
|
||||
with self.assertRaises(asyncio.TimeoutError):
|
||||
await asyncio.wait_for(switch_event, timeout=1)
|
||||
|
||||
async def test_direct_sync(self):
|
||||
await self.ledger.stop()
|
||||
@@ -98,10 +97,13 @@ class ReconnectTests(IntegrationTestCase):
|
||||
async def test_connection_drop_still_receives_events_after_reconnected(self):
|
||||
address1 = await self.account.receiving.get_or_create_usable_address()
|
||||
# disconnect and send a new tx, should reconnect and get it
|
||||
self.ledger.network.client.connection_lost(Exception())
|
||||
self.ledger.network.client.transport.close()
|
||||
self.assertFalse(self.ledger.network.is_connected)
|
||||
await self.ledger.resolve([], 'derp')
|
||||
sendtxid = await self.blockchain.send_to_address(address1, 1.1337)
|
||||
await asyncio.wait_for(self.on_transaction_id(sendtxid), 1.0) # mempool
|
||||
# await self.ledger.resolve([], 'derp')
|
||||
# self.assertTrue(self.ledger.network.is_connected)
|
||||
await asyncio.wait_for(self.on_transaction_id(sendtxid), 10.0) # mempool
|
||||
await self.blockchain.generate(1)
|
||||
await self.on_transaction_id(sendtxid) # confirmed
|
||||
self.assertLess(self.ledger.network.client.response_time, 1) # response time properly set lower, we are fine
|
||||
@@ -123,7 +125,7 @@ class ReconnectTests(IntegrationTestCase):
|
||||
await self.blockchain.generate(1)
|
||||
# (this is just so the test doesn't hang forever if it doesn't reconnect)
|
||||
if not self.ledger.network.is_connected:
|
||||
await asyncio.wait_for(self.ledger.network.on_connected.first, timeout=1.0)
|
||||
await asyncio.wait_for(self.ledger.network.on_connected.first, timeout=10.0)
|
||||
# omg, the burned cable still works! torba is fire proof!
|
||||
await self.ledger.network.get_transaction(sendtxid)
|
||||
|
||||
@@ -136,15 +138,19 @@ class ReconnectTests(IntegrationTestCase):
|
||||
await self.ledger.network.on_connected.first
|
||||
self.assertTrue(self.ledger.network.is_connected)
|
||||
|
||||
async def test_online_but_still_unavailable(self):
|
||||
# Edge case. See issue #2445 for context
|
||||
self.assertIsNotNone(self.ledger.network.session_pool.fastest_session)
|
||||
for session in self.ledger.network.session_pool.sessions:
|
||||
session.response_time = None
|
||||
self.assertIsNone(self.ledger.network.session_pool.fastest_session)
|
||||
# async def test_online_but_still_unavailable(self):
|
||||
# # Edge case. See issue #2445 for context
|
||||
# self.assertIsNotNone(self.ledger.network.session_pool.fastest_session)
|
||||
# for session in self.ledger.network.session_pool.sessions:
|
||||
# session.response_time = None
|
||||
# self.assertIsNone(self.ledger.network.session_pool.fastest_session)
|
||||
|
||||
|
||||
class ServerPickingTestCase(AsyncioTestCase):
|
||||
async def _make_udp_server(self, port):
|
||||
s = StatusServer()
|
||||
await s.start(0, b'\x00' * 32, '127.0.0.1', port)
|
||||
self.addCleanup(s.stop)
|
||||
|
||||
async def _make_fake_server(self, latency=1.0, port=1):
|
||||
# local fake server with artificial latency
|
||||
@@ -156,6 +162,7 @@ class ServerPickingTestCase(AsyncioTestCase):
|
||||
return {'height': 1}
|
||||
server = await self.loop.create_server(lambda: FakeSession(), host='127.0.0.1', port=port)
|
||||
self.addCleanup(server.close)
|
||||
await self._make_udp_server(port)
|
||||
return '127.0.0.1', port
|
||||
|
||||
async def _make_bad_server(self, port=42420):
|
||||
@@ -164,9 +171,10 @@ class ServerPickingTestCase(AsyncioTestCase):
|
||||
writer.write(await reader.read())
|
||||
server = await asyncio.start_server(echo, host='127.0.0.1', port=port)
|
||||
self.addCleanup(server.close)
|
||||
await self._make_udp_server(port)
|
||||
return '127.0.0.1', port
|
||||
|
||||
async def test_pick_fastest(self):
|
||||
async def _test_pick_fastest(self):
|
||||
ledger = Mock(config={
|
||||
'default_servers': [
|
||||
# fast but unhealthy, should be discarded
|
||||
@@ -182,8 +190,8 @@ class ServerPickingTestCase(AsyncioTestCase):
|
||||
|
||||
network = Network(ledger)
|
||||
self.addCleanup(network.stop)
|
||||
asyncio.ensure_future(network.start())
|
||||
await asyncio.wait_for(network.on_connected.first, timeout=1)
|
||||
await network.start()
|
||||
await asyncio.wait_for(network.on_connected.first, timeout=10)
|
||||
self.assertTrue(network.is_connected)
|
||||
self.assertTupleEqual(network.client.server, ('127.0.0.1', 1337))
|
||||
self.assertTrue(all([not session.is_closing() for session in network.session_pool.available_sessions]))
|
||||
|
||||
@@ -11,7 +11,9 @@ class PurchaseCommandTests(CommandTestCase):
|
||||
await super().asyncSetUp()
|
||||
self.merchant_address = await self.blockchain.get_raw_change_address()
|
||||
|
||||
async def priced_stream(self, name='stream', price: Optional[str] = '2.0', currency='LBC') -> Transaction:
|
||||
async def priced_stream(
|
||||
self, name='stream', price: Optional[str] = '2.0', currency='LBC', mine=False
|
||||
) -> Transaction:
|
||||
kwargs = {}
|
||||
if price and currency:
|
||||
kwargs = {
|
||||
@@ -19,6 +21,8 @@ class PurchaseCommandTests(CommandTestCase):
|
||||
'fee_currency': currency,
|
||||
'fee_address': self.merchant_address
|
||||
}
|
||||
if not mine:
|
||||
kwargs['claim_address'] = self.merchant_address
|
||||
file_path = self.create_upload_file(data=b'high value content')
|
||||
tx = await self.daemon.jsonrpc_stream_create(
|
||||
name, '0.01', file_path=file_path, **kwargs
|
||||
@@ -182,3 +186,12 @@ class PurchaseCommandTests(CommandTestCase):
|
||||
await self.ledger.wait(spend)
|
||||
await self.assertBalance(self.account, '0.487695')
|
||||
self.assertItemCount(await self.daemon.jsonrpc_utxo_list(), 1)
|
||||
|
||||
async def test_owner_not_required_purchase_own_content(self):
|
||||
await self.priced_stream(mine=True)
|
||||
# check that `get` doesn't purchase own claim
|
||||
balance = await self.account.get_balance()
|
||||
response = await self.daemon.jsonrpc_get('lbry://stream')
|
||||
self.assertIsNone(response.content_fee)
|
||||
self.assertEqual(await self.account.get_balance(), balance)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_purchase_list(), 0)
|
||||
@@ -43,12 +43,14 @@ class WalletCommands(CommandTestCase):
|
||||
)
|
||||
|
||||
async def test_wallet_reconnect(self):
|
||||
status = await self.daemon.jsonrpc_status()
|
||||
self.assertEqual(len(status['wallet']['servers']), 1)
|
||||
self.assertEqual(status['wallet']['servers'][0]['port'], 50002)
|
||||
await self.conductor.spv_node.stop(True)
|
||||
self.conductor.spv_node.port = 54320
|
||||
await self.conductor.spv_node.start(self.conductor.blockchain_node)
|
||||
status = await self.daemon.jsonrpc_status()
|
||||
self.assertEqual(len(status['wallet']['servers']), 1)
|
||||
self.assertEqual(status['wallet']['servers'][0]['port'], 50002)
|
||||
self.assertEqual(len(status['wallet']['servers']), 0)
|
||||
self.daemon.jsonrpc_settings_set('lbryum_servers', ['localhost:54320'])
|
||||
await self.daemon.jsonrpc_wallet_reconnect()
|
||||
status = await self.daemon.jsonrpc_status()
|
||||
|
||||
@@ -4,6 +4,7 @@ import lbry
|
||||
import lbry.wallet
|
||||
from lbry.error import ServerPaymentFeeAboveMaxAllowedError
|
||||
from lbry.wallet.network import ClientSession
|
||||
from lbry.wallet.server.session import LBRYElectrumX
|
||||
from lbry.testcase import IntegrationTestCase, CommandTestCase
|
||||
from lbry.wallet.orchstr8.node import SPVNode
|
||||
|
||||
@@ -46,7 +47,7 @@ class TestSessions(IntegrationTestCase):
|
||||
|
||||
|
||||
class TestUsagePayment(CommandTestCase):
|
||||
async def test_single_server_payment(self):
|
||||
async def _test_single_server_payment(self):
|
||||
wallet_pay_service = self.daemon.component_manager.get_component('wallet_server_payments')
|
||||
wallet_pay_service.payment_period = 1
|
||||
# only starts with a positive max key fee
|
||||
@@ -68,21 +69,21 @@ class TestUsagePayment(CommandTestCase):
|
||||
self.addCleanup(node.stop)
|
||||
self.daemon.jsonrpc_settings_set('lbryum_servers', [f"{node.hostname}:{node.port}"])
|
||||
await self.daemon.jsonrpc_wallet_reconnect()
|
||||
|
||||
LBRYElectrumX.set_server_features(node.server.env)
|
||||
features = await self.ledger.network.get_server_features()
|
||||
self.assertEqual(features["payment_address"], address)
|
||||
self.assertEqual(features["daily_fee"], "1.1")
|
||||
with self.assertRaises(ServerPaymentFeeAboveMaxAllowedError):
|
||||
await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=8)
|
||||
|
||||
await node.stop(False)
|
||||
await node.start(self.blockchain, extraconf={"PAYMENT_ADDRESS": address, "DAILY_FEE": "1.0"})
|
||||
self.daemon.jsonrpc_settings_set('lbryum_servers', [f"{node.hostname}:{node.port}"])
|
||||
await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=30)
|
||||
node.server.env.daily_fee = "1.0"
|
||||
node.server.env.payment_address = address
|
||||
LBRYElectrumX.set_server_features(node.server.env)
|
||||
# self.daemon.jsonrpc_settings_set('lbryum_servers', [f"{node.hostname}:{node.port}"])
|
||||
await self.daemon.jsonrpc_wallet_reconnect()
|
||||
features = await self.ledger.network.get_server_features()
|
||||
self.assertEqual(features["payment_address"], address)
|
||||
self.assertEqual(features["daily_fee"], "1.0")
|
||||
tx = await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=8)
|
||||
tx = await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=30)
|
||||
self.assertIsNotNone(await self.blockchain.get_raw_transaction(tx.id)) # verify its broadcasted
|
||||
self.assertEqual(tx.outputs[0].amount, 100000000)
|
||||
self.assertEqual(tx.outputs[0].get_address(self.ledger), address)
|
||||
@@ -9,6 +9,10 @@ from lbry.wallet import Transaction
|
||||
|
||||
|
||||
class FileCommands(CommandTestCase):
|
||||
def __init__(self, *a, **kw):
|
||||
super().__init__(*a, **kw)
|
||||
self.skip_libtorrent = False
|
||||
|
||||
async def initialize_torrent(self, tx_to_update=None):
|
||||
if not hasattr(self, 'seeder_session'):
|
||||
self.seeder_session = TorrentSession(self.loop, None)
|
||||
@@ -354,7 +358,8 @@ class FileCommands(CommandTestCase):
|
||||
# FAIL: beyond available balance
|
||||
await self.stream_create(
|
||||
'expensive', '0.01', data=b'pay me if you can',
|
||||
fee_currency='LBC', fee_amount='11.0', fee_address=target_address
|
||||
fee_currency='LBC', fee_amount='11.0',
|
||||
fee_address=target_address, claim_address=target_address
|
||||
)
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='expensive')
|
||||
response = await self.out(self.daemon.jsonrpc_get('lbry://expensive'))
|
||||
@@ -364,7 +369,8 @@ class FileCommands(CommandTestCase):
|
||||
# FAIL: beyond maximum key fee
|
||||
await self.stream_create(
|
||||
'maxkey', '0.01', data=b'no pay me, no',
|
||||
fee_currency='LBC', fee_amount='111.0', fee_address=target_address
|
||||
fee_currency='LBC', fee_amount='111.0',
|
||||
fee_address=target_address, claim_address=target_address
|
||||
)
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='maxkey')
|
||||
response = await self.out(self.daemon.jsonrpc_get('lbry://maxkey'))
|
||||
@@ -376,7 +382,8 @@ class FileCommands(CommandTestCase):
|
||||
# PASS: purchase is successful
|
||||
await self.stream_create(
|
||||
'icanpay', '0.01', data=b'I got the power!',
|
||||
fee_currency='LBC', fee_amount='1.0', fee_address=target_address
|
||||
fee_currency='LBC', fee_amount='1.0',
|
||||
fee_address=target_address, claim_address=target_address
|
||||
)
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='icanpay')
|
||||
await self.assertBalance(self.account, '9.925679')
|
||||
@@ -427,7 +434,8 @@ class FileCommands(CommandTestCase):
|
||||
|
||||
await self.stream_create(
|
||||
'somename', '0.5', data=b'Yes, please',
|
||||
fee_currency='LBC', fee_amount='1.0', fee_address=target_address
|
||||
fee_currency='LBC', fee_amount='1.0',
|
||||
fee_address=target_address, claim_address=target_address
|
||||
)
|
||||
self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='somename'))
|
||||
# Assert the fee and bid are subtracted
|
||||
|
||||
@@ -7,7 +7,8 @@ from lbry.extras import cli
|
||||
from lbry.extras.daemon.components import (
|
||||
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
|
||||
HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT,
|
||||
LIBTORRENT_COMPONENT
|
||||
)
|
||||
from lbry.extras.daemon.daemon import Daemon
|
||||
|
||||
@@ -22,7 +23,8 @@ class CLIIntegrationTest(AsyncioTestCase):
|
||||
conf.components_to_skip = (
|
||||
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
|
||||
HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT,
|
||||
LIBTORRENT_COMPONENT
|
||||
)
|
||||
Daemon.component_attributes = {}
|
||||
self.daemon = Daemon(conf)
|
||||
|
||||
@@ -4,9 +4,9 @@ from lbry.extras.daemon.exchange_rate_manager import ExchangeRate, ExchangeRateM
|
||||
|
||||
|
||||
class TestExchangeRateManager(AsyncioTestCase):
|
||||
|
||||
async def test_exchange_rate_manager(self):
|
||||
# TODO: re-enable cryptonator.com
|
||||
# TODO: this uses real exchange rate feeds... update to use mocks
|
||||
manager = ExchangeRateManager(FEEDS)
|
||||
manager.start()
|
||||
self.addCleanup(manager.stop)
|
||||
@@ -18,5 +18,5 @@ class TestExchangeRateManager(AsyncioTestCase):
|
||||
self.assertTrue(feed.is_online)
|
||||
self.assertIsInstance(feed.rate, ExchangeRate)
|
||||
lbc = manager.convert_currency('USD', 'LBC', Decimal('0.01'))
|
||||
self.assertGreaterEqual(lbc, 0.1)
|
||||
self.assertGreaterEqual(lbc, 0.01)
|
||||
self.assertLessEqual(lbc, 10.0)
|
||||
@@ -93,8 +93,7 @@ class TestReflector(AsyncioTestCase):
|
||||
await incoming.wait()
|
||||
stop.set()
|
||||
# this used to raise (and then propagate) a CancelledError
|
||||
with self.assertRaises(asyncio.CancelledError):
|
||||
await reflect_task
|
||||
self.assertListEqual(await reflect_task, [])
|
||||
self.assertFalse(self.stream.is_fully_reflected)
|
||||
self.assertFalse(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
|
||||
|
||||
@@ -114,8 +113,8 @@ class TestReflector(AsyncioTestCase):
|
||||
await incoming.wait()
|
||||
await not_incoming.wait()
|
||||
stop.set()
|
||||
with self.assertRaises(asyncio.CancelledError):
|
||||
await reflect_task
|
||||
sent = await reflect_task
|
||||
self.assertListEqual([self.stream.sd_hash], sent)
|
||||
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
|
||||
self.assertFalse(self.stream.is_fully_reflected)
|
||||
|
||||
@@ -137,8 +136,8 @@ class TestReflector(AsyncioTestCase):
|
||||
await incoming.wait()
|
||||
await not_incoming.wait()
|
||||
stop.set()
|
||||
with self.assertRaises(asyncio.CancelledError):
|
||||
await reflect_task
|
||||
sent = await reflect_task
|
||||
self.assertListEqual([self.stream.sd_hash, self.stream.descriptor.blobs[0].blob_hash], sent)
|
||||
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
|
||||
self.assertTrue(self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified())
|
||||
self.assertFalse(self.stream.is_fully_reflected)
|
||||
@@ -160,9 +159,7 @@ class TestReflector(AsyncioTestCase):
|
||||
await not_incoming.wait()
|
||||
await incoming.wait()
|
||||
stop.set()
|
||||
with self.assertRaises(asyncio.CancelledError):
|
||||
await reflect_task
|
||||
# self.assertListEqual(await reflect_task, [self.stream.sd_hash])
|
||||
self.assertListEqual(await reflect_task, [self.stream.sd_hash])
|
||||
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
|
||||
self.assertFalse(
|
||||
self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified()
|
||||
@@ -187,8 +184,8 @@ class TestReflector(AsyncioTestCase):
|
||||
await incoming.wait()
|
||||
await self.stream_manager.delete(self.stream, delete_file=True)
|
||||
# this used to raise OSError when it can't read the deleted blob for the upload
|
||||
with self.assertRaises(asyncio.CancelledError):
|
||||
await reflect_task
|
||||
sent = await reflect_task
|
||||
self.assertListEqual([self.stream.sd_hash], sent)
|
||||
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
|
||||
self.assertFalse(
|
||||
self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified()
|
||||
|
||||
@@ -40,7 +40,7 @@ class MockNetwork:
|
||||
merkle = await self.get_merkle(tx_hash, known_height)
|
||||
return tx, merkle
|
||||
|
||||
async def get_transaction_batch(self, txids):
|
||||
async def get_transaction_batch(self, txids, restricted):
|
||||
return {
|
||||
txid: await self.get_transaction_and_merkle(txid)
|
||||
for txid in txids
|
||||
@@ -126,9 +126,7 @@ class TestSynchronization(LedgerTestCase):
|
||||
|
||||
self.ledger.network.get_history_called = []
|
||||
self.ledger.network.get_transaction_called = []
|
||||
self.assertFalse(self.ledger._tx_cache[txid1].tx.is_verified)
|
||||
self.assertFalse(self.ledger._tx_cache[txid2].tx.is_verified)
|
||||
self.assertFalse(self.ledger._tx_cache[txid3].tx.is_verified)
|
||||
self.assertEqual(0, len(self.ledger._tx_cache))
|
||||
await self.ledger.update_history(address, '')
|
||||
self.assertListEqual(self.ledger.network.get_history_called, [address])
|
||||
self.assertListEqual(self.ledger.network.get_transaction_called, [])
|
||||
|
||||
Reference in new issue
Block a user