Compare commits

...
Author SHA1 Message Date
Victor Shyba 51a9e0307d simple address subs instead of batch 2019-08-22 16:32:14 -03:00
Jack Robison 99556d1ce4 debug 2019-08-22 14:35:22 -04:00
Jack Robison a667278c99 switch_to_fastest 2019-08-22 13:51:08 -04:00
Jack Robison a432c47e62 logging 2019-08-22 13:51:07 -04:00
Jack Robison ff4b2f2dec Merge pull request #2411 from lbryio/fix-offline-connection-errors
Fix misc errors when going offline/online
2019-08-22 13:48:02 -04:00
Jack Robison aedce256af fix wallet status silently breaking when offline
-update wallet status to use real latency values and the full list of connected servers
2019-08-22 13:11:17 -04:00
Jack Robison 48b4b97132 fix uncaught exchange rate manager errors when going offline 2019-08-22 13:11:17 -04:00
Jack Robison 6b1f88df66 fix status breaking (and loggly) when the internet connection is lost 2019-08-22 13:11:17 -04:00
shyba d246c46227 Merge pull request #2397 from lbryio/unpack_leftovers
fix unpack error known leftovers
2019-08-22 13:41:25 -03:00
Victor Shyba 7fdd9dc293 fix annoying assertEquals warning 2019-08-21 22:03:24 -03:00
Victor Shyba 202861a900 fix unpack leftovers 2019-08-21 22:03:24 -03:00
Victor Shyba 4c41e7cb29 check headers bounds on access 2019-08-21 22:03:24 -03:00
Jack Robison 92823dcea9 Merge pull request #2407 from lbryio/temporary_speedup_ci
travis: remount /tmp as tmpfs
2019-08-21 20:08:27 -04:00
Victor Shyba 795a5227bc travis: remount /tmp as tmpfs 2019-08-21 20:05:39 -03:00
Alex Grintsvayg bbd4bb99b7 committing my claim search test script 2019-08-21 13:28:30 -04:00
Alex Grin f7443061ef less verbose logging 2019-08-21 09:54:10 -04:00
Jack Robison 416c3c16ab v0.39.2 2019-08-20 20:37:20 -04:00
Jack Robison a11956ece0 Merge pull request #2398 from lbryio/retriable_batcheable_networking
Improve wallet server selection
2019-08-20 20:10:05 -04:00
Jack Robison c4e0c659a7 fix missing perf_counter 2019-08-20 19:22:22 -04:00
Jack Robison f6ba09b171 mypy 2019-08-20 15:41:27 -04:00
Jack Robison b827881024 mypy 2019-08-20 15:35:06 -04:00
Jack Robison 89e395b5f4 pylint 2019-08-20 15:26:00 -04:00
Jack Robison c590954eb9 include connection latency in server selection 2019-08-20 15:11:01 -04:00
Jack Robison a5af3866fd time.time -> time.perf_counter 2019-08-20 14:56:13 -04:00
Jack Robison 1bef56f030 disconnect from duplicate servers (same ip, different dns) 2019-08-20 14:55:24 -04:00
Jack Robison 9045737504 track response time for server.version only 2019-08-20 14:54:55 -04:00
Jack Robison 0e48772c84 track connection latency 2019-08-20 14:45:53 -04:00
Jack Robison 72690b5cff raise asyncio.TimeoutError 2019-08-20 14:19:50 -04:00
Jack Robison 07bae26fd3 close connection and reraise on CancelledError 2019-08-20 14:16:56 -04:00
Victor Shyba 7b0e4617d3 fix db test stability 2019-08-20 04:11:04 -03:00
Victor Shyba af797296ed rolling averages on response time 2019-08-19 16:50:24 -03:00
Victor Shyba c826c7da0d take load into consideration when picking fastest 2019-08-19 14:02:30 -03:00
Victor Shyba 20af1396b9 refactor stop 2019-08-19 14:02:30 -03:00
Victor Shyba f567aca532 retry and batch requests, fix some loose ends 2019-08-19 14:02:30 -03:00
24 changed files with 289 additions and 526 deletions

No files matched your search

+1
View File
@@ -25,6 +25,7 @@ jobs:
- name: "LBRY Integration Tests"
install:
- pip install coverage tox-travis
- sudo mount -o mode=1777,nosuid,nodev -t tmpfs tmpfs /tmp
script: cd lbry && tox
after_success:
- coverage combine lbry
+1 -1
View File
@@ -1,4 +1,4 @@
__name__ = "lbry"
__version__ = "0.39.1"
__version__ = "0.39.2"
version = tuple(__version__.split('.'))
+1 -1
View File
@@ -151,7 +151,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
return self._blob_bytes_received, self.close()
msg = f"downloading {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}," \
f" timeout in {self.peer_timeout}"
log.info(msg)
log.debug(msg)
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
log.info("%s at %fMB/s", msg,
+1 -2
View File
@@ -22,13 +22,12 @@ class BlobAnnouncer:
if peers > 4:
return blob_hash
else:
log.warning("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
log.debug("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
except Exception as err:
if isinstance(err, asyncio.CancelledError):
raise err
log.warning("error announcing %s: %s", blob_hash[:8], str(err))
async def _announce(self, batch_size: typing.Optional[int] = 10):
while batch_size:
if not self.node.joined.is_set():
+2 -2
View File
@@ -88,10 +88,10 @@ class Node:
)
stored_to = [node_id for node_id, contacted in stored_to_tup if contacted]
if stored_to:
log.info("Stored %s to %i of %i attempted peers", binascii.hexlify(hash_value).decode()[:8],
log.debug("Stored %s to %i of %i attempted peers", binascii.hexlify(hash_value).decode()[:8],
len(stored_to), len(peers))
else:
log.warning("Failed announcing %s, stored to 0 peers", blob_hash[:8])
log.debug("Failed announcing %s, stored to 0 peers", blob_hash[:8])
return stored_to
def stop(self) -> None:
+6 -5
View File
@@ -218,6 +218,7 @@ class HeadersComponent(Component):
await self.headers.close()
else:
log.info("Header checkpoint verified.")
async def stop(self):
pass
@@ -239,14 +240,14 @@ class WalletComponent(Component):
local_height = self.wallet_manager.ledger.headers.height
remote_height = self.wallet_manager.ledger.network.remote_height
best_hash = self.wallet_manager.get_best_blockhash()
server = self.wallet_manager.ledger.network.client.server
return {
'connected_servers': [
{
'host': server[0],
'port': server[1],
'latency': 0 # TODO: use real latency
}
'host': session.server[0],
'port': session.server[1],
'latency': round(session.connection_latency, 2),
} for session in self.wallet_manager.ledger.network.session_pool.sessions
if session and session.available
],
'blocks': max(local_height, 0),
'blocks_behind': max(remote_height - local_height, 0),
+4
View File
@@ -392,6 +392,10 @@ class Daemon(metaclass=JSONRPCServerType):
async def update_connection_status(self):
connected = await utils.async_check_connection()
if connected and not self._connection_status[1]:
log.info("detected internet connection is working")
elif not connected and self._connection_status[1]:
log.warning("detected internet connection was lost")
self._connection_status = (self.component_manager.loop.time(), connected)
async def get_connection_status(self) -> str:
@@ -3,7 +3,8 @@ import time
import logging
import json
from decimal import Decimal
from typing import Optional
from aiohttp.client_exceptions import ClientError
from lbry.error import InvalidExchangeRateResponse, CurrencyConversionError
from lbry.utils import aiohttp_request
@@ -44,7 +45,7 @@ class MarketFeed:
self.params = params
self.fee = fee
self.rate = None
self._task: asyncio.Task = None
self._task: Optional[asyncio.Task] = None
self._online = True
def rate_is_initialized(self):
@@ -80,7 +81,7 @@ class MarketFeed:
try:
response = await asyncio.wait_for(self._make_request(), self.REQUESTS_TIMEOUT)
self._save_price(self._subtract_fee(self._handle_response(response)))
except (asyncio.TimeoutError, InvalidExchangeRateResponse) as err:
except (asyncio.TimeoutError, InvalidExchangeRateResponse, ClientError) as err:
self._on_error(err)
await asyncio.sleep(self.EXCHANGE_RATE_UPDATE_RATE_SEC)
@@ -148,7 +148,7 @@ class JSONResponseEncoder(JSONEncoder):
'amount': dewies_to_lbc(txo.amount),
'address': txo.get_address(self.ledger),
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
'timestamp': self.ledger.headers[tx_height]['timestamp'] if tx_height > 0 else None
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None
}
if txo.is_change is not None:
output['is_change'] = txo.is_change
@@ -200,7 +200,7 @@ class JSONResponseEncoder(JSONEncoder):
if key.endswith('_amount'):
if isinstance(value, int):
meta[key] = dewies_to_lbc(value)
if meta.get('creation_height', 0) > 0:
if 0 < meta.get('creation_height', 0) <= self.ledger.headers.height:
meta['creation_timestamp'] = self.ledger.headers[meta['creation_height']]['timestamp']
return meta
@@ -224,7 +224,7 @@ class JSONResponseEncoder(JSONEncoder):
file.update({
'height': tx_height,
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
'timestamp': self.ledger.headers[tx_height]['timestamp'] if tx_height > 0 else None
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None
})
return file
+7 -3
View File
@@ -1,4 +1,5 @@
import asyncio
from aiohttp.client_exceptions import ClientError
import json
import logging.handlers
import traceback
@@ -49,9 +50,12 @@ class HTTPSLogglyHandler(logging.Handler):
return record.getMessage()
async def _emit(self, record):
payload = self.format(record)
async with utils.aiohttp_request('post', self.url, data=payload.encode(), cookies=self.cookies) as response:
self.cookies.update(response.cookies)
try:
async with utils.aiohttp_request('post', self.url, data=self.format(record).encode(),
cookies=self.cookies) as response:
self.cookies.update(response.cookies)
except ClientError:
pass
def emit(self, record):
asyncio.ensure_future(self._emit(record))
+5 -11
View File
@@ -81,28 +81,22 @@ def check_connection(server="lbry.com", port=80, timeout=5) -> bool:
try:
server = socket.gethostbyname(server)
socket.create_connection((server, port), timeout).close()
log.debug('Connection successful')
return True
except (socket.gaierror, socket.herror) as ex:
log.warning("Failed to connect to %s:%s. Unable to resolve domain. Trying to bypass DNS",
server, port)
log.debug("Failed to connect to %s:%s. Unable to resolve domain. Trying to bypass DNS",
server, port)
try:
server = "8.8.8.8"
port = 53
socket.create_connection((server, port), timeout).close()
log.debug('Connection successful')
return True
except Exception:
log.error("Failed to connect to %s:%s. Maybe the internet connection is not working",
server, port)
except OSError:
return False
except Exception:
log.error("Failed to connect to %s:%s. Maybe the internet connection is not working",
server, port)
except OSError:
return False
async def async_check_connection(server="lbry.com", port=80, timeout=5) -> bool:
async def async_check_connection(server="lbry.com", port=80, timeout=1) -> bool:
return await asyncio.get_event_loop().run_in_executor(None, check_connection, server, port, timeout)
-2
View File
@@ -9,7 +9,6 @@ from torba.client.baseaccount import SingleKey
from lbry.schema.result import Outputs
from lbry.schema.url import URL
from lbry.wallet.dewies import dewies_to_lbc
from lbry.wallet.resolve import Resolver
from lbry.wallet.account import Account
from lbry.wallet.network import Network
from lbry.wallet.database import WalletDatabase
@@ -52,7 +51,6 @@ class MainNetLedger(BaseLedger):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.fee_per_name_char = self.config.get('fee_per_name_char', self.default_fee_per_name_char)
self.resolver = Resolver(self)
async def _inflate_outputs(self, query):
outputs = Outputs.from_base64(await query)
-404
View File
@@ -1,404 +0,0 @@
import logging
import asyncio
from binascii import unhexlify, hexlify
from lbry.utils import lru_cache_concurrent
from lbry.wallet.account import validate_claim_id
from lbry.wallet.dewies import dewies_to_lbc
from lbry.error import UnknownNameError, UnknownClaimID, UnknownURI, UnknownOutpoint
from lbry.schema.claim import Claim
from google.protobuf.message import DecodeError
from lbry.schema.url import URL
from lbry.wallet.claim_proofs import verify_proof, InvalidProofError
log = logging.getLogger(__name__)
class Resolver:
def __init__(self, ledger):
self.transaction_class = ledger.transaction_class
self.network = ledger.network
self.ledger = ledger
async def resolve(self, page, page_size, *uris):
uris = set(uris)
try:
for uri in uris:
for part in URL.parse(uri).parts:
if part.claim_id:
validate_claim_id(part.claim_id)
claim_trie_root = self.ledger.headers.claim_trie_root
resolutions = await self.network.get_values_for_uris(self.ledger.headers.hash().decode(), *uris)
if len(uris) > 1:
return await self._batch_handle(resolutions, uris, page, page_size, claim_trie_root)
return await self._handle_resolutions(resolutions, uris, page, page_size, claim_trie_root)
except ValueError as err:
return {'error': err.args[0]}
except Exception as e:
log.exception(e)
return {'error': str(e)}
async def _batch_handle(self, resolutions, uris, page, page_size, claim_trie_root):
futs = []
for uri in uris:
futs.append(
asyncio.ensure_future(self._handle_resolutions(resolutions, [uri], page, page_size, claim_trie_root))
)
results = await asyncio.gather(*futs)
return dict(list(map(lambda result: list(result.items())[0], results)))
@lru_cache_concurrent(256)
async def _fetch_tx(self, txid):
return self.transaction_class(unhexlify(await self.network.get_transaction(txid)))
async def _handle_resolutions(self, resolutions, requested_uris, page, page_size, claim_trie_root):
results = {}
for uri in requested_uris:
resolution = (resolutions or {}).get(uri, {})
if resolution:
try:
results[uri] = _handle_claim_result(
await self._handle_resolve_uri_response(uri, resolution, claim_trie_root, page, page_size),
uri
)
except (UnknownNameError, UnknownClaimID, UnknownURI) as err:
log.exception(err)
results[uri] = {'error': str(err)}
else:
results[uri] = {'error': "URI lbry://{} cannot be resolved".format(uri.replace("lbry://", ""))}
return results
async def _handle_resolve_uri_response(self, uri, resolution, claim_trie_root, page=0, page_size=10):
result = {}
parsed_uri = URL.parse(uri)
certificate_response = None
# parse an included certificate
if 'certificate' in resolution:
certificate_response = resolution['certificate']['result']
certificate_resolution_type = resolution['certificate']['resolution_type']
if certificate_resolution_type == "winning" and certificate_response:
if 'height' in certificate_response:
certificate_response = _verify_proof(parsed_uri.stream.name,
claim_trie_root,
certificate_response,
ledger=self.ledger)
elif certificate_resolution_type not in ['winning', 'claim_id', 'sequence']:
raise Exception(f"unknown response type: {certificate_resolution_type}")
result['certificate'] = await self.parse_and_validate_claim_result(certificate_response)
result['claims_in_channel'] = len(resolution.get('unverified_claims_in_channel', []))
# if this was a resolution for a name, parse the result
if 'claim' in resolution:
claim_response = resolution['claim']['result']
claim_resolution_type = resolution['claim']['resolution_type']
if claim_resolution_type == "winning" and claim_response:
if 'height' in claim_response:
claim_response = _verify_proof(parsed_uri.stream.name,
claim_trie_root,
claim_response,
ledger=self.ledger)
elif claim_resolution_type not in ["sequence", "winning", "claim_id"]:
raise Exception(f"unknown response type: {claim_resolution_type}")
result['claim'] = await self.parse_and_validate_claim_result(claim_response,
certificate_response)
# if this was a resolution for a name in a channel make sure there is only one valid
# match
elif 'unverified_claims_for_name' in resolution and 'certificate' in result:
unverified_claims_for_name = resolution['unverified_claims_for_name']
channel_info = await self.get_channel_claims_page(unverified_claims_for_name,
result['certificate'], page=1)
claims_in_channel, upper_bound = channel_info
if not claims_in_channel:
log.error("No valid claims for this name for this channel")
elif len(claims_in_channel) > 1:
log.warning("Multiple signed claims for the same name.")
winner = pick_winner_from_channel_path_collision(claims_in_channel)
if winner:
result['claim'] = winner
else:
log.error("No valid claims for this name for this channel")
else:
result['claim'] = claims_in_channel[0]
# parse and validate claims in a channel iteratively into pages of results
elif 'unverified_claims_in_channel' in resolution and 'certificate' in result:
ids_to_check = resolution['unverified_claims_in_channel']
channel_info = await self.get_channel_claims_page(ids_to_check, result['certificate'],
page=page, page_size=page_size)
claims_in_channel, upper_bound = channel_info
if claims_in_channel:
result['total_claims'] = upper_bound
result['claims_in_channel'] = claims_in_channel
elif 'error' not in result:
return {'error': 'claim not found', 'success': False, 'uri': str(parsed_uri)}
# invalid signatures can only return outside a channel
if result.get('claim', {}).get('has_signature', False):
if parsed_uri.has_stream and not result['claim']['signature_is_valid']:
return {'error': 'claim not found', 'success': False, 'uri': str(parsed_uri)}
return result
async def parse_and_validate_claim_result(self, claim_result, certificate=None):
if not claim_result or 'value' not in claim_result:
return claim_result
claim_result = _decode_claim_result(claim_result)
if claim_result.get('height'):
claim_result['timestamp'] = self.ledger.headers[claim_result['height']]['timestamp']
if claim_result.get('depth'):
claim_result['confirmations'] = claim_result.pop('depth')
if claim_result['value']:
claim_result['has_signature'] = False
if claim_result['value'].is_signed:
claim_result['has_signature'] = True
claim_tx = await self._fetch_tx(claim_result['txid'])
if certificate is None:
log.info("fetching certificate to check claim signature")
channel_id = claim_result['value'].signing_channel_id
certificate = (await self.network.get_claims_by_ids([channel_id])).get(channel_id)
if not certificate:
log.warning('Certificate %s not found', channel_id)
claim_result['channel_name'] = certificate['name'] if certificate else None
cert_tx = await self._fetch_tx(certificate['txid']) if certificate else None
claim_result['signature_is_valid'] = validate_claim_signature_and_get_channel_name(
claim_result, certificate, self.ledger, claim_tx=claim_tx, cert_tx=cert_tx
)
# fixme: workaround while json encoder isnt used here
if cert_tx:
channel_txo = cert_tx.outputs[certificate['nout']]
claim_result['signing_channel'] = {
'name': channel_txo.claim_name,
'claim_id': channel_txo.claim_id,
'value': channel_txo.claim
}
claim_result['is_channel_signature_valid'] = claim_result['signature_is_valid']
if 'amount' in claim_result:
claim_result['amount'] = dewies_to_lbc(claim_result['amount'])
claim_result['effective_amount'] = dewies_to_lbc(claim_result['effective_amount'])
claim_result['supports'] = [
{'txid': txid, 'nout': nout, 'amount': dewies_to_lbc(amount)}
for (txid, nout, amount) in claim_result['supports']
]
claim_result['height'] = claim_result.get('height', -1) or -1
claim_result['permanent_url'] = f"lbry://{claim_result['name']}#{claim_result['claim_id']}"
return claim_result
@staticmethod
def prepare_claim_queries(start_position, query_size, channel_claim_infos):
queries = [tuple()]
names = {}
# a table of index counts for the sorted claim ids, including ignored claims
absolute_position_index = {}
block_sorted_infos = sorted(channel_claim_infos.items(), key=lambda x: int(x[1][1]))
per_block_infos = {}
for claim_id, (name, height) in block_sorted_infos:
claims = per_block_infos.get(height, [])
claims.append((claim_id, name))
per_block_infos[height] = sorted(claims, key=lambda x: int(x[0], 16))
abs_position = 0
for height in sorted(per_block_infos.keys(), reverse=True):
for claim_id, name in per_block_infos[height]:
names[claim_id] = name
absolute_position_index[claim_id] = abs_position
if abs_position >= start_position:
if len(queries[-1]) >= query_size:
queries.append(tuple())
queries[-1] += (claim_id,)
abs_position += 1
return queries, names, absolute_position_index
async def iter_channel_claims_pages(self, queries, claim_positions, claim_names, certificate,
page_size=10):
# lbryum server returns a dict of {claim_id: (name, claim_height)}
# first, sort the claims by block height (and by claim id int value within a block).
# map the sorted claims into getclaimsbyids queries of query_size claim ids each
# send the batched queries to lbryum server and iteratively validate and parse
# the results, yield a page of results at a time.
# these results can include those where `signature_is_valid` is False. if they are skipped,
# page indexing becomes tricky, as the number of results isn't known until after having
# processed them.
# TODO: fix ^ in lbry.schema
async def iter_validate_channel_claims():
formatted_claims = []
for claim_ids in queries:
batch_result = await self.network.get_claims_by_ids(claim_ids)
for claim_id in claim_ids:
claim = batch_result[claim_id]
if claim['name'] == claim_names[claim_id]:
formatted_claim = await self.parse_and_validate_claim_result(claim, certificate)
formatted_claim['absolute_channel_position'] = claim_positions[
claim['claim_id']]
formatted_claims.append(formatted_claim)
else:
log.warning("ignoring claim with name mismatch %s %s", claim['name'],
claim['claim_id'])
return formatted_claims
results = []
for claim in await iter_validate_channel_claims():
results.append(claim)
# if there is a full page of results, yield it
if len(results) and len(results) % page_size == 0:
return results[-page_size:]
return results
async def get_channel_claims_page(self, channel_claim_infos, certificate, page, page_size=10):
page = page or 0
page_size = max(page_size, 1)
if page_size > 500:
raise Exception("page size above maximum allowed")
start_position = (page - 1) * page_size
queries, names, claim_positions = self.prepare_claim_queries(start_position, page_size,
channel_claim_infos)
upper_bound = len(claim_positions)
if not page:
return None, upper_bound
if start_position > upper_bound:
raise IndexError("claim %i greater than max %i" % (start_position, upper_bound))
page_generator = await self.iter_channel_claims_pages(queries, claim_positions, names,
certificate, page_size=page_size)
return page_generator, upper_bound
def _verify_proof(name, claim_trie_root, result, ledger):
"""
Verify proof for name claim
"""
support_amount = sum([amt for (stxid, snout, amt) in result['supports']])
def _build_response(name, tx, nOut):
output = tx.outputs[nOut]
r = {
'name': name,
'value': hexlify(output.script.values['claim']),
'claim_id': output.claim_id,
'txid': tx.id,
'nout': nOut,
'amount': output.amount,
'effective_amount': output.amount + support_amount,
'height': result['height'],
'confirmations': result['depth'],
'claim_sequence': result['claim_sequence'],
'address': output.get_address(ledger),
'valid_at_height': result['valid_at_height'],
'timestamp': ledger.headers[result['height']]['timestamp'],
'supports': result['supports']
}
return r
def _parse_proof_result(name, result):
if 'txhash' in result['proof'] and 'nOut' in result['proof']:
if 'transaction' in result:
tx = ledger.transaction_class(raw=unhexlify(result['transaction']))
nOut = result['proof']['nOut']
if result['proof']['txhash'] == tx.id:
if 0 <= nOut < len(tx.outputs):
if tx.outputs[nOut].claim_name == name:
return _build_response(name, tx, nOut)
return {'error': 'name in proof did not match requested name'}
outputs = len(tx['outputs'])
return {'error': 'invalid nOut: %d (let(outputs): %d' % (nOut, outputs)}
return {'error': "computed txid did not match given transaction: %s vs %s" %
(tx.id, result['proof']['txhash'])
}
return {'error': "didn't receive a transaction with the proof"}
return {'error': 'name is not claimed'}
if 'proof' in result:
name = result.get('name', name)
proof_name = result.get('normalized_name', name)
try:
verify_proof(result['proof'], claim_trie_root, proof_name)
except InvalidProofError:
return {'error': "Proof was invalid"}
return _parse_proof_result(name, result)
else:
return {'error': "proof not in result"}
def validate_claim_signature_and_get_channel_name(claim_result, certificate_claim, ledger,
claim_tx=None, cert_tx=None):
valid_signature = False
if cert_tx and certificate_claim and claim_tx and claim_result:
valid_signature = claim_tx.outputs[claim_result['nout']].is_signed_by(
cert_tx.outputs[certificate_claim['nout']], ledger
)
if not valid_signature:
log.warning("lbry://%s#%s has an invalid signature",
claim_result['name'], claim_result['claim_id'])
return valid_signature
# TODO: The following came from code handling lbryum results. Now that it's all in one place a refactor should unify it.
def _decode_claim_result(claim):
if 'decoded_claim' in claim:
return claim
if 'value' not in claim:
log.warning('Got an invalid claim while parsing, please report: %s', claim)
claim['protobuf'] = None
claim['value'] = None
backend_message = ' SDK message: ' + claim.get('error', '')
claim['error'] = "Failed to parse: missing value." + backend_message
return claim
try:
if not isinstance(claim['value'], Claim):
claim['value'] = Claim.from_bytes(unhexlify(claim['value']))
claim['protobuf'] = hexlify(claim['value'].to_bytes())
claim['decoded_claim'] = True
except DecodeError:
claim['decoded_claim'] = False
claim['protobuf'] = claim['value']
claim['value'] = None
return claim
def _handle_claim_result(results, uri):
if not results:
raise UnknownURI(uri)
if 'error' in results:
if results['error'] in ['name is not claimed', 'claim not found']:
if 'claim_id' in results:
raise UnknownClaimID(results['claim_id'])
if 'name' in results:
raise UnknownNameError(results['name'])
if 'uri' in results:
raise UnknownURI(results['uri'])
if 'outpoint' in results:
raise UnknownOutpoint(results['outpoint'])
raise Exception(results['error'])
if not {'value', 'claim', 'certificate'}.intersection(results.keys()):
raise Exception(f'result in unexpected format:{results}')
return results
def pick_winner_from_channel_path_collision(claims_in_channel):
# we should be doing this by effective amount so we pick the controlling claim, however changing the resolved
# claim triggers another issue where 2 claims cant be saved for the same file. This code picks the oldest, so it
# stays the same. Using effective amount would change the resolved claims for a channel path on takeovers,
# potentially triggering that.
winner = {}
for claim in claims_in_channel:
if not winner or claim['height'] < winner['height'] or \
(claim['height'] == winner['height'] and claim['nout'] < winner['nout']):
winner = claim if claim['signature_is_valid'] else winner
return winner or None
+47
View File
@@ -0,0 +1,47 @@
import asyncio
from torba.client.basenetwork import ClientSession
from torba.rpc.jsonrpc import RPCError
import logging
import json
import sys
logging.getLogger('torba').setLevel(logging.CRITICAL)
async def main():
try:
hostname = sys.argv[1]
except IndexError:
hostname = 'spv1.lbry.com'
loop = asyncio.get_event_loop()
client = ClientSession(network=None, server=(hostname, 50001))
error = None
args = {
'any_tags': ['art'],
'not_tags': ['xxx', 'porn', 'mature', 'nsfw', 'titan'],
'order_by': ["name"],
'offset': 3000,
'limit': 200,
'no_totals': False,
}
start = loop.time()
try:
await client.create_connection()
try:
await client.send_request('blockchain.claimtrie.search', args)
except RPCError as err:
error = err
finally:
await client.close()
finally:
print(json.dumps({
"time": loop.time() - start,
"error": error.__str__() if error else None,
"args": args,
}, indent=4))
if __name__ == "__main__":
asyncio.run(main())
+1 -1
View File
@@ -34,7 +34,7 @@ class DHTIntegrationTest(AsyncioTestCase):
async def test_replace_bad_nodes(self):
await self.setup_network(20)
self.assertEquals(len(self.nodes), 20)
self.assertEqual(len(self.nodes), 20)
node = self.nodes[0]
bad_peers = []
for candidate in self.nodes[1:10]:
@@ -255,7 +255,7 @@ class TestQueries(AsyncioTestCase):
self.ledger.db.db.execute_fetchall = check_parameters_length
account = await self.create_account()
tx = await self.create_tx_from_nothing(account, 0)
for height in range(1200):
for height in range(1, 1200):
tx = await self.create_tx_from_txo(tx.outputs[0], account, height=height)
variable_limit = self.ledger.db.MAX_QUERY_VARIABLES
for limit in range(variable_limit-2, variable_limit+2):
@@ -94,6 +94,17 @@ class BasicHeadersTests(BitcoinHeadersTestCase):
await headers.connect(len(headers), remainder)
self.assertEqual(headers.height, 32259)
async def test_bounds(self):
headers = MainHeaders(':memory:')
await headers.connect(0, self.get_bytes(block_bytes(3001)))
self.assertEqual(headers.height, 3000)
with self.assertRaises(IndexError):
_ = headers[3001]
with self.assertRaises(IndexError):
_ = headers[-1]
self.assertIsNotNone(headers[3000])
self.assertIsNotNone(headers[0])
async def test_repair(self):
headers = MainHeaders(':memory:')
await headers.connect(0, self.get_bytes(block_bytes(3001)))
+5 -1
View File
@@ -18,6 +18,9 @@ class MockNetwork:
self.get_transaction_called = []
self.is_connected = False
def retriable_call(self, function, *args, **kwargs):
return function(*args, **kwargs)
async def get_history(self, address):
self.get_history_called.append(address)
self.address = address
@@ -121,8 +124,9 @@ class TestSynchronization(LedgerTestCase):
)
class MocHeaderNetwork:
class MocHeaderNetwork(MockNetwork):
def __init__(self, responses):
super().__init__(None, None)
self.responses = responses
async def get_headers(self, height, blocks):
+4 -2
View File
@@ -70,8 +70,10 @@ class BaseHeaders:
return True
def __getitem__(self, height) -> dict:
assert not isinstance(height, slice), \
"Slicing of header chain has not been implemented yet."
if isinstance(height, slice):
raise NotImplementedError("Slicing of header chain has not been implemented yet.")
if not 0 <= height <= self.height:
raise IndexError(f"{height} is out of bounds, current height: {self.height}")
return self.deserialize(height, self.get_raw_header(height))
def get_raw_header(self, height) -> bytes:
+7 -6
View File
@@ -310,7 +310,7 @@ class BaseLedger(metaclass=LedgerRegistry):
subscription_update = False
if not headers:
header_response = await self.network.get_headers(height, 2001)
header_response = await self.network.retriable_call(self.network.get_headers, height, 2001)
headers = header_response['hex']
if not headers:
@@ -417,7 +417,7 @@ class BaseLedger(metaclass=LedgerRegistry):
if local_status == remote_status:
return
remote_history = await self.network.get_history(address)
remote_history = await self.network.retriable_call(self.network.get_history, address)
cache_tasks = []
synced_history = StringIO()
@@ -489,7 +489,7 @@ class BaseLedger(metaclass=LedgerRegistry):
if tx is None:
# fetch from network
_raw = await self.network.get_transaction(txid)
_raw = await self.network.retriable_call(self.network.get_transaction, txid)
if _raw:
tx = self.transaction_class(unhexlify(_raw))
await self.maybe_verify_transaction(tx, remote_height)
@@ -509,8 +509,8 @@ class BaseLedger(metaclass=LedgerRegistry):
async def maybe_verify_transaction(self, tx, remote_height):
tx.height = remote_height
if 0 < remote_height <= len(self.headers):
merkle = await self.network.get_merkle(tx.id, remote_height)
if 0 < remote_height < len(self.headers):
merkle = await self.network.retriable_call(self.network.get_merkle, tx.id, remote_height)
merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash)
header = self.headers[remote_height]
tx.position = merkle['pos']
@@ -524,6 +524,7 @@ class BaseLedger(metaclass=LedgerRegistry):
return None
def broadcast(self, tx):
# broadcast cant be a retriable call yet
return self.network.broadcast(hexlify(tx.raw).decode())
async def wait(self, tx: basetransaction.BaseTransaction, height=-1, timeout=None):
@@ -545,4 +546,4 @@ class BaseLedger(metaclass=LedgerRegistry):
)) for address_record in records
], timeout=timeout)
if pending:
raise TimeoutError('Timed out waiting for transaction.')
raise asyncio.TimeoutError('Timed out waiting for transaction.')
+160 -65
View File
@@ -1,8 +1,8 @@
import logging
import asyncio
from operator import itemgetter
from typing import Dict, Optional
from time import time, perf_counter
from typing import Dict, Optional, Tuple
from time import perf_counter
from torba.rpc import RPCSession as BaseClientSession, Connector, RPCError
@@ -13,17 +13,20 @@ log = logging.getLogger(__name__)
class ClientSession(BaseClientSession):
def __init__(self, *args, network, server, timeout=30, on_connect_callback=None, **kwargs):
self.network = network
self.server = server
super().__init__(*args, **kwargs)
self._on_disconnect_controller = StreamController()
self.on_disconnected = self._on_disconnect_controller.stream
self.bw_limit = self.framer.max_size = self.max_errors = 1 << 32
self.framer.max_size = self.max_errors = 1 << 32
self.bw_limit = -1
self.timeout = timeout
self.max_seconds_idle = timeout * 2
self.response_time: Optional[float] = None
self.connection_latency: Optional[float] = None
self._response_samples = 0
self.pending_amount = 0
self._on_connect_cb = on_connect_callback or (lambda: None)
self.trigger_urgent_reconnect = asyncio.Event()
@@ -31,20 +34,55 @@ class ClientSession(BaseClientSession):
def available(self):
return not self.is_closing() and self._can_send.is_set() and self.response_time is not None
async def send_request(self, method, args=()):
@property
def server_address_and_port(self) -> Optional[Tuple[str, int]]:
if not self.transport:
return None
return self.transport.get_extra_info('peername')
async def send_timed_server_version_request(self, args=(), timeout=None):
timeout = timeout or self.timeout
log.debug("send version request to %s:%i", *self.server)
start = perf_counter()
result = await asyncio.wait_for(
super().send_request('server.version', args), timeout=timeout
)
current_response_time = perf_counter() - start
response_sum = (self.response_time or 0) * self._response_samples + current_response_time
self.response_time = response_sum / (self._response_samples + 1)
self._response_samples += 1
return result
async def send_request(self, method, args=(), timeout=None):
log.info("send %s to %s:%i", method, *self.server)
timeout = timeout or self.timeout
self.pending_amount += 1
try:
start = perf_counter()
result = await asyncio.wait_for(
super().send_request(method, args), timeout=self.timeout
)
self.response_time = perf_counter() - start
return result
if method == 'server.version':
reply = await self.send_timed_server_version_request(args, timeout)
else:
reply = await asyncio.wait_for(
super().send_request(method, args), timeout=timeout
)
log.info("got reply for %s from %s:%i", method, *self.server)
return reply
except RPCError as e:
log.warning("Wallet server returned an error. Code: %s Message: %s", *e.args)
log.warning("Wallet server (%s:%i) returned an error. Code: %s Message: %s",
*self.server, *e.args)
raise e
except TimeoutError:
self.response_time = None
except ConnectionError:
log.warning("connection to %s:%i lost", *self.server)
self.synchronous_close()
raise asyncio.CancelledError(f"connection to {self.server[0]}:{self.server[1]} lost")
except asyncio.TimeoutError:
log.info("timeout sending %s to %s:%i", method, *self.server)
raise
except asyncio.CancelledError:
log.info("cancelled sending %s to %s:%i", method, *self.server)
self.synchronous_close()
raise
finally:
self.pending_amount -= 1
async def ensure_session(self):
# Handles reconnecting and maintaining a session alive
@@ -56,13 +94,13 @@ class ClientSession(BaseClientSession):
await self.create_connection(self.timeout)
await self.ensure_server_version()
self._on_connect_cb()
if (time() - self.last_send) > self.max_seconds_idle or self.response_time is None:
await self.send_request('server.banner')
if (perf_counter() - self.last_send) > self.max_seconds_idle or self.response_time is None:
await self.ensure_server_version()
retry_delay = default_delay
except (asyncio.TimeoutError, OSError):
await self.close()
retry_delay = min(60, retry_delay * 2)
log.warning("Wallet server timeout (retry in %s seconds): %s:%d", retry_delay, *self.server)
log.debug("Wallet server timeout (retry in %s seconds): %s:%d", retry_delay, *self.server)
try:
await asyncio.wait_for(self.trigger_urgent_reconnect.wait(), timeout=retry_delay)
except asyncio.TimeoutError:
@@ -70,12 +108,14 @@ class ClientSession(BaseClientSession):
finally:
self.trigger_urgent_reconnect.clear()
def ensure_server_version(self, required='1.2'):
return self.send_request('server.version', [__version__, required])
async def ensure_server_version(self, required='1.2', timeout=3):
return await self.send_request('server.version', [__version__, required], timeout)
async def create_connection(self, timeout=6):
connector = Connector(lambda: self, *self.server)
start = perf_counter()
await asyncio.wait_for(connector.create_connection(), timeout=timeout)
self.connection_latency = perf_counter() - start
async def handle_request(self, request):
controller = self.network.subscription_controllers[request.method]
@@ -85,13 +125,15 @@ class ClientSession(BaseClientSession):
log.debug("Connection lost: %s:%d", *self.server)
super().connection_lost(exc)
self.response_time = None
self.connection_latency = None
self._response_samples = 0
self.pending_amount = 0
self._on_disconnect_controller.add(True)
class BaseNetwork:
def __init__(self, ledger):
self.switch_event = asyncio.Event()
self.config = ledger.config
self.session_pool = SessionPool(network=self, timeout=self.config.get('connect_timeout', 6))
self.client: Optional[ClientSession] = None
@@ -112,54 +154,72 @@ class BaseNetwork:
'blockchain.address.subscribe': self._on_status_controller,
}
async def switch_to_fastest(self):
try:
client = await asyncio.wait_for(self.session_pool.wait_for_fastest_session(), 30)
except asyncio.TimeoutError:
if self.client:
await self.client.close()
self.client = None
for session in self.session_pool.sessions:
session.synchronous_close()
log.warning("not connected to any wallet servers")
return
if not self.client or client.server_address_and_port != self.client.server_address_and_port:
current_client = self.client
self.client = client
log.info("Switching to SPV wallet server: %s:%d", *self.client.server)
try:
self._update_remote_height((await self.subscribe_headers(),))
log.info("Subscribed to headers: %s:%d", *self.client.server)
if current_client:
await current_client.close()
log.info("Closed connection to %s:%i", *current_client.server)
except asyncio.TimeoutError:
if self.client:
await self.client.close()
self.client = current_client
return
self._on_connected_controller.add(True)
await asyncio.sleep(30)
async def start(self):
self.running = True
self.session_pool.start(self.config['default_servers'])
self.on_header.listen(self._update_remote_height)
while self.running:
try:
self.client = await self.session_pool.wait_for_fastest_session()
self._update_remote_height((await self.subscribe_headers(),))
log.info("Switching to SPV wallet server: %s:%d", *self.client.server)
self._on_connected_controller.add(True)
self.client.on_disconnected.listen(lambda _: self.switch_event.set())
await self.switch_event.wait()
self.switch_event.clear()
except asyncio.CancelledError:
await self.stop()
raise
except asyncio.TimeoutError:
pass
await self.switch_to_fastest()
async def stop(self):
self.running = False
if self.session_pool:
self.session_pool.stop()
if self.is_connected:
disconnected = self.client.on_disconnected.first
await self.client.close()
await disconnected
self.session_pool.stop()
@property
def is_connected(self):
return self.client and not self.client.is_closing()
def rpc(self, list_or_method, args):
fastest = self.session_pool.fastest_session
if fastest is not None and self.client != fastest:
self.switch_event.set()
if self.is_connected:
return self.client.send_request(list_or_method, args)
def rpc(self, list_or_method, args, session=None):
session = session or self.session_pool.fastest_session
if session:
return session.send_request(list_or_method, args)
else:
self.session_pool.trigger_nodelay_connect()
raise ConnectionError("Attempting to send rpc request when connection is not available.")
async def retriable_call(self, function, *args, **kwargs):
while self.running:
try:
return await function(*args, **kwargs)
except asyncio.TimeoutError:
log.warning("Wallet server call timed out, retrying.")
except ConnectionError:
if not self.is_connected and self.running:
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
await self.on_connected.first
def _update_remote_height(self, header_args):
self.remote_height = header_args[0]["height"]
def broadcast(self, raw_transaction):
return self.rpc('blockchain.transaction.broadcast', [raw_transaction])
def get_history(self, address):
return self.rpc('blockchain.address.get_history', [address])
@@ -175,11 +235,15 @@ class BaseNetwork:
def get_headers(self, height, count=10000):
return self.rpc('blockchain.block.headers', [height, count])
# --- Subscribes and broadcasts are always aimed towards the master client directly
def broadcast(self, raw_transaction):
return self.rpc('blockchain.transaction.broadcast', [raw_transaction], session=self.client)
def subscribe_headers(self):
return self.rpc('blockchain.headers.subscribe', [True])
return self.rpc('blockchain.headers.subscribe', [True], session=self.client)
def subscribe_address(self, address):
return self.rpc('blockchain.address.subscribe', [address])
return self.rpc('blockchain.address.subscribe', [address], session=self.client)
class SessionPool:
@@ -203,30 +267,61 @@ class SessionPool:
if not self.available_sessions:
return None
return min(
[(session.response_time, session) for session in self.available_sessions], key=itemgetter(0)
[((session.response_time + session.connection_latency) * (session.pending_amount + 1), session)
for session in self.available_sessions],
key=itemgetter(0)
)[1]
def _get_session_connect_callback(self, session: ClientSession):
loop = asyncio.get_event_loop()
def callback():
duplicate_connections = [
s for s in self.sessions
if s is not session and s.server_address_and_port == session.server_address_and_port
]
already_connected = None if not duplicate_connections else duplicate_connections[0]
if already_connected:
self.sessions.pop(session).cancel()
session.synchronous_close()
log.debug("wallet server %s resolves to the same server as %s, rechecking in an hour",
session.server[0], already_connected.server[0])
loop.call_later(3600, self._connect_session, session.server)
return
self.new_connection_event.set()
log.info("connected to %s:%i", *session.server)
return callback
def _connect_session(self, server: Tuple[str, int]):
session = None
for s in self.sessions:
if s.server == server:
session = s
break
if not session:
session = ClientSession(
network=self.network, server=server
)
session._on_connect_cb = self._get_session_connect_callback(session)
task = self.sessions.get(session, None)
if not task or task.done():
task = asyncio.create_task(session.ensure_session())
task.add_done_callback(lambda _: self.ensure_connections())
self.sessions[session] = task
def start(self, default_servers):
callback = self.new_connection_event.set
self.sessions = {
ClientSession(
network=self.network, server=server, on_connect_callback=callback
): None for server in default_servers
}
self.ensure_connections()
for server in default_servers:
self._connect_session(server)
def stop(self):
for session, task in self.sessions.items():
for task in self.sessions.values():
task.cancel()
session.abort()
self.sessions.clear()
def ensure_connections(self):
for session, task in list(self.sessions.items()):
if not task or task.done():
task = asyncio.create_task(session.ensure_session())
task.add_done_callback(lambda _: self.ensure_connections())
self.sessions[session] = task
for session in self.sessions:
self._connect_session(session.server)
def trigger_nodelay_connect(self):
# used when other parts of the system sees we might have internet back
+2 -4
View File
@@ -746,10 +746,8 @@ class JSONRPCConnection(object):
self._protocol = item
return self.receive_message(message)
def time_out_pending_requests(self):
"""Times out all pending requests."""
# this used to be CancelledError, but thats confusing as in are we closing the whole sdk or failing?
exception = asyncio.TimeoutError()
def raise_pending_requests(self, exception):
exception = exception or asyncio.TimeoutError()
for request, event in self._requests.values():
event.result = exception
event.set()
+15 -8
View File
@@ -103,7 +103,7 @@ class SessionBase(asyncio.Protocol):
# Force-close a connection if a send doesn't succeed in this time
self.max_send_delay = 60
# Statistics. The RPC object also keeps its own statistics.
self.start_time = time.time()
self.start_time = time.perf_counter()
self.errors = 0
self.send_count = 0
self.send_size = 0
@@ -123,7 +123,7 @@ class SessionBase(asyncio.Protocol):
# A non-positive value means not to limit concurrency
if self.bw_limit <= 0:
return
now = time.time()
now = time.perf_counter()
# Reduce the recorded usage in proportion to the elapsed time
refund = (now - self.bw_time) * (self.bw_limit / 3600)
self.bw_charge = max(0, self.bw_charge - int(refund))
@@ -146,7 +146,7 @@ class SessionBase(asyncio.Protocol):
await asyncio.wait_for(self._can_send.wait(), secs)
except asyncio.TimeoutError:
self.abort()
raise asyncio.CancelledError(f'task timed out after {secs}s')
raise asyncio.TimeoutError(f'task timed out after {secs}s')
async def _send_message(self, message):
if not self._can_send.is_set():
@@ -156,7 +156,7 @@ class SessionBase(asyncio.Protocol):
self.send_size += len(framed_message)
self._using_bandwidth(len(framed_message))
self.send_count += 1
self.last_send = time.time()
self.last_send = time.perf_counter()
if self.verbosity >= 4:
self.logger.debug(f'Sending framed message {framed_message}')
self.transport.write(framed_message)
@@ -215,7 +215,8 @@ class SessionBase(asyncio.Protocol):
self._address = None
self.transport = None
self._task_group.cancel()
self._pm_task.cancel()
if self._pm_task:
self._pm_task.cancel()
# Release waiting tasks
self._can_send.set()
@@ -253,6 +254,7 @@ class SessionBase(asyncio.Protocol):
if self.transport:
self.transport.abort()
# TODO: replace with synchronous_close
async def close(self, *, force_after=30):
"""Close the connection and return when closed."""
self._close()
@@ -262,6 +264,11 @@ class SessionBase(asyncio.Protocol):
self.abort()
await self._pm_task
def synchronous_close(self):
self._close()
if self._pm_task and not self._pm_task.done():
self._pm_task.cancel()
class MessageSession(SessionBase):
"""Session class for protocols where messages are not tied to responses,
@@ -296,7 +303,7 @@ class MessageSession(SessionBase):
)
self._bump_errors()
else:
self.last_recv = time.time()
self.last_recv = time.perf_counter()
self.recv_count += 1
if self.recv_count % 10 == 0:
await self._update_concurrency()
@@ -416,7 +423,7 @@ class RPCSession(SessionBase):
self.logger.warning(f'{e!r}')
continue
self.last_recv = time.time()
self.last_recv = time.perf_counter()
self.recv_count += 1
if self.recv_count % 10 == 0:
await self._update_concurrency()
@@ -456,7 +463,7 @@ class RPCSession(SessionBase):
def connection_lost(self, exc):
# Cancel pending requests and message processing
self.connection.time_out_pending_requests()
self.connection.raise_pending_requests(exc)
super().connection_lost(exc)
# External API
+1 -1
View File
@@ -258,7 +258,7 @@ class SessionManager:
session_timeout = self.env.session_timeout
while True:
await sleep(session_timeout // 10)
stale_cutoff = time.time() - session_timeout
stale_cutoff = time.perf_counter() - session_timeout
stale_sessions = [session for session in self.sessions
if session.last_recv < stale_cutoff]
if stale_sessions: