Compare commits

...
32 Commits
Author SHA1 Message Date
Victor Shyba 593422f70c niko 2019-08-23 21:50:11 -03:00
Alex Grin de12e25450 v0.38.6 2019-07-29 15:11:18 -04:00
Oleg Silkin 2545ebbece Adds unittests & fixes comment_client functions 2019-07-29 14:55:42 -04:00
Oleg Silkin 14ec6ce43b Adds unit-testing for delete comment signatures 2019-07-29 14:55:42 -04:00
Oleg Silkin 0cac1e65aa Imports comment_client and uses dot notation for rest of modules 2019-07-29 14:55:42 -04:00
Oleg Silkin 1e4d57ab83 Simplifies signing function into one to make API symmetric 2019-07-29 14:55:42 -04:00
Oleg Silkin 49b42b7e03 Simplifies 2019-07-29 14:55:42 -04:00
Oleg Silkin 89c0b5dbdf adds comment abandon function 2019-07-29 14:55:42 -04:00
Lex Berezhny 8138675808 added index for claim_type searches 2019-07-29 14:55:07 -04:00
Jack Robison 726129836a Merge pull request #2323 from lbryio/refactor_ttfb
refactor and fix ttfb for new api
2019-07-28 22:28:02 -04:00
Jack Robison 356b3c85e5 use app default order_by 2019-07-28 21:42:04 -04:00
Jack Robison 1a0680ead9 update claim_search_performance.py, add new test cases shown by time_to_first_byte.py 2019-07-28 21:22:07 -04:00
Jack Robison c876d891fa use perf_counter 2019-07-28 21:18:29 -04:00
Jack Robison a08232f164 fix claim search kwargs in time to first byte script 2019-07-28 21:18:13 -04:00
Victor Shyba 0f8cf46626 refactor and fix ttfb for new api 2019-07-28 17:33:37 -04:00
Alex Grin 95017b5fc8 v0.38.5 2019-07-25 13:22:54 -04:00
Jack Robison 1374d337d8 Merge pull request #2348 from lbryio/probemon
Probe if peers can be connected to during blob downloads to more quickly skip bad peers
2019-07-25 13:21:42 -04:00
Jack Robison 35e7cc685f fix logging error 2019-07-24 22:12:51 -04:00
Victor Shyba 2ac4883bda cleanup and fix prober 2019-07-24 22:32:20 -03:00
Victor Shyba 5a7212aebb gotta probe 'em all 2019-07-24 20:26:38 -03:00
Lex Berezhny 3f39cc2955 v0.38.4 2019-07-24 18:02:03 -04:00
Lex Berezhny 8a5bfe884a save channel keys into wallet before broadcasting 2019-07-24 18:00:06 -04:00
Lex Berezhny 046dd53913 add index to filter by fee_amount and order by release_time 2019-07-24 14:27:59 -04:00
Alex Grintsvayg d2f174635c add script to monitor wallet servers for slow queries 2019-07-24 13:36:22 -04:00
Lex Berezhny 01a760d8f0 v0.38.3 2019-07-23 17:44:38 -04:00
Victor Shyba 9215d2c9f9 log that checkpoint succeeded 2019-07-23 17:44:13 -04:00
Victor Shyba 0bb09ae91e use checkpoint to speed up 2019-07-23 17:44:13 -04:00
Victor Shyba afeff61268 fix repair and tests 2019-07-23 17:44:13 -04:00
Victor Shyba 6f5a595975 check from final size instead of partials 2019-07-23 17:44:13 -04:00
Victor Shyba 8a69ec062e download headers to disk and repair instead of connecting each 2019-07-23 17:44:13 -04:00
Victor Shyba 45fb673e49 dont fail when s3 is out of reach 2019-07-23 17:44:13 -04:00
Lex Berezhny 93b5f1809e revert to v0.38.2 so we can release v0.38.3 again 2019-07-23 17:43:04 -04:00
22 changed files with 789 additions and 421 deletions

No files matched your search

+1 -1
View File
@@ -1,4 +1,4 @@
__name__ = "lbry"
__version__ = "0.38.3"
__version__ = "0.38.6"
version = tuple(__version__.split('.'))
+5 -3
View File
@@ -187,7 +187,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
return await self._download_blob()
except OSError as e:
# i'm not sure how to fix this race condition - jack
log.warning("race happened downloading %s from %s:%i", blob_hash, self.peer_address, self.peer_port)
log.warning("race happened downloading %s from %s:%s", blob_hash, self.peer_address, self.peer_port)
# return self._blob_bytes_received, self.transport
raise
except asyncio.TimeoutError:
@@ -220,7 +220,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
@cache_concurrent
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'AbstractBlob', address: str, tcp_port: int,
async def request_blob(loop: asyncio.BaseEventLoop, blob: typing.Optional['AbstractBlob'], address: str, tcp_port: int,
peer_connect_timeout: float, blob_download_timeout: float,
connected_transport: asyncio.Transport = None, connection_id: int = 0,
connection_manager: typing.Optional['ConnectionManager'] = None)\
@@ -242,7 +242,9 @@ async def request_blob(loop: asyncio.BaseEventLoop, blob: 'AbstractBlob', addres
if not connected_transport:
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
peer_connect_timeout, loop=loop)
if blob.get_is_verified() or not blob.is_writeable():
connected_transport = protocol.transport
if blob is None or blob.get_is_verified() or not blob.is_writeable():
# blob is None happens when we are just opening a connection
# file exists but not verified means someone is writing right now, give it time, come back later
return 0, connected_transport
return await protocol.download_blob(blob)
+13 -9
View File
@@ -30,17 +30,19 @@ class BlobDownloader:
self.is_running = asyncio.Event(loop=self.loop)
def should_race_continue(self, blob: 'AbstractBlob'):
if len(self.active_connections) >= self.config.max_connections_per_download:
max_probes = self.config.max_connections_per_download * (1 if self.connections else 10)
if len(self.active_connections) >= max_probes:
return False
return not (blob.get_is_verified() or not blob.is_writeable())
async def request_blob_from_peer(self, blob: 'AbstractBlob', peer: 'KademliaPeer', connection_id: int = 0):
async def request_blob_from_peer(self, blob: 'AbstractBlob', peer: 'KademliaPeer', connection_id: int = 0,
just_probe: bool = False):
if blob.get_is_verified():
return
transport = self.connections.get(peer)
start = self.loop.time()
bytes_received, transport = await request_blob(
self.loop, blob, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
self.loop, blob if not just_probe else None, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
self.config.blob_download_timeout, connected_transport=transport, connection_id=connection_id,
connection_manager=self.blob_manager.connection_manager
@@ -91,16 +93,18 @@ class BlobDownloader:
if batch:
self.peer_queue.put_nowait(list(batch))
log.debug(
"running, %d peers, %d ignored, %d active",
len(batch), len(self.ignored), len(self.active_connections)
"running, %d peers, %d ignored, %d active, %s connections",
len(batch), len(self.ignored), len(self.active_connections), len(self.connections)
)
for peer in sorted(batch, key=lambda peer: self.scores.get(peer, 0), reverse=True):
if peer in self.ignored or peer in self.active_connections:
continue
if not self.should_race_continue(blob):
break
if peer not in self.active_connections and peer not in self.ignored:
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id))
self.active_connections[peer] = t
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
just_probe = len(self.connections) == 0
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id, just_probe))
self.active_connections[peer] = t
await self.new_peer_or_finished()
self.cleanup_active()
log.debug("downloaded %s", blob_hash[:8])
+50 -40
View File
@@ -1,3 +1,4 @@
import hashlib
import os
import asyncio
import logging
@@ -96,12 +97,10 @@ class DatabaseComponent(Component):
self.storage = None
HEADERS_URL = "https://headers.lbry.io/blockchain_headers_latest"
HEADER_SIZE = 112
class HeadersComponent(Component):
component_name = HEADERS_COMPONENT
HEADERS_URL = "https://headers.lbry.io/blockchain_headers_latest"
CHECKPOINT = ('100b33ca3d0b86a48f0d6d6f30458a130ecb89d5affefe4afccb134d5a40f4c2', 600_000)
def __init__(self, component_manager):
super().__init__(component_manager)
@@ -135,31 +134,26 @@ class HeadersComponent(Component):
} if progress is not None and progress < 100 else {}
async def fetch_headers_from_s3(self):
local_header_size = self.headers.bytes_size
local_header_size = self.local_header_file_size()
resume_header = {"Range": f"bytes={local_header_size}-"}
async with utils.aiohttp_request('get', HEADERS_URL, headers=resume_header) as response:
if response.status == 406 or response.content_length < HEADER_SIZE: # our file is bigger
async with utils.aiohttp_request('get', self.HEADERS_URL, headers=resume_header) as response:
if response.status == 406 or response.content_length < self.headers.header_size: # our file is bigger
log.warning("s3 is more out of date than we are")
return
if response.content_length % HEADER_SIZE != 0:
final_size_after_download = response.content_length + local_header_size
if final_size_after_download % self.headers.header_size != 0:
log.warning("s3 appears to have corrupted header")
return
final_size_after_download = response.content_length + local_header_size
write_mode = "wb"
if local_header_size > 0:
log.info("Resuming download of %i bytes from s3", response.content_length)
buffer, header_size = b'', self.headers.header_size
async for chunk in response.content.iter_any():
chunk = buffer + chunk
remaining = len(chunk) % header_size
chunk, buffer = chunk[:-remaining], bytes(chunk[-remaining:])
if not chunk:
continue
if not await self.headers.connect(len(self.headers), chunk):
log.warning("Error connecting downloaded headers from at %s.", self.headers.height)
return
self._headers_progress_percent = self._round_progress(
self.headers.bytes_size, final_size_after_download
)
write_mode = "a+b"
with open(self.headers_file, write_mode) as fd:
while not response.content.at_eof():
local_header_size += fd.write(await response.content.readany())
self._headers_progress_percent = self._round_progress(
local_header_size, final_size_after_download
)
def local_header_file_size(self) -> int:
if os.path.isfile(self.headers_file):
@@ -167,13 +161,11 @@ class HeadersComponent(Component):
return 0
async def get_downloadable_header_height(self) -> typing.Optional[int]:
try:
async with utils.aiohttp_request('HEAD', HEADERS_URL) as response:
if response.status != 200:
log.warning("Header download error, unexpected response code: %s", response.status)
return response.content_length // HEADER_SIZE
except OSError:
log.exception("Failed to download headers using https.")
async with utils.aiohttp_request('HEAD', self.HEADERS_URL) as response:
if response.status != 200:
log.warning("Header download error, unexpected response code: %s", response.status)
return -1
return response.content_length // self.headers.header_size
async def should_download_headers_from_s3(self) -> bool:
if self.conf.blockchain_name != "lbrycrd_main":
@@ -182,7 +174,7 @@ class HeadersComponent(Component):
if not s3_headers_depth:
return False
local_height = self.headers.height
local_height = self.local_header_file_size() // self.headers.header_size
remote_height = await self.get_downloadable_header_height()
if remote_height is not None:
log.info("remote height: %i, local height: %i", remote_height, local_height)
@@ -190,24 +182,42 @@ class HeadersComponent(Component):
return True
return False
def verify_checkpoint(self):
expected_hash, at_height = self.CHECKPOINT
if self.local_header_file_size() // self.headers.header_size < at_height:
return False
hash = hashlib.sha256()
chunk_size = self.headers.header_size * 1000
with open(self.headers_file, 'rb') as header_file:
data = header_file.read(chunk_size)
while data and header_file.tell() <= at_height * self.headers.header_size:
hash.update(data)
data = header_file.read(chunk_size)
return hash.hexdigest() == expected_hash
async def start(self):
if not os.path.exists(self.headers_dir):
os.mkdir(self.headers_dir)
if os.path.exists(self.old_file):
log.warning("Moving old headers from %s to %s.", self.old_file, self.headers_file)
os.rename(self.old_file, self.headers_file)
await self.headers.open()
await self.headers.repair()
if await self.should_download_headers_from_s3():
try:
try:
if await self.should_download_headers_from_s3():
self.is_downloading_headers = True
await self.fetch_headers_from_s3()
except Exception as err:
log.error("failed to fetch headers from s3: %s", err)
finally:
self.is_downloading_headers = False
await self.headers.close()
except Exception as err:
log.error("failed to fetch headers from s3: %s", err)
finally:
self.is_downloading_headers = False
# fixme: workaround, this should happen before download but happens after because headers.connect fail
if not self.verify_checkpoint():
log.info("Checkpoint failed, verifying headers using slower method.")
await self.headers.open()
await self.headers.repair()
await self.headers.close()
else:
log.info("Header checkpoint verified.")
async def stop(self):
pass
+39 -8
View File
@@ -33,7 +33,7 @@ from lbry.extras.daemon.Components import EXCHANGE_RATE_MANAGER_COMPONENT, UPNP_
from lbry.extras.daemon.ComponentManager import RequiredCondition
from lbry.extras.daemon.ComponentManager import ComponentManager
from lbry.extras.daemon.json_response_encoder import JSONResponseEncoder
from lbry.extras.daemon.comment_client import jsonrpc_post, sign_comment, is_comment_signed_by_channel
from lbry.extras.daemon import comment_client
from lbry.extras.daemon.undecorated import undecorated
from lbry.wallet.transaction import Transaction, Output, Input
from lbry.wallet.account import Account as LBCAccount
@@ -1946,9 +1946,9 @@ class Daemon(metaclass=JSONRPCServerType):
if not preview:
await tx.sign([account])
await self.broadcast_or_release(account, tx, blocking)
account.add_channel_private_key(txo.private_key)
self.default_wallet.save()
await self.broadcast_or_release(account, tx, blocking)
await self.storage.save_claims([self._old_get_temp_claim_info(
tx, txo, claim_address, claim, name, dewies_to_lbc(amount)
)])
@@ -2085,9 +2085,9 @@ class Daemon(metaclass=JSONRPCServerType):
if not preview:
await tx.sign([account])
await self.broadcast_or_release(account, tx, blocking)
account.add_channel_private_key(new_txo.private_key)
self.default_wallet.save()
await self.broadcast_or_release(account, tx, blocking)
await self.storage.save_claims([self._old_get_temp_claim_info(
tx, new_txo, claim_address, new_txo.claim, new_txo.claim_name, dewies_to_lbc(amount)
)])
@@ -3416,7 +3416,7 @@ class Daemon(metaclass=JSONRPCServerType):
]
}
"""
result = await jsonrpc_post(
result = await comment_client.jsonrpc_post(
self.conf.comment_server,
"get_claim_comments",
claim_id=claim_id,
@@ -3432,7 +3432,7 @@ class Daemon(metaclass=JSONRPCServerType):
continue
resolve_response = await self.resolve([channel_url])
if isinstance(resolve_response[channel_url], Output):
comment['is_channel_signature_valid'] = is_comment_signed_by_channel(
comment['is_channel_signature_valid'] = comment_client.is_comment_signed_by_channel(
comment, resolve_response[channel_url]
)
else:
@@ -3490,12 +3490,43 @@ class Daemon(metaclass=JSONRPCServerType):
'channel_id': channel.claim_id,
'channel_name': channel.claim_name,
})
sign_comment(comment_body, channel)
response = await jsonrpc_post(self.conf.comment_server, 'create_comment', **comment_body)
comment_client.sign_comment(comment_body, channel)
response = await comment_client.jsonrpc_post(self.conf.comment_server, 'create_comment', comment_body)
if 'signature' in response:
response['is_claim_signature_valid'] = is_comment_signed_by_channel(response, channel)
response['is_claim_signature_valid'] = comment_client.is_comment_signed_by_channel(response, channel)
return response
async def jsonrpc_comment_abandon(self, comment_id):
""""
Delete a comment published under your channel identity
Usage:
comment_delete (<comment_id> | --comment_id=<comment_id>)
Options:
--comment_id=<comment_id> : (str) The ID of the comment to be deleted.
Returns:
(dict) Object with the `comment_id` passed in as the key, and a flag indicating if it was deleted
{
<comment_id> (str): {
"deleted": (bool)
}
}
"""
abandon_comment_body = {'comment_id': comment_id}
channel = await comment_client.jsonrpc_post(
self.conf.comment_server, 'get_channel_from_comment_id', comment_id=comment_id
)
if 'error' in channel:
return {comment_id: {'deleted': False}}
channel = await self.get_channel_or_none(None, **channel)
abandon_comment_body.update({
'channel_id': channel.claim_id,
'channel_name': channel.claim_name,
})
comment_client.sign_comment(abandon_comment_body, channel, abandon=True)
return await comment_client.jsonrpc_post(self.conf.comment_server, 'delete_comment', abandon_comment_body)
async def broadcast_or_release(self, account, tx, blocking=False):
try:
+20 -10
View File
@@ -18,13 +18,18 @@ def get_encoded_signature(signature):
return ecdsa.util.sigencode_der(r, s, len(signature) * 4)
def is_comment_signed_by_channel(comment: dict, channel: Output):
def cid2hash(claim_id: str) -> bytes:
return binascii.unhexlify(claim_id.encode())[::-1]
def is_comment_signed_by_channel(comment: dict, channel: Output, abandon=False):
if type(channel) is Output:
try:
signing_field = comment['comment_id'] if abandon else comment['comment']
pieces = [
comment['signing_ts'].encode(),
channel.claim_hash,
comment['comment'].encode()
cid2hash(comment['channel_id']),
signing_field.encode()
]
return Output.is_signature_valid(
get_encoded_signature(comment['signature']),
@@ -36,16 +41,21 @@ def is_comment_signed_by_channel(comment: dict, channel: Output):
return False
def sign_comment(comment: dict, channel: Output):
timestamp = str(int(time.time())).encode()
pieces = [timestamp, channel.claim_hash, comment['comment'].encode()]
def sign_comment(comment: dict, channel: Output, abandon=False):
timestamp = str(int(time.time()))
signing_field = comment['comment_id'] if abandon else comment['comment']
pieces = [timestamp.encode(), channel.claim_hash, signing_field.encode()]
digest = sha256(b''.join(pieces))
signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
comment['signature'] = binascii.hexlify(signature).decode()
comment['signing_ts'] = timestamp.decode()
comment.update({
'signature': binascii.hexlify(signature).decode(),
'signing_ts': timestamp
})
async def jsonrpc_post(url: str, method: str, **params) -> any:
async def jsonrpc_post(url: str, method: str, params: dict = None, **kwargs) -> any:
params = params or {}
params.update(kwargs)
json_body = {'jsonrpc': '2.0', 'id': None, 'method': method, 'params': params}
headers = {'Content-Type': 'application/json'}
async with utils.aiohttp_request('POST', url, json=json_body, headers=headers) as response:
@@ -53,5 +63,5 @@ async def jsonrpc_post(url: str, method: str, **params) -> any:
result = await response.json()
return result['result'] if 'result' in result else result
except Exception as cte:
log.exception('Unable to decode respose from server: %s', cte)
log.exception('Unable to decode response from server: %s', cte)
return await response.text()
+3 -3
View File
@@ -121,6 +121,9 @@ class SQLDB:
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash, release_time);
create unique index if not exists claim_release_time_idx on claim (release_time, claim_hash);
create unique index if not exists claim_trending_global_mixed_idx on claim (trending_global, trending_mixed, claim_hash);
create unique index if not exists filter_fee_amount_order_release_time_idx on claim (fee_amount, release_time, claim_hash);
create unique index if not exists claim_type_trending_idx on claim (claim_type, trending_global, trending_mixed, claim_hash);
-- TODO: verify that all indexes below are used
create index if not exists claim_height_normalized_idx on claim (height, normalized asc);
@@ -131,11 +134,8 @@ class SQLDB:
create index if not exists claim_timestamp_idx on claim (timestamp);
create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
create index if not exists claim_claim_type_idx on claim (claim_type);
create index if not exists claim_stream_type_idx on claim (stream_type);
create index if not exists claim_media_type_idx on claim (media_type);
create index if not exists claim_fee_amount_idx on claim (fee_amount);
create index if not exists claim_fee_currency_idx on claim (fee_currency);
create index if not exists claim_signature_valid_idx on claim (signature_valid);
"""
+77 -46
View File
@@ -1,5 +1,6 @@
import os
import time
import textwrap
import argparse
import asyncio
import logging
@@ -9,7 +10,7 @@ from lbry.wallet.ledger import MainNetLedger
log = logging.getLogger(__name__)
log.addHandler(logging.StreamHandler())
log.setLevel(logging.INFO)
log.setLevel(logging.CRITICAL)
DEFAULT_ANY_TAGS = [
'blockchain',
@@ -68,43 +69,46 @@ def get_args(limit=20):
args = []
any_tags_combinations = [DEFAULT_ANY_TAGS, COMMON_AND_RARE, RARE_ANY_TAGS, COMMON_AND_RARE2, CITY_FIX, []]
not_tags_combinations = [MATURE_TAGS, []]
for no_totals in [True]:
for offset in [0, 100]:
for any_tags in any_tags_combinations:
for not_tags in not_tags_combinations:
for order_by in ORDER_BY:
kw = {
'order_by': order_by,
'offset': offset,
'limit': limit,
'no_totals': no_totals
}
if not_tags:
kw['not_tags'] = not_tags
if any_tags:
kw['any_tags'] = any_tags
args.append(kw)
print(len(args), "argument combinations")
for no_fee in [False, True]:
for claim_type in [None, 'stream', 'channel']:
for no_totals in [True]:
for offset in [0, 100]:
for any_tags in any_tags_combinations:
for not_tags in not_tags_combinations:
for order_by in ORDER_BY:
kw = {
'order_by': order_by,
'offset': offset,
'limit': limit,
'no_totals': no_totals
}
if not_tags:
kw['not_tags'] = not_tags
if any_tags:
kw['any_tags'] = any_tags
if claim_type:
kw['claim_type'] = claim_type
if no_fee:
kw['fee_amount'] = 0
args.append(kw)
print(f"-- Trying {len(args)} argument combinations")
return args
def _search(kwargs):
start = time.time()
start = time.perf_counter()
error = None
try:
search_to_bytes(kwargs)
t = time.time() - start
return t, kwargs
except Exception as err:
return -1, f"failed: error={str(type(err))}({str(err)})"
error = str(err)
return time.perf_counter() - start, kwargs, error
async def search(executor, kwargs):
try:
return await asyncio.get_running_loop().run_in_executor(
executor, _search, kwargs
)
except Exception as err:
return f"failed (err={str(type(err))}({err}))- {kwargs}"
return await asyncio.get_running_loop().run_in_executor(
executor, _search, kwargs
)
async def main(db_path, max_query_time):
@@ -115,23 +119,50 @@ async def main(db_path, max_query_time):
tasks = [search(query_executor, constraints) for constraints in get_args()]
try:
results = await asyncio.gather(*tasks)
for ts, constraints in results:
if ts >= max_query_time:
sql = interpolate(*_get_claims("""
claimtrie.claim_hash as is_controlling,
claimtrie.last_take_over_height,
claim.claim_hash, claim.txo_hash,
claim.claims_in_channel,
claim.height, claim.creation_height,
claim.activation_height, claim.expiration_height,
claim.effective_amount, claim.support_amount,
claim.trending_group, claim.trending_mixed,
claim.trending_local, claim.trending_global,
claim.short_url, claim.canonical_url,
claim.channel_hash, channel.txo_hash AS channel_txo_hash,
channel.height AS channel_height, claim.signature_valid
""", **constraints))
print(f"Query took {int(ts * 1000)}ms\n{sql}")
query_times = [
{
'sql': interpolate(*_get_claims("""
claimtrie.claim_hash as is_controlling,
claimtrie.last_take_over_height,
claim.claim_hash, claim.txo_hash,
claim.claims_in_channel,
claim.height, claim.creation_height,
claim.activation_height, claim.expiration_height,
claim.effective_amount, claim.support_amount,
claim.trending_group, claim.trending_mixed,
claim.trending_local, claim.trending_global,
claim.short_url, claim.canonical_url,
claim.channel_hash, channel.txo_hash AS channel_txo_hash,
channel.height AS channel_height, claim.signature_valid
""", **constraints)),
'duration': ts,
'error': error
}
for ts, constraints, error in results
]
errored = [query_info for query_info in query_times if query_info['error']]
errors = {str(query_info['error']): [] for query_info in errored}
for error in errored:
errors[str(error['error'])].append(error['sql'])
slow = [
query_info for query_info in query_times
if not query_info['error'] and query_info['duration'] > (max_query_time / 2.0)
]
fast = [
query_info for query_info in query_times
if not query_info['error'] and query_info['duration'] <= (max_query_time / 2.0)
]
print(f"-- {len(fast)} queries were fast")
slow.sort(key=lambda query_info: query_info['duration'], reverse=True)
print(f"-- Failing queries:")
for error in errors:
print(f"-- Failure: \"{error}\"")
for failing_query in errors[error]:
print(f"{textwrap.dedent(failing_query)};\n")
print()
print(f"-- Slow queries:")
for slow_query in slow:
print(f"-- Query took {slow_query['duration']}\n{textwrap.dedent(slow_query['sql'])};\n")
finally:
query_executor.shutdown()
@@ -139,7 +170,7 @@ async def main(db_path, max_query_time):
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--db_path', dest='db_path', default=os.path.expanduser('~/claims.db'), type=str)
parser.add_argument('--max_time', dest='max_time', default=0.0, type=float)
parser.add_argument('--max_time', dest='max_time', default=0.25, type=float)
args = parser.parse_args()
db_path = args.db_path
max_query_time = args.max_time
+49
View File
@@ -0,0 +1,49 @@
import os, asyncio, aiohttp, json, slack, sqlparse
async def listen(slack_client, url):
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(3)) as session:
print(f"connecting to {url}")
try:
ws = await session.ws_connect(url)
except (aiohttp.ClientConnectorError, asyncio.TimeoutError):
print(f"failed to connect to {url}")
return
print(f"connected to {url}")
async for msg in ws:
r = json.loads(msg.data)
try:
queries = r["api"]["search"]["interrupted_queries"]
except KeyError:
continue
for q in queries:
# clean = re.sub(r"\s+", " ", q)
clean = sqlparse.format(q, reindent=True, keyword_case='upper')
print(f'{url}: {clean}')
response = await slack_client.chat_postMessage(
username=url,
icon_emoji=":hourglass_flowing_sand:",
channel='#clubhouse-de-obscure',
text="*Query timed out:* " + clean
)
if not response["ok"]:
print("SLACK ERROR:\n", response)
print()
async def main():
try:
slack_client = slack.WebClient(token=os.environ['SLACK_TOKEN'], run_async=True)
except KeyError:
print("Error: SLACK_TOKEN env var required")
return
num_servers = 5
tasks = []
for i in range(1, num_servers+1):
tasks.append(asyncio.create_task(listen(slack_client, f'http://spv{i}.lbry.com:50005')))
await asyncio.gather(*tasks)
asyncio.run(main())
+86 -103
View File
@@ -1,43 +1,19 @@
import os
import sys
import json
import argparse
import asyncio
import aiohttp
import time
import aiohttp
from aiohttp import ClientConnectorError
from lbry import __version__
from lbry.blob.blob_file import MAX_BLOB_SIZE
from lbry.conf import Config
from lbry.schema.uri import parse_lbry_uri
from lbry.extras.daemon.client import daemon_rpc
from lbry.extras import system_info
def extract_uris(response):
uris = list()
for key in response:
for value in response[key]:
uris.append(value)
return uris
async def get_frontpage_uris():
session = aiohttp.ClientSession()
try:
response = await session.get("https://api.lbry.com/file/list_homepage", timeout=10.0)
if response.status != 200:
print("API returned non 200 code!!")
return
body = await response.json()
await session.close()
uris = extract_uris(body['data']['Uris'])
return uris
finally:
await session.close()
async def report_to_slack(output, webhook):
payload = {
"text": f"lbrynet {__version__} ({system_info.get_platform()['platform']}) time to first byte:\n{output}"
@@ -58,120 +34,127 @@ def variance(times):
return round(sum(((i - mean) ** 2.0 for i in times)) / (len(times) - 1), 3)
async def wait_for_done(conf, uri, timeout):
name = uri.split("#")[0]
last_complete = 0
hang_count = 0
async def wait_for_done(conf, claim_name, timeout):
blobs_completed, last_completed = 0, time.perf_counter()
while True:
files = await daemon_rpc(conf, "file_list", claim_name=name)
file = files[0]
file = (await daemon_rpc(conf, "file_list", claim_name=claim_name))[0]
if file['status'] in ['finished', 'stopped']:
return True, file['blobs_completed'], file['blobs_in_stream']
if last_complete < int(file['blobs_completed']):
hang_count = 0
last_complete = int(file['blobs_completed'])
else:
hang_count += 1
await asyncio.sleep(1.0)
if hang_count > timeout:
elif blobs_completed < int(file['blobs_completed']):
blobs_completed, last_completed = int(file['blobs_completed']), time.perf_counter()
elif (time.perf_counter() - last_completed) > timeout:
return False, file['blobs_completed'], file['blobs_in_stream']
await asyncio.sleep(1.0)
async def main(uris=None, cmd_args=None):
if not uris:
uris = await get_frontpage_uris()
async def main(cmd_args=None):
print('Time to first byte started using parameters:')
for key, value in vars(cmd_args).items():
print(f"{key}: {value}")
conf = Config()
url_to_claim = {}
try:
await daemon_rpc(conf, 'status')
for page in range(1, cmd_args.download_pages + 1):
start = time.perf_counter()
kwargs = {
'page': page,
# 'claim_type': 'stream',
'order_by': ['trending_global', 'trending_mixed'],
'no_totals': True
}
# if not cmd_args.allow_fees:
# kwargs['fee_amount'] = 0
response = await daemon_rpc(
conf, 'claim_search', **kwargs
)
if 'error' in response or not response.get('items'):
print(f'Error getting claim list page {page}:')
print(response)
return 1
else:
url_to_claim.update({
claim['permanent_url']: claim for claim in response['items']
})
print(f'Claim search page {page} took: {time.perf_counter() - start}')
except (ClientConnectorError, ConnectionError):
print("Could not connect to daemon")
return 1
print(f"Checking {len(uris)} uris from the front page")
print("**********************************************")
resolvable = []
async def __resolve(name):
resolved = await daemon_rpc(conf, 'resolve', urls=[name])
if 'error' not in resolved.get(name, {}):
if ("fee" not in resolved[name]['claim']['value']) or cmd_args.allow_fees:
resolvable.append(name)
else:
print(f"{name} has a fee, skipping it")
else:
print(f"failed to resolve {name}: {resolved[name]['error']}")
await asyncio.gather(*(__resolve(name) for name in uris))
print(f"attempting to download {len(resolvable)}/{len(uris)} frontpage streams")
print(f"Attempting to download {len(url_to_claim)} claim_search streams")
first_byte_times = []
download_speeds = []
download_successes = []
failed_to_start = []
download_failures = []
failed_to = {}
for uri in resolvable:
await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=parse_lbry_uri(uri).name)
await asyncio.gather(*(
daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=claim['name'])
for claim in url_to_claim.values() if not cmd_args.keep_files
))
for i, uri in enumerate(resolvable):
start = time.time()
try:
await daemon_rpc(conf, 'get', uri=uri, save_file=True)
first_byte = time.time()
first_byte_times.append(first_byte - start)
print(f"{i + 1}/{len(resolvable)} - {first_byte - start} {uri}")
if not cmd_args.head_blob_only:
downloaded, amount_downloaded, blobs_in_stream = await wait_for_done(
conf, uri, cmd_args.stall_download_timeout
)
if downloaded:
download_successes.append(uri)
else:
download_failures.append(uri)
mbs = round((blobs_in_stream * (MAX_BLOB_SIZE - 1)) / (time.time() - start) / 1000000, 2)
download_speeds.append(mbs)
print(f"downloaded {amount_downloaded}/{blobs_in_stream} blobs for {uri} at "
f"{mbs}mb/s")
except Exception as e:
print(f"{i + 1}/{len(uris)} - failed to start {uri}: {e}")
failed_to_start.append(uri)
for i, (url, claim) in enumerate(url_to_claim.items()):
start = time.perf_counter()
response = await daemon_rpc(conf, 'get', uri=url, save_file=not cmd_args.head_blob_only)
if 'error' in response:
print(f"{i + 1}/{len(url_to_claim)} - failed to start {url}: {response['error']}")
failed_to[url] = 'start'
if cmd_args.exit_on_error:
return
if cmd_args.delete_after_download or cmd_args.head_blob_only:
await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=parse_lbry_uri(uri).name)
continue
first_byte = time.perf_counter()
first_byte_times.append(first_byte - start)
print(f"{i + 1}/{len(url_to_claim)} - {first_byte - start} {url}")
if not cmd_args.head_blob_only:
downloaded, amount_downloaded, blobs_in_stream = await wait_for_done(
conf, claim['name'], cmd_args.stall_download_timeout
)
if downloaded:
download_successes.append(url)
else:
failed_to[url] = 'finish'
mbs = round((blobs_in_stream * (MAX_BLOB_SIZE - 1)) / (time.perf_counter() - start) / 1000000, 2)
download_speeds.append(mbs)
print(f"downloaded {amount_downloaded}/{blobs_in_stream} blobs for {url} at "
f"{mbs}mb/s")
if not cmd_args.keep_files:
await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=claim['name'])
await asyncio.sleep(0.1)
print("**********************************************")
result = f"Started {len(first_byte_times)} of {len(resolvable)} attempted front page streams\n" \
f"Worst first byte time: {round(max(first_byte_times), 2)}\n" \
f"Best first byte time: {round(min(first_byte_times), 2)}\n" \
f"95% confidence time-to-first-byte: {confidence(first_byte_times, 1.984)}s\n" \
f"99% confidence time-to-first-byte: {confidence(first_byte_times, 2.626)}s\n" \
f"Variance: {variance(first_byte_times)}\n"
if not cmd_args.head_blob_only:
result += f"Downloaded {len(download_successes)}/{len(resolvable)}\n" \
result = f"Started {len(first_byte_times)} of {len(url_to_claim)} attempted front page streams\n"
if first_byte_times:
result += f"Worst first byte time: {round(max(first_byte_times), 2)}\n" \
f"Best first byte time: {round(min(first_byte_times), 2)}\n" \
f"95% confidence time-to-first-byte: {confidence(first_byte_times, 1.984)}s\n" \
f"99% confidence time-to-first-byte: {confidence(first_byte_times, 2.626)}s\n" \
f"Variance: {variance(first_byte_times)}\n"
if download_successes:
result += f"Downloaded {len(download_successes)}/{len(url_to_claim)}\n" \
f"Best stream download speed: {round(max(download_speeds), 2)}mb/s\n" \
f"Worst stream download speed: {round(min(download_speeds), 2)}mb/s\n" \
f"95% confidence download speed: {confidence(download_speeds, 1.984, False)}mb/s\n" \
f"99% confidence download speed: {confidence(download_speeds, 2.626, False)}mb/s\n"
if failed_to_start:
result += "\nFailed to start:" + "\n".join([f for f in failed_to_start])
if download_failures:
result += "\nFailed to finish:" + "\n".join([f for f in download_failures])
for reason in ('start', 'finish'):
failures = [url for url, why in failed_to.items() if reason == why]
if failures:
result += f"\nFailed to {reason}:\n" + "\n".join(failures)
print(result)
webhook = os.environ.get('TTFB_SLACK_TOKEN', None)
if webhook:
await report_to_slack(result, webhook)
return 0
if __name__ == "__main__":
parser = argparse.ArgumentParser()
#parser.add_argument("--data_dir")
#parser.add_argument("--wallet_dir")
#parser.add_argument("--download_directory")
parser.add_argument("--allow_fees", action='store_true')
parser.add_argument("--exit_on_error", action='store_true')
parser.add_argument("--stall_download_timeout", default=10, type=int)
parser.add_argument("--delete_after_download", action='store_true')
parser.add_argument("--stall_download_timeout", default=0, type=int)
parser.add_argument("--keep_files", action='store_true')
parser.add_argument("--head_blob_only", action='store_true')
asyncio.run(main(cmd_args=parser.parse_args()))
parser.add_argument("--download_pages", type=int, default=10)
sys.exit(asyncio.run(main(cmd_args=parser.parse_args())) or 0)
+87 -12
View File
@@ -1,3 +1,4 @@
import time
from math import ceil
from aiohttp import web
@@ -14,6 +15,19 @@ class MockedCommentServer:
'INVALID_METHOD': {'code': -32604, 'message': 'The Requested method does not exist'}
}
COMMENT_SCHEMA = {
'comment': None,
'comment_id': None,
'claim_id': None,
'parent_id': None,
'channel_name': None,
'channel_id': None,
'signature': None,
'signing_ts': None,
'timestamp': None,
'channel_url': None,
}
def __init__(self, port=2903):
self.port = port
self.app = web.Application(debug=True)
@@ -23,13 +37,43 @@ class MockedCommentServer:
self.comments = []
self.comment_id = 0
def create_comment(self, **comment):
@classmethod
def _create_comment(cls, **kwargs):
schema = cls.COMMENT_SCHEMA.copy()
schema.update(**kwargs)
return schema
@staticmethod
def clean(d: dict):
return {k: v for k, v in d.items() if v}
def create_comment(self, channel_name=None, channel_id=None, **kwargs):
self.comment_id += 1
comment['comment_id'] = self.comment_id
if 'channel_id' in comment:
comment['channel_url'] = 'lbry://' + comment['channel_name'] + '#' + comment['channel_id']
comment_id = self.comment_id
channel_url = 'lbry://' + channel_name + '#' + channel_id if channel_id else None
comment = self._create_comment(
comment_id=str(comment_id),
channel_name=channel_name,
channel_id=channel_id,
channel_url=channel_url,
timestamp=str(int(time.time())),
**kwargs
)
self.comments.append(comment)
return comment
return self.clean(comment)
def delete_comment(self, comment_id: int, channel_id: str, **kwargs):
deleted = False
try:
if 0 <= comment_id <= len(self.comments) and self.comments[comment_id - 1]['channel_id'] == channel_id:
self.comments.pop(comment_id - 1)
deleted = True
finally:
return {
str(comment_id): {
'deleted': deleted
}
}
def get_claim_comments(self, page=1, page_size=50, **kwargs):
return {
@@ -37,22 +81,36 @@ class MockedCommentServer:
'page_size': page_size,
'total_pages': ceil(len(self.comments)/page_size),
'total_items': len(self.comments),
'items': (self.comments[::-1])[(page - 1) * page_size: page * page_size]
'items': [self.clean(c) for c in (self.comments[::-1])[(page - 1) * page_size: page * page_size]]
}
def get_comment_channel_by_id(self, comment_id: int, **kwargs):
comment = self.comments[comment_id - 1]
return {
'channel_id': comment.get('channel_id'),
'channel_name': comment.get('channel_name')
}
methods = {
'get_claim_comments': get_claim_comments,
'create_comment': create_comment,
'delete_comment': delete_comment,
'get_channel_from_comment_id': get_comment_channel_by_id,
}
def process_json(self, body) -> dict:
response = {'jsonrpc': '2.0', 'id': body['id']}
if body['method'] in self.methods:
params = body.get('params', {})
result = self.methods[body['method']](self, **params)
response['result'] = result
else:
response['error'] = self.ERRORS['INVALID_METHOD']
try:
if body['method'] in self.methods:
params = body.get('params', {})
if 'comment_id' in params and type(params['comment_id']) is str:
params['comment_id'] = int(params['comment_id'])
result = self.methods[body['method']](self, **params)
response['result'] = result
else:
response['error'] = self.ERRORS['INVALID_METHOD']
except Exception:
response['error'] = self.ERRORS['UNKNOWN']
return response
async def start(self):
@@ -173,3 +231,20 @@ class CommentCommands(CommandTestCase):
is_channel_signature_valid=True
)
self.assertIs(len(signed_comment_list['items']), 28)
async def test04_comment_abandons(self):
rswanson = (await self.channel_create('@RonSwanson'))['outputs'][0]
stream = (await self.stream_create('Pawnee Town Hall of Fame by Leslie Knope'))['outputs'][0]
comment = await self.daemon.jsonrpc_comment_create(
comment='KNOPE! WHAT DID I TELL YOU ABOUT PUTTING MY INFORMATION UP LIKE THAT',
claim_id=stream['claim_id'],
channel_id=rswanson['claim_id']
)
self.assertIn('signature', comment)
deleted = await self.daemon.jsonrpc_comment_abandon(comment['comment_id'])
self.assertIn(comment['comment_id'], deleted)
self.assertTrue(deleted[comment['comment_id']]['deleted'])
deleted = await self.daemon.jsonrpc_comment_abandon(comment['comment_id'])
self.assertIn(comment['comment_id'], deleted)
self.assertFalse(deleted[comment['comment_id']]['deleted'])
@@ -1,9 +1,11 @@
import asyncio
import os
from torba.client.basenetwork import ClientSession
from torba.orchstr8 import Conductor
from torba.testcase import IntegrationTestCase
import lbry.wallet
from lbry.testcase import CommandTestCase
from lbry.extras.daemon.Components import HeadersComponent
from torba.client.basenetwork import ClientSession
from torba.testcase import IntegrationTestCase
class TestSessionBloat(IntegrationTestCase):
@@ -29,3 +31,22 @@ class TestSessionBloat(IntegrationTestCase):
await session.send_request('server.banner', ())
self.assertTrue(session.is_closing())
self.assertEqual(len(self.conductor.spv_node.server.session_mgr.sessions), 0)
class TestHeadersComponent(CommandTestCase):
LEDGER = lbry.wallet
async def asyncSetUp(self):
await super().asyncSetUp()
self.component_manager = self.daemon.component_manager
self.component_manager.conf.blockchain_name = 'lbrycrd_main'
self.headers_component = HeadersComponent(self.component_manager)
async def test_cant_reach_host(self):
HeadersComponent.HEADERS_URL = 'notthere/'
os.unlink(self.headers_component.headers.path)
# test is that this doesnt raise
await self.headers_component.start()
self.assertTrue(self.component_manager.get_components_status()['blockchain_headers'])
self.assertEqual(await self.headers_component.get_status(), {})
@@ -1,5 +1,5 @@
from torba.testcase import AsyncioTestCase
import hashlib
from lbry.extras.daemon.comment_client import sign_comment
from lbry.extras.daemon.comment_client import is_comment_signed_by_channel
@@ -14,7 +14,8 @@ class TestSigningComments(AsyncioTestCase):
'claim_id': claim.claim_id,
'channel_name': channel.claim_name,
'channel_id': channel.claim_id,
'comment': comment
'comment': comment,
'comment_id': hashlib.sha256(comment.encode()).hexdigest()
}
def test01_successful_create_sign_and_validate_comment(self):
@@ -31,3 +32,28 @@ class TestSigningComments(AsyncioTestCase):
comment = self.create_claim_comment_body('Woahh This is Sick!! Shout out 2 my boy Tommy H', stream, pdiddy)
sign_comment(comment, channel2)
self.assertFalse(is_comment_signed_by_channel(comment, pdiddy))
def test03_successful_sign_abandon_comment(self):
rswanson = get_channel('@RonSwanson')
dsilver = get_stream('Welcome to the Pawnee, and give a big round for Ron Swanson, AKA Duke Silver')
comment_body = self.create_claim_comment_body('COMPUTER, DELETE ALL VIDEOS OF RON.', dsilver, rswanson)
sign_comment(comment_body, rswanson, abandon=True)
self.assertTrue(is_comment_signed_by_channel(comment_body, rswanson, abandon=True))
def test04_invalid_signature(self):
rswanson = get_channel('@RonSwanson')
jeanralphio = get_channel('@JeanRalphio')
chair = get_stream('This is a nice chair. I made it with Mahogany wood and this electric saw')
chair_comment = self.create_claim_comment_body(
'Hah. You use an electric saw? Us swansons have been making chairs with handsaws just three after birth.',
chair,
rswanson
)
sign_comment(chair_comment, rswanson)
self.assertTrue(is_comment_signed_by_channel(chair_comment, rswanson))
self.assertFalse(is_comment_signed_by_channel(chair_comment, jeanralphio))
fake_abandon_signal = chair_comment.copy()
sign_comment(fake_abandon_signal, jeanralphio, abandon=True)
self.assertFalse(is_comment_signed_by_channel(fake_abandon_signal, rswanson, abandon=True))
self.assertFalse(is_comment_signed_by_channel(fake_abandon_signal, jeanralphio, abandon=True))
@@ -109,7 +109,6 @@ class BasicHeadersTests(BitcoinHeadersTestCase):
# corrupt by appending
headers.io.seek(block_bytes(len(headers)))
headers.io.write(b"appending")
headers._size = None
await headers.repair()
self.assertEqual(headers.height, 1499)
await headers.connect(len(headers), self.get_bytes(block_bytes(3001 - 1500), after=block_bytes(1500)))
+5 -1
View File
@@ -18,6 +18,9 @@ class MockNetwork:
self.get_transaction_called = []
self.is_connected = False
def retriable_call(self, function, *args, **kwargs):
return function(*args, **kwargs)
async def get_history(self, address):
self.get_history_called.append(address)
self.address = address
@@ -121,8 +124,9 @@ class TestSynchronization(LedgerTestCase):
)
class MocHeaderNetwork:
class MocHeaderNetwork(MockNetwork):
def __init__(self, responses):
super().__init__(None, None)
self.responses = responses
async def get_headers(self, height, blocks):
+11 -7
View File
@@ -70,8 +70,10 @@ class BaseHeaders:
return True
def __getitem__(self, height) -> dict:
assert not isinstance(height, slice), \
"Slicing of header chain has not been implemented yet."
if isinstance(height, slice):
raise NotImplementedError("Slicing of header chain has not been implemented yet.")
if not 0 <= height <= self.height:
raise IndexError(f"{height} is out of bounds, current height: {self.height}")
return self.deserialize(height, self.get_raw_header(height))
def get_raw_header(self, height) -> bytes:
@@ -115,7 +117,7 @@ class BaseHeaders:
# .seek()/.write()/.truncate() might also .flush() when needed
# the goal here is mainly to ensure we're definitely flush()'ing
self.io.flush()
self._size = None
self._size = self.io.tell() // self.header_size
added += written
if bail:
break
@@ -170,10 +172,12 @@ class BaseHeaders:
async def repair(self):
previous_header_hash = fail = None
self.io.seek(0)
batch_size = 10000
batch_size = 36
for start_height in range(0, self.height, batch_size):
self.io.seek(self.header_size * start_height)
headers = self.io.read(self.header_size*batch_size)
if len(headers) % self.header_size != 0:
headers = headers[:(len(headers) // self.header_size) * self.header_size]
for header_hash, header in self._iterate_headers(start_height, headers):
height = header['block_height']
if height:
@@ -184,7 +188,7 @@ class BaseHeaders:
fail = True
if fail:
log.warning("Header file corrupted at height %s, truncating it.", height - 1)
self.io.seek((height - 1) * self.header_size, os.SEEK_SET)
self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET)
self.io.truncate()
self.io.flush()
self._size = None
@@ -205,7 +209,7 @@ class BaseHeaders:
end = min(len(headers), end + self.chunk_size * self.header_size)
def _iterate_headers(self, height: int, headers: bytes) -> Iterator[Tuple[bytes, dict]]:
assert len(headers) % self.header_size == 0
assert len(headers) % self.header_size == 0, len(headers)
for idx in range(len(headers) // self.header_size):
start, end = idx * self.header_size, (idx + 1) * self.header_size
header = headers[start:end]
+18 -6
View File
@@ -287,7 +287,7 @@ class BaseLedger(metaclass=LedgerRegistry):
subscription_update = False
if not headers:
header_response = await self.network.get_headers(height, 2001)
header_response = await self.network.retriable_call(self.network.get_headers, height, 2001)
headers = header_response['hex']
if not headers:
@@ -394,7 +394,7 @@ class BaseLedger(metaclass=LedgerRegistry):
if local_status == remote_status:
return
remote_history = await self.network.get_history(address)
remote_history = await self.network.retriable_call(self.network.get_history, address)
cache_tasks = []
synced_history = StringIO()
@@ -447,6 +447,17 @@ class BaseLedger(metaclass=LedgerRegistry):
if address_manager is not None:
await address_manager.ensure_address_gap()
local_status, local_history = await self.get_local_status_and_history(address)
if local_status != remote_status:
log.debug(
"Wallet is out of sync after syncing. Remote: %s with %d items, local: %s with %d items",
remote_status, len(remote_history), local_status, len(local_history)
)
log.debug("local: %s", local_history)
log.debug("remote: %s", remote_history)
else:
log.debug("Sync completed for: %s", address)
async def cache_transaction(self, txid, remote_height):
cache_item = self._tx_cache.get(txid)
if cache_item is None:
@@ -466,7 +477,7 @@ class BaseLedger(metaclass=LedgerRegistry):
if tx is None:
# fetch from network
_raw = await self.network.get_transaction(txid)
_raw = await self.network.retriable_call(self.network.get_transaction, txid)
if _raw:
tx = self.transaction_class(unhexlify(_raw))
await self.maybe_verify_transaction(tx, remote_height)
@@ -486,8 +497,8 @@ class BaseLedger(metaclass=LedgerRegistry):
async def maybe_verify_transaction(self, tx, remote_height):
tx.height = remote_height
if 0 < remote_height <= len(self.headers):
merkle = await self.network.get_merkle(tx.id, remote_height)
if 0 < remote_height < len(self.headers):
merkle = await self.network.retriable_call(self.network.get_merkle, tx.id, remote_height)
merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash)
header = self.headers[remote_height]
tx.position = merkle['pos']
@@ -501,6 +512,7 @@ class BaseLedger(metaclass=LedgerRegistry):
return None
def broadcast(self, tx):
# broadcast cant be a retriable call yet
return self.network.broadcast(hexlify(tx.raw).decode())
async def wait(self, tx: basetransaction.BaseTransaction, height=-1, timeout=None):
@@ -522,4 +534,4 @@ class BaseLedger(metaclass=LedgerRegistry):
)) for address_record in records
], timeout=timeout)
if pending:
raise TimeoutError('Timed out waiting for transaction.')
raise asyncio.TimeoutError('Timed out waiting for transaction.')
+244 -150
View File
@@ -1,9 +1,8 @@
import logging
import asyncio
from asyncio import CancelledError
from time import time
from typing import List
import socket
from operator import itemgetter
from typing import Dict, Optional, Tuple
from time import perf_counter
from torba.rpc import RPCSession as BaseClientSession, Connector, RPCError
@@ -14,71 +13,146 @@ log = logging.getLogger(__name__)
class ClientSession(BaseClientSession):
def __init__(self, *args, network, server, timeout=30, **kwargs):
def __init__(self, *args, network, server, timeout=30, on_connect_callback=None, **kwargs):
self.network = network
self.server = server
super().__init__(*args, **kwargs)
self._on_disconnect_controller = StreamController()
self.on_disconnected = self._on_disconnect_controller.stream
self.bw_limit = self.framer.max_size = self.max_errors = 1 << 32
self.framer.max_size = self.max_errors = 1 << 32
self.bw_limit = -1
self.timeout = timeout
self.max_seconds_idle = timeout * 2
self.ping_task = None
self.response_time: Optional[float] = None
self.connection_latency: Optional[float] = None
self._response_samples = 0
self.pending_amount = 0
self._on_connect_cb = on_connect_callback or (lambda: None)
self.trigger_urgent_reconnect = asyncio.Event()
self._semaphore = asyncio.Semaphore(int(self.timeout))
@property
def available(self):
return not self.is_closing() and self._can_send.is_set() and self.response_time is not None
@property
def server_address_and_port(self) -> Optional[Tuple[str, int]]:
if not self.transport:
return None
return self.transport.get_extra_info('peername')
async def send_timed_server_version_request(self, args=(), timeout=None):
timeout = timeout or self.timeout
log.debug("send version request to %s:%i", *self.server)
start = perf_counter()
result = await asyncio.wait_for(
super().send_request('server.version', args), timeout=timeout
)
current_response_time = perf_counter() - start
response_sum = (self.response_time or 0) * self._response_samples + current_response_time
self.response_time = response_sum / (self._response_samples + 1)
self._response_samples += 1
return result
async def send_request(self, method, args=()):
try:
return await asyncio.wait_for(super().send_request(method, args), timeout=self.timeout)
except RPCError as e:
log.warning("Wallet server returned an error. Code: %s Message: %s", *e.args)
raise e
except asyncio.TimeoutError:
self.abort()
raise
self.pending_amount += 1
async with self._semaphore:
return await self._send_request(method, args)
async def ping_forever(self):
async def _send_request(self, method, args=()):
log.debug("send %s to %s:%i", method, *self.server)
try:
if method == 'server.version':
reply = await self.send_timed_server_version_request(args, self.timeout)
else:
reply = await asyncio.wait_for(
super().send_request(method, args), timeout=self.timeout
)
log.debug("got reply for %s from %s:%i", method, *self.server)
return reply
except RPCError as e:
log.warning("Wallet server (%s:%i) returned an error. Code: %s Message: %s",
*self.server, *e.args)
raise e
except ConnectionError:
log.warning("connection to %s:%i lost", *self.server)
self.synchronous_close()
raise asyncio.CancelledError(f"connection to {self.server[0]}:{self.server[1]} lost")
except asyncio.TimeoutError:
log.info("timeout sending %s to %s:%i", method, *self.server)
raise
except asyncio.CancelledError:
log.info("cancelled sending %s to %s:%i", method, *self.server)
self.synchronous_close()
raise
finally:
self.pending_amount -= 1
async def ensure_session(self):
# Handles reconnecting and maintaining a session alive
# TODO: change to 'ping' on newer protocol (above 1.2)
while not self.is_closing():
if (time() - self.last_send) > self.max_seconds_idle:
try:
await self.send_request('server.banner')
except:
self.abort()
raise
await asyncio.sleep(self.max_seconds_idle//3)
retry_delay = default_delay = 1.0
while True:
try:
if self.is_closing():
await self.create_connection(self.timeout)
await self.ensure_server_version()
self._on_connect_cb()
if (perf_counter() - self.last_send) > self.max_seconds_idle or self.response_time is None:
await self.ensure_server_version()
retry_delay = default_delay
except (asyncio.TimeoutError, OSError):
await self.close()
retry_delay = min(60, retry_delay * 2)
log.debug("Wallet server timeout (retry in %s seconds): %s:%d", retry_delay, *self.server)
try:
await asyncio.wait_for(self.trigger_urgent_reconnect.wait(), timeout=retry_delay)
except asyncio.TimeoutError:
pass
finally:
self.trigger_urgent_reconnect.clear()
async def ensure_server_version(self, required='1.2', timeout=3):
return await asyncio.wait_for(
self.send_request('server.version', [__version__, required]), timeout=timeout
)
async def create_connection(self, timeout=6):
connector = Connector(lambda: self, *self.server)
start = perf_counter()
await asyncio.wait_for(connector.create_connection(), timeout=timeout)
self.ping_task = asyncio.create_task(self.ping_forever())
self.connection_latency = perf_counter() - start
async def handle_request(self, request):
controller = self.network.subscription_controllers[request.method]
controller.add(request.args)
def connection_lost(self, exc):
log.debug("Connection lost: %s:%d", *self.server)
super().connection_lost(exc)
self.response_time = None
self.connection_latency = None
self._response_samples = 0
self.pending_amount = 0
self._on_disconnect_controller.add(True)
if self.ping_task:
self.ping_task.cancel()
class BaseNetwork:
def __init__(self, ledger):
self.config = ledger.config
self.client: ClientSession = None
self.session_pool: SessionPool = None
self.session_pool = SessionPool(network=self, timeout=self.config.get('connect_timeout', 6))
self.client: Optional[ClientSession] = None
self.running = False
self.remote_height: int = 0
self._on_connected_controller = StreamController()
self.on_connected = self._on_connected_controller.stream
self._on_header_controller = StreamController()
self._on_header_controller = StreamController(merge_repeated_events=True)
self.on_header = self._on_header_controller.stream
self._on_status_controller = StreamController()
self._on_status_controller = StreamController(merge_repeated_events=True)
self.on_status = self._on_status_controller.stream
self.subscription_controllers = {
@@ -86,79 +160,72 @@ class BaseNetwork:
'blockchain.address.subscribe': self._on_status_controller,
}
async def switch_to_fastest(self):
try:
client = await asyncio.wait_for(self.session_pool.wait_for_fastest_session(), 30)
except asyncio.TimeoutError:
if self.client:
await self.client.close()
self.client = None
for session in self.session_pool.sessions:
session.synchronous_close()
log.warning("not connected to any wallet servers")
return
current_client = self.client
self.client = client
log.info("Switching to SPV wallet server: %s:%d", *self.client.server)
self._on_connected_controller.add(True)
try:
self._update_remote_height((await self.subscribe_headers(),))
log.info("Subscribed to headers: %s:%d", *self.client.server)
except asyncio.TimeoutError:
if self.client:
await self.client.close()
self.client = current_client
return
self.session_pool.new_connection_event.clear()
return await self.session_pool.new_connection_event.wait()
async def start(self):
self.running = True
connect_timeout = self.config.get('connect_timeout', 6)
self.session_pool = SessionPool(network=self, timeout=connect_timeout)
self.session_pool.start(self.config['default_servers'])
self.on_header.listen(self._update_remote_height)
while True:
try:
self.client = await self.pick_fastest_session()
if self.is_connected:
await self.ensure_server_version()
self._update_remote_height((await self.subscribe_headers(),))
log.info("Successfully connected to SPV wallet server: %s:%d", *self.client.server)
self._on_connected_controller.add(True)
await self.client.on_disconnected.first
except CancelledError:
self.running = False
except asyncio.TimeoutError:
log.warning("Timed out while trying to find a server!")
except Exception: # pylint: disable=broad-except
log.exception("Exception while trying to find a server!")
if not self.running:
return
elif self.client:
await self.client.close()
self.client.connection.cancel_pending_requests()
while self.running:
await self.switch_to_fastest()
async def stop(self):
self.running = False
if self.session_pool:
self.session_pool.stop()
if self.is_connected:
disconnected = self.client.on_disconnected.first
await self.client.close()
await disconnected
self.session_pool.stop()
@property
def is_connected(self):
return self.client is not None and not self.client.is_closing()
return self.client and not self.client.is_closing()
def rpc(self, list_or_method, args):
if self.is_connected:
return self.client.send_request(list_or_method, args)
def rpc(self, list_or_method, args, session=None):
session = session or self.session_pool.fastest_session
if session:
return session.send_request(list_or_method, args)
else:
self.session_pool.trigger_nodelay_connect()
raise ConnectionError("Attempting to send rpc request when connection is not available.")
async def pick_fastest_session(self):
sessions = await self.session_pool.get_online_sessions()
done, pending = await asyncio.wait([
self.probe_session(session)
for session in sessions if not session.is_closing()
], return_when='FIRST_COMPLETED')
for task in pending:
task.cancel()
for session in done:
return await session
async def probe_session(self, session: ClientSession):
await session.send_request('server.banner')
return session
async def retriable_call(self, function, *args, **kwargs):
while self.running:
if not self.is_connected:
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
await self.on_connected.first
await self.session_pool.wait_for_fastest_session()
try:
return await function(*args, **kwargs)
except asyncio.TimeoutError:
log.warning("Wallet server call timed out, retrying.")
except ConnectionError:
pass
raise asyncio.CancelledError() # if we got here, we are shutting down
def _update_remote_height(self, header_args):
self.remote_height = header_args[0]["height"]
def ensure_server_version(self, required='1.2'):
return self.rpc('server.version', [__version__, required])
def broadcast(self, raw_transaction):
return self.rpc('blockchain.transaction.broadcast', [raw_transaction])
def get_history(self, address):
return self.rpc('blockchain.address.get_history', [address])
def get_transaction(self, tx_hash):
return self.rpc('blockchain.transaction.get', [tx_hash])
@@ -171,84 +238,111 @@ class BaseNetwork:
def get_headers(self, height, count=10000):
return self.rpc('blockchain.block.headers', [height, count])
def subscribe_headers(self):
return self.rpc('blockchain.headers.subscribe', [True])
# --- Subscribes, history and broadcasts are always aimed towards the master client directly
def get_history(self, address):
return self.rpc('blockchain.address.get_history', [address], session=self.client)
def subscribe_address(self, address):
return self.rpc('blockchain.address.subscribe', [address])
def broadcast(self, raw_transaction):
return self.rpc('blockchain.transaction.broadcast', [raw_transaction], session=self.client)
def subscribe_headers(self):
return self.rpc('blockchain.headers.subscribe', [True], session=self.client)
async def subscribe_address(self, address):
try:
return await self.rpc('blockchain.address.subscribe', [address], session=self.client)
except asyncio.TimeoutError:
# abort and cancel, we cant lose a subscription, it will happen again on reconnect
self.client.abort()
raise asyncio.CancelledError()
class SessionPool:
def __init__(self, network: BaseNetwork, timeout: float):
self.network = network
self.sessions: List[ClientSession] = []
self._dead_servers: List[ClientSession] = []
self.maintain_connections_task = None
self.sessions: Dict[ClientSession, Optional[asyncio.Task]] = dict()
self.timeout = timeout
# triggered when the master server is out, to speed up reconnect
self._lost_master = asyncio.Event()
self.new_connection_event = asyncio.Event()
@property
def online(self):
for session in self.sessions:
if not session.is_closing():
return True
return False
return any(not session.is_closing() for session in self.sessions)
@property
def available_sessions(self):
return [session for session in self.sessions if session.available]
@property
def fastest_session(self):
if not self.available_sessions:
return None
return min(
[((session.response_time + session.connection_latency) * (session.pending_amount + 1), session)
for session in self.available_sessions],
key=itemgetter(0)
)[1]
def _get_session_connect_callback(self, session: ClientSession):
loop = asyncio.get_event_loop()
def callback():
duplicate_connections = [
s for s in self.sessions
if s is not session and s.server_address_and_port == session.server_address_and_port
]
already_connected = None if not duplicate_connections else duplicate_connections[0]
if already_connected:
self.sessions.pop(session).cancel()
session.synchronous_close()
log.debug("wallet server %s resolves to the same server as %s, rechecking in an hour",
session.server[0], already_connected.server[0])
loop.call_later(3600, self._connect_session, session.server)
return
self.new_connection_event.set()
log.info("connected to %s:%i", *session.server)
return callback
def _connect_session(self, server: Tuple[str, int]):
session = None
for s in self.sessions:
if s.server == server:
session = s
break
if not session:
session = ClientSession(
network=self.network, server=server
)
session._on_connect_cb = self._get_session_connect_callback(session)
task = self.sessions.get(session, None)
if not task or task.done():
task = asyncio.create_task(session.ensure_session())
task.add_done_callback(lambda _: self.ensure_connections())
self.sessions[session] = task
def start(self, default_servers):
self.sessions = [
ClientSession(network=self.network, server=server)
for server in default_servers
]
self.maintain_connections_task = asyncio.create_task(self.ensure_connections())
for server in default_servers:
self._connect_session(server)
def stop(self):
if self.maintain_connections_task:
self.maintain_connections_task.cancel()
for task in self.sessions.values():
task.cancel()
self.sessions.clear()
def ensure_connections(self):
for session in self.sessions:
if not session.is_closing():
session.abort()
self.sessions, self._dead_servers, self.maintain_connections_task = [], [], None
self._connect_session(session.server)
async def ensure_connections(self):
while True:
await asyncio.gather(*[
self.ensure_connection(session)
for session in self.sessions
], return_exceptions=True)
try:
await asyncio.wait_for(self._lost_master.wait(), timeout=3)
except asyncio.TimeoutError:
pass
self._lost_master.clear()
if not self.sessions:
self.sessions.extend(self._dead_servers)
self._dead_servers = []
def trigger_nodelay_connect(self):
# used when other parts of the system sees we might have internet back
# bypasses the retry interval
for session in self.sessions:
session.trigger_urgent_reconnect.set()
async def ensure_connection(self, session):
self._dead_servers.append(session)
self.sessions.remove(session)
try:
if session.is_closing():
await session.create_connection(self.timeout)
await asyncio.wait_for(session.send_request('server.banner'), timeout=self.timeout)
self.sessions.append(session)
self._dead_servers.remove(session)
except asyncio.TimeoutError:
log.warning("Timeout connecting to %s:%d", *session.server)
except asyncio.CancelledError: # pylint: disable=try-except-raise
raise
except socket.gaierror:
log.warning("Could not resolve IP for %s", session.server[0])
except Exception as err: # pylint: disable=broad-except
if 'Connect call failed' in str(err):
log.warning("Could not connect to %s:%d", *session.server)
else:
log.exception("Connecting to %s:%d raised an exception:", *session.server)
async def get_online_sessions(self):
while not self.online:
self._lost_master.set()
await asyncio.sleep(0.5)
return self.sessions
async def wait_for_fastest_session(self):
while not self.fastest_session:
self.trigger_nodelay_connect()
self.new_connection_event.clear()
await self.new_connection_event.wait()
return self.fastest_session
+3 -3
View File
@@ -33,6 +33,7 @@ __all__ = ('JSONRPC', 'JSONRPCv1', 'JSONRPCv2', 'JSONRPCLoose',
import itertools
import json
import typing
import asyncio
from functools import partial
from numbers import Number
@@ -745,9 +746,8 @@ class JSONRPCConnection(object):
self._protocol = item
return self.receive_message(message)
def cancel_pending_requests(self):
"""Cancel all pending requests."""
exception = CancelledError()
def raise_pending_requests(self, exception):
exception = exception or asyncio.TimeoutError()
for request, event in self._requests.values():
event.result = exception
event.set()
+17 -8
View File
@@ -103,7 +103,7 @@ class SessionBase(asyncio.Protocol):
# Force-close a connection if a send doesn't succeed in this time
self.max_send_delay = 60
# Statistics. The RPC object also keeps its own statistics.
self.start_time = time.time()
self.start_time = time.perf_counter()
self.errors = 0
self.send_count = 0
self.send_size = 0
@@ -123,7 +123,7 @@ class SessionBase(asyncio.Protocol):
# A non-positive value means not to limit concurrency
if self.bw_limit <= 0:
return
now = time.time()
now = time.perf_counter()
# Reduce the recorded usage in proportion to the elapsed time
refund = (now - self.bw_time) * (self.bw_limit / 3600)
self.bw_charge = max(0, self.bw_charge - int(refund))
@@ -146,7 +146,7 @@ class SessionBase(asyncio.Protocol):
await asyncio.wait_for(self._can_send.wait(), secs)
except asyncio.TimeoutError:
self.abort()
raise asyncio.CancelledError(f'task timed out after {secs}s')
raise asyncio.TimeoutError(f'task timed out after {secs}s')
async def _send_message(self, message):
if not self._can_send.is_set():
@@ -156,7 +156,7 @@ class SessionBase(asyncio.Protocol):
self.send_size += len(framed_message)
self._using_bandwidth(len(framed_message))
self.send_count += 1
self.last_send = time.time()
self.last_send = time.perf_counter()
if self.verbosity >= 4:
self.logger.debug(f'Sending framed message {framed_message}')
self.transport.write(framed_message)
@@ -215,7 +215,8 @@ class SessionBase(asyncio.Protocol):
self._address = None
self.transport = None
self._task_group.cancel()
self._pm_task.cancel()
if self._pm_task:
self._pm_task.cancel()
# Release waiting tasks
self._can_send.set()
@@ -253,6 +254,7 @@ class SessionBase(asyncio.Protocol):
if self.transport:
self.transport.abort()
# TODO: replace with synchronous_close
async def close(self, *, force_after=30):
"""Close the connection and return when closed."""
self._close()
@@ -262,6 +264,11 @@ class SessionBase(asyncio.Protocol):
self.abort()
await self._pm_task
def synchronous_close(self):
self._close()
if self._pm_task and not self._pm_task.done():
self._pm_task.cancel()
class MessageSession(SessionBase):
"""Session class for protocols where messages are not tied to responses,
@@ -296,7 +303,7 @@ class MessageSession(SessionBase):
)
self._bump_errors()
else:
self.last_recv = time.time()
self.last_recv = time.perf_counter()
self.recv_count += 1
if self.recv_count % 10 == 0:
await self._update_concurrency()
@@ -416,7 +423,7 @@ class RPCSession(SessionBase):
self.logger.warning(f'{e!r}')
continue
self.last_recv = time.time()
self.last_recv = time.perf_counter()
self.recv_count += 1
if self.recv_count % 10 == 0:
await self._update_concurrency()
@@ -456,7 +463,7 @@ class RPCSession(SessionBase):
def connection_lost(self, exc):
# Cancel pending requests and message processing
self.connection.cancel_pending_requests()
self.connection.raise_pending_requests(exc)
super().connection_lost(exc)
# External API
@@ -473,6 +480,8 @@ class RPCSession(SessionBase):
async def send_request(self, method, args=()):
"""Send an RPC request over the network."""
if self.is_closing():
raise asyncio.TimeoutError("Trying to send request on a recently dropped connection.")
message, event = self.connection.send_request(Request(method, args))
await self._send_message(message)
await event.wait()
+1 -1
View File
@@ -258,7 +258,7 @@ class SessionManager:
session_timeout = self.env.session_timeout
while True:
await sleep(session_timeout // 10)
stale_cutoff = time.time() - session_timeout
stale_cutoff = time.perf_counter() - session_timeout
stale_sessions = [session for session in self.sessions
if session.last_recv < stale_cutoff]
if stale_sessions:
+8 -4
View File
@@ -45,10 +45,12 @@ class BroadcastSubscription:
class StreamController:
def __init__(self):
def __init__(self, merge_repeated_events=False):
self.stream = Stream(self)
self._first_subscription = None
self._last_subscription = None
self._last_event = None
self._merge_repeated = merge_repeated_events
@property
def has_listener(self):
@@ -76,8 +78,10 @@ class StreamController:
return f
def add(self, event):
skip = self._merge_repeated and event == self._last_event
self._last_event = event
return self._notify_and_ensure_future(
lambda subscription: subscription._add(event)
lambda subscription: None if skip else subscription._add(event)
)
def add_error(self, exception):
@@ -141,8 +145,8 @@ class Stream:
def first(self):
future = asyncio.get_event_loop().create_future()
subscription = self.listen(
lambda value: self._cancel_and_callback(subscription, future, value),
lambda exception: self._cancel_and_error(subscription, future, exception)
lambda value: not future.done() and self._cancel_and_callback(subscription, future, value),
lambda exception: not future.done() and self._cancel_and_error(subscription, future, exception)
)
return future