Compare commits
62
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f1d7560098 | ||
|
|
bd83ee7931 | ||
|
|
f6bdf7c09a | ||
|
|
2db8afb8c2 | ||
|
|
e033129dd3 | ||
|
|
f9dc590100 | ||
|
|
8996aafe0d | ||
|
|
9dc25ef7af | ||
|
|
2b84f4d407 | ||
|
|
097c8b674c | ||
|
|
ba649d4b94 | ||
|
|
6ed1614db0 | ||
|
|
df5b6a8380 | ||
|
|
1f82a8b99e | ||
|
|
0c95d96f32 | ||
|
|
c2f5f84118 | ||
|
|
b3b5e3d8f0 | ||
|
|
506d3f3cd9 | ||
|
|
516a8c5ee5 | ||
|
|
2f81e9d374 | ||
|
|
2d8703bb8d | ||
|
|
76e60d9bc3 | ||
|
|
9d5370be5f | ||
|
|
fc1a06bc45 | ||
|
|
fce80374f4 | ||
|
|
420c9f10c2 | ||
|
|
5a39681a2e | ||
|
|
7a1b7db7c8 | ||
|
|
03a643da52 | ||
|
|
383f0ce450 | ||
|
|
c6c668676c | ||
|
|
7b01dde063 | ||
|
|
8c25f65024 | ||
|
|
1e478e3545 | ||
|
|
6c75a8978b | ||
|
|
640267fc2b | ||
|
|
33c7c3ee12 | ||
|
|
f9b41d34ae | ||
|
|
d4bec79451 | ||
|
|
ac1a8b4daf | ||
|
|
28838c1759 | ||
|
|
50ecb0dac9 | ||
|
|
e22bc01cbd | ||
|
|
6c28713a4c | ||
|
|
fc9023386c | ||
|
|
e6cae9bcc3 | ||
|
|
a9eeca1302 | ||
|
|
8c695e42ca | ||
|
|
0aa7fd47d5 | ||
|
|
70596042d6 | ||
|
|
caf616234b | ||
|
|
375187aa70 | ||
|
|
71eccdc0e3 | ||
|
|
639b1e48f5 | ||
|
|
0bb4cb4472 | ||
|
|
cc51543851 | ||
|
|
22540390e1 | ||
|
|
98565eb67c | ||
|
|
cf6a47ecb7 | ||
|
|
fa60b9f9d3 | ||
|
|
644120ca31 | ||
|
|
a50a625b3b |
No files matched your search
@@ -67,6 +67,9 @@ jobs:
|
||||
run: |
|
||||
pip install pyinstaller
|
||||
pip install -e .
|
||||
# https://stackoverflow.com/a/61693590
|
||||
# https://github.com/pypa/setuptools/issues/1963
|
||||
pip install --upgrade 'setuptools<45.0.0'
|
||||
- if: startsWith(matrix.os, 'windows') == false
|
||||
name: Build & Run (Unix)
|
||||
run: |
|
||||
|
||||
+4
-1
@@ -14,4 +14,7 @@ _trial_temp/
|
||||
/tests/integration/blockchain/files
|
||||
/tests/.coverage.*
|
||||
|
||||
/lbry/wallet/bin
|
||||
/lbry/wallet/bin
|
||||
|
||||
/.vscode
|
||||
/.gitignore
|
||||
+1
-1
@@ -1,2 +1,2 @@
|
||||
__version__ = "0.77.0"
|
||||
__version__ = "0.79.1"
|
||||
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
|
||||
+4
-5
@@ -5,9 +5,8 @@ import typing
|
||||
import logging
|
||||
from argparse import ArgumentParser
|
||||
from contextlib import contextmanager
|
||||
|
||||
import yaml
|
||||
from appdirs import user_data_dir, user_config_dir
|
||||
import yaml
|
||||
from lbry.error import InvalidCurrencyError
|
||||
from lbry.dht import constants
|
||||
from lbry.wallet.coinselection import STRATEGIES
|
||||
@@ -334,7 +333,7 @@ class ConfigFileAccess:
|
||||
cls = type(self.configuration)
|
||||
with open(self.path, 'r') as config_file:
|
||||
raw = config_file.read()
|
||||
serialized = yaml.load(raw) or {}
|
||||
serialized = yaml.safe_load(raw) or {}
|
||||
for key, value in serialized.items():
|
||||
attr = getattr(cls, key, None)
|
||||
if attr is None:
|
||||
@@ -479,7 +478,7 @@ class TranscodeConfig(BaseConfig):
|
||||
video_encoder = String('FFmpeg codec and parameters for the video encoding. '
|
||||
'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental',
|
||||
'libx264 -crf 24 -preset faster -pix_fmt yuv420p')
|
||||
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 8400000)
|
||||
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 5_000_000)
|
||||
video_scaler = String('FFmpeg scaling parameters for reducing bitrate. '
|
||||
'Example: -vf "scale=-2:720,fps=24" -maxrate 5M -bufsize 3M',
|
||||
r'-vf "scale=if(gte(iw\,ih)\,min(1920\,iw)\,-2):if(lt(iw\,ih)\,min(1920\,ih)\,-2)" '
|
||||
@@ -487,7 +486,7 @@ class TranscodeConfig(BaseConfig):
|
||||
audio_encoder = String('FFmpeg codec and parameters for the audio encoding. '
|
||||
'Example: libopus -b:a 128k',
|
||||
'aac -b:a 160k')
|
||||
volume_filter = String('FFmpeg filter for audio normalization.', '-af loudnorm')
|
||||
volume_filter = String('FFmpeg filter for audio normalization. Exmple: -af loudnorm', '')
|
||||
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', 240)
|
||||
|
||||
|
||||
|
||||
@@ -1459,7 +1459,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
@requires(WALLET_COMPONENT)
|
||||
async def jsonrpc_wallet_send(
|
||||
self, amount, addresses, wallet_id=None,
|
||||
change_account_id=None, funding_account_ids=None, preview=False):
|
||||
change_account_id=None, funding_account_ids=None, preview=False, blocking=True):
|
||||
"""
|
||||
Send the same number of credits to multiple addresses using all accounts in wallet to
|
||||
fund the transaction and the default account to receive any change.
|
||||
@@ -1467,12 +1467,14 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
Usage:
|
||||
wallet_send <amount> <addresses>... [--wallet_id=<wallet_id>] [--preview]
|
||||
[--change_account_id=None] [--funding_account_ids=<funding_account_ids>...]
|
||||
[--blocking]
|
||||
|
||||
Options:
|
||||
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
|
||||
--change_account_id=<wallet_id> : (str) account where change will go
|
||||
--funding_account_ids=<funding_account_ids> : (str) accounts to fund the transaction
|
||||
--preview : (bool) do not broadcast the transaction
|
||||
--preview : (bool) do not broadcast the transaction
|
||||
--blocking : (bool) wait until tx has synced
|
||||
|
||||
Returns: {Transaction}
|
||||
"""
|
||||
@@ -1498,13 +1500,11 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
tx = await Transaction.create(
|
||||
[], outputs, accounts, account
|
||||
)
|
||||
|
||||
if not preview:
|
||||
await self.ledger.broadcast(tx)
|
||||
await self.broadcast_or_release(tx, blocking)
|
||||
self.component_manager.loop.create_task(self.analytics_manager.send_credits_sent())
|
||||
else:
|
||||
await self.ledger.release_tx(tx)
|
||||
|
||||
return tx
|
||||
|
||||
ACCOUNT_DOC = """
|
||||
@@ -1714,7 +1714,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
change_made = True
|
||||
|
||||
if change_made:
|
||||
account.modified_on = time.time()
|
||||
account.modified_on = int(time.time())
|
||||
wallet.save()
|
||||
|
||||
return account
|
||||
@@ -1782,24 +1782,26 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
)
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
def jsonrpc_account_send(self, amount, addresses, account_id=None, wallet_id=None, preview=False):
|
||||
def jsonrpc_account_send(self, amount, addresses, account_id=None, wallet_id=None, preview=False, blocking=False):
|
||||
"""
|
||||
Send the same number of credits to multiple addresses from a specific account (or default account).
|
||||
|
||||
Usage:
|
||||
account_send <amount> <addresses>... [--account_id=<account_id>] [--wallet_id=<wallet_id>] [--preview]
|
||||
[--blocking]
|
||||
|
||||
Options:
|
||||
--account_id=<account_id> : (str) account to fund the transaction
|
||||
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
|
||||
--preview : (bool) do not broadcast the transaction
|
||||
--blocking : (bool) wait until tx has synced
|
||||
|
||||
Returns: {Transaction}
|
||||
"""
|
||||
return self.jsonrpc_wallet_send(
|
||||
amount=amount, addresses=addresses, wallet_id=wallet_id,
|
||||
change_account_id=account_id, funding_account_ids=[account_id] if account_id else [],
|
||||
preview=preview
|
||||
preview=preview, blocking=blocking
|
||||
)
|
||||
|
||||
SYNC_DOC = """
|
||||
@@ -4987,10 +4989,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
View, create and abandon comments.
|
||||
"""
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
async def jsonrpc_comment_list(self, claim_id, parent_id=None, page=1, page_size=50,
|
||||
include_replies=True, is_channel_signature_valid=False,
|
||||
hidden=False, visible=False):
|
||||
include_replies=False, skip_validation=False,
|
||||
is_channel_signature_valid=False, hidden=False, visible=False):
|
||||
"""
|
||||
List comments associated with a claim.
|
||||
|
||||
@@ -4998,7 +4999,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
comment_list (<claim_id> | --claim_id=<claim_id>)
|
||||
[(--page=<page> --page_size=<page_size>)]
|
||||
[--parent_id=<parent_id>] [--include_replies]
|
||||
[--is_channel_signature_valid]
|
||||
[--skip_validation] [--is_channel_signature_valid]
|
||||
[--visible | --hidden]
|
||||
|
||||
Options:
|
||||
@@ -5006,6 +5007,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--parent_id=<parent_id> : (str) CommentId of a specific thread you'd like to see
|
||||
--page=<page> : (int) The page you'd like to see in the comment list.
|
||||
--page_size=<page_size> : (int) The amount of comments that you'd like to retrieve
|
||||
--skip_validation : (bool) Skip resolving comments to validate channel names
|
||||
--include_replies : (bool) Whether or not you want to include replies in list
|
||||
--is_channel_signature_valid : (bool) Only include comments with valid signatures.
|
||||
[Warning: Paginated total size will not change, even
|
||||
@@ -5055,21 +5057,26 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
page_size=page_size,
|
||||
top_level=not include_replies
|
||||
)
|
||||
for comment in result.get('items', []):
|
||||
channel_url = comment.get('channel_url')
|
||||
if not channel_url:
|
||||
continue
|
||||
resolve_response = await self.resolve([], [channel_url])
|
||||
if isinstance(resolve_response[channel_url], Output):
|
||||
comment['is_channel_signature_valid'] = comment_client.is_comment_signed_by_channel(
|
||||
comment, resolve_response[channel_url]
|
||||
)
|
||||
else:
|
||||
comment['is_channel_signature_valid'] = False
|
||||
if is_channel_signature_valid:
|
||||
result['items'] = [
|
||||
c for c in result.get('items', []) if c.get('is_channel_signature_valid', False)
|
||||
]
|
||||
if not skip_validation:
|
||||
channel_urls = []
|
||||
for comment in result.get('items', []):
|
||||
channel_url = comment.get('channel_url')
|
||||
if channel_url:
|
||||
channel_urls.append(channel_url)
|
||||
resolve_response = await self.resolve([], channel_urls)
|
||||
|
||||
for comment in result.get('items', []):
|
||||
channel_url = comment.get('channel_url')
|
||||
if isinstance(resolve_response[channel_url], Output):
|
||||
comment['is_channel_signature_valid'] = comment_client.is_comment_signed_by_channel(
|
||||
comment, resolve_response[channel_url]
|
||||
)
|
||||
else:
|
||||
comment['is_channel_signature_valid'] = False
|
||||
if is_channel_signature_valid:
|
||||
result['items'] = [
|
||||
c for c in result.get('items', []) if c.get('is_channel_signature_valid', False)
|
||||
]
|
||||
return result
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
|
||||
@@ -199,6 +199,8 @@ class VideoFileAnalyzer:
|
||||
if not {"aac", "mp3", "flac", "vorbis", "opus"}.intersection(codec.split(",")):
|
||||
return "Audio codec is not in the approved list of AAC, FLAC, MP3, Vorbis, and Opus. " \
|
||||
f"Actual: {codec} [{stream['codec_long_name']}]"
|
||||
if int(stream['sample_rate']) > 48000:
|
||||
return "Sample rate out of range"
|
||||
|
||||
return ""
|
||||
|
||||
@@ -304,9 +306,6 @@ class VideoFileAnalyzer:
|
||||
|
||||
raise Exception(f"The audio encoder is not available. Requested: {encoder or 'aac'}")
|
||||
|
||||
async def _get_volume_filter(self):
|
||||
return self._conf.volume_filter if self._conf.volume_filter else "-af loudnorm"
|
||||
|
||||
@staticmethod
|
||||
def _get_best_container_extension(scan_data, video_encoder):
|
||||
# the container is chosen by the video format
|
||||
@@ -410,7 +409,7 @@ class VideoFileAnalyzer:
|
||||
# the plan for transcoding:
|
||||
# we have to re-encode the video if it is in a nonstandard format
|
||||
# we also re-encode if we are h264 but not yuv420p (both errors caught in video_msg)
|
||||
# we also re-encode if our bitrate is too high
|
||||
# we also re-encode if our bitrate or sample rate is too high
|
||||
|
||||
try:
|
||||
transcode_command = [f'-i "{file_path}" -y -c:s copy -c:d copy -c:v']
|
||||
@@ -430,9 +429,10 @@ class VideoFileAnalyzer:
|
||||
if audio_msg or volume_msg:
|
||||
audio_encoder = await self._get_audio_encoder(extension)
|
||||
transcode_command.append(audio_encoder)
|
||||
if volume_msg:
|
||||
volume_filter = await self._get_volume_filter()
|
||||
transcode_command.append(volume_filter)
|
||||
if volume_msg and self._conf.volume_filter:
|
||||
transcode_command.append(self._conf.volume_filter)
|
||||
if audio_msg == "Sample rate out of range":
|
||||
transcode_command.append(" -ar 48000 ")
|
||||
else:
|
||||
transcode_command.append("copy")
|
||||
|
||||
|
||||
@@ -47,7 +47,7 @@ types_map = {
|
||||
'.latex': ('application/x-latex', 'binary'),
|
||||
'.m1v': ('video/mpeg', 'video'),
|
||||
'.m3u': ('application/vnd.apple.mpegurl', 'audio'),
|
||||
'.m3u8': ('application/vnd.apple.mpegurl', 'audio'),
|
||||
'.m3u8': ('application/vnd.apple.mpegurl', 'video'),
|
||||
'.man': ('application/x-troff-man', 'document'),
|
||||
'.markdown': ('text/markdown', 'document'),
|
||||
'.md': ('text/markdown', 'document'),
|
||||
@@ -61,6 +61,7 @@ types_map = {
|
||||
'.mp3': ('audio/mpeg', 'audio'),
|
||||
'.mp4': ('video/mp4', 'video'),
|
||||
'.mpa': ('video/mpeg', 'video'),
|
||||
'.mpd': ('application/dash+xml', 'video'),
|
||||
'.mpe': ('video/mpeg', 'video'),
|
||||
'.mpeg': ('video/mpeg', 'video'),
|
||||
'.mpg': ('video/mpeg', 'video'),
|
||||
|
||||
@@ -356,6 +356,9 @@ class ManagedStream(ManagedDownloadSource):
|
||||
return sent
|
||||
except ConnectionRefusedError:
|
||||
return sent
|
||||
except OSError:
|
||||
# raised if a blob is deleted while it's being sent
|
||||
return sent
|
||||
finally:
|
||||
if protocol.transport:
|
||||
protocol.transport.close()
|
||||
|
||||
@@ -60,10 +60,16 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
|
||||
async def send_request(self, request_dict: typing.Dict, timeout: int = 180):
|
||||
msg = json.dumps(request_dict)
|
||||
self.transport.write(msg.encode())
|
||||
try:
|
||||
self.transport.write(msg.encode())
|
||||
self.pending_request = self.loop.create_task(asyncio.wait_for(self.response_queue.get(), timeout))
|
||||
return await self.pending_request
|
||||
except (AttributeError, asyncio.CancelledError):
|
||||
# attribute error happens when we transport.write after disconnect
|
||||
# cancelled error happens when the pending_request task is cancelled by a disconnect
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
raise asyncio.TimeoutError()
|
||||
finally:
|
||||
self.pending_request = None
|
||||
|
||||
|
||||
@@ -15,7 +15,9 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ReflectorServerProtocol(asyncio.Protocol):
|
||||
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000):
|
||||
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000,
|
||||
stop_event: asyncio.Event = None, incoming_event: asyncio.Event = None,
|
||||
not_incoming_event: asyncio.Event = None):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
@@ -27,11 +29,25 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
self.descriptor: typing.Optional['StreamDescriptor'] = None
|
||||
self.sd_blob: typing.Optional['BlobFile'] = None
|
||||
self.received = []
|
||||
self.incoming = asyncio.Event(loop=self.loop)
|
||||
self.incoming = incoming_event or asyncio.Event(loop=self.loop)
|
||||
self.not_incoming = not_incoming_event or asyncio.Event(loop=self.loop)
|
||||
self.stop_event = stop_event or asyncio.Event(loop=self.loop)
|
||||
self.chunk_size = response_chunk_size
|
||||
self.wait_for_stop_task: typing.Optional[asyncio.Task] = None
|
||||
|
||||
async def wait_for_stop(self):
|
||||
await self.stop_event.wait()
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
|
||||
def connection_made(self, transport):
|
||||
self.transport = transport
|
||||
self.wait_for_stop_task = self.loop.create_task(self.wait_for_stop())
|
||||
|
||||
def connection_lost(self, exc):
|
||||
if self.wait_for_stop_task:
|
||||
self.wait_for_stop_task.cancel()
|
||||
self.wait_for_stop_task = None
|
||||
|
||||
def data_received(self, data: bytes):
|
||||
if self.incoming.is_set():
|
||||
@@ -73,6 +89,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
self.sd_blob = self.blob_manager.get_blob(request['sd_blob_hash'], request['sd_blob_size'])
|
||||
if not self.sd_blob.get_is_verified():
|
||||
self.writer = self.sd_blob.get_blob_writer(self.transport.get_extra_info('peername'))
|
||||
self.not_incoming.clear()
|
||||
self.incoming.set()
|
||||
self.send_response({"send_sd_blob": True})
|
||||
try:
|
||||
@@ -86,6 +103,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
self.transport.close()
|
||||
finally:
|
||||
self.incoming.clear()
|
||||
self.not_incoming.set()
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
else:
|
||||
@@ -93,6 +111,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
self.loop, self.blob_manager.blob_dir, self.sd_blob
|
||||
)
|
||||
self.incoming.clear()
|
||||
self.not_incoming.set()
|
||||
if self.writer:
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
@@ -112,6 +131,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
blob = self.blob_manager.get_blob(request['blob_hash'], request['blob_size'])
|
||||
if not blob.get_is_verified():
|
||||
self.writer = blob.get_blob_writer(self.transport.get_extra_info('peername'))
|
||||
self.not_incoming.clear()
|
||||
self.incoming.set()
|
||||
self.send_response({"send_blob": True})
|
||||
try:
|
||||
@@ -120,6 +140,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
except asyncio.TimeoutError:
|
||||
self.send_response({"received_blob": False})
|
||||
self.incoming.clear()
|
||||
self.not_incoming.set()
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
else:
|
||||
@@ -130,12 +151,18 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
|
||||
|
||||
class ReflectorServer:
|
||||
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000):
|
||||
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000,
|
||||
stop_event: asyncio.Event = None, incoming_event: asyncio.Event = None,
|
||||
not_incoming_event: asyncio.Event = None):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: typing.Optional[asyncio.Task] = None
|
||||
self.started_listening = asyncio.Event(loop=self.loop)
|
||||
self.stopped_listening = asyncio.Event(loop=self.loop)
|
||||
self.incoming_event = incoming_event or asyncio.Event(loop=self.loop)
|
||||
self.not_incoming_event = not_incoming_event or asyncio.Event(loop=self.loop)
|
||||
self.response_chunk_size = response_chunk_size
|
||||
self.stop_event = stop_event
|
||||
|
||||
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
|
||||
if self.server_task is not None:
|
||||
@@ -143,13 +170,20 @@ class ReflectorServer:
|
||||
|
||||
async def _start_server():
|
||||
server = await self.loop.create_server(
|
||||
lambda: ReflectorServerProtocol(self.blob_manager, self.response_chunk_size),
|
||||
lambda: ReflectorServerProtocol(
|
||||
self.blob_manager, self.response_chunk_size, self.stop_event, self.incoming_event,
|
||||
self.not_incoming_event
|
||||
),
|
||||
interface, port
|
||||
)
|
||||
self.started_listening.set()
|
||||
self.stopped_listening.clear()
|
||||
log.info("Reflector server listening on TCP %s:%i", interface, port)
|
||||
async with server:
|
||||
await server.serve_forever()
|
||||
try:
|
||||
async with server:
|
||||
await server.serve_forever()
|
||||
finally:
|
||||
self.stopped_listening.set()
|
||||
|
||||
self.server_task = self.loop.create_task(_start_server())
|
||||
|
||||
|
||||
@@ -159,8 +159,18 @@ class StreamManager(SourceManager):
|
||||
))
|
||||
|
||||
async def reflect_streams(self):
|
||||
try:
|
||||
return await self._reflact_streams()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
log.exception("reflector task encountered an unexpected error!")
|
||||
|
||||
async def _reflact_streams(self):
|
||||
# todo: those debug statements are temporary for #2987 - remove them if its closed
|
||||
while True:
|
||||
if self.config.reflect_streams and self.config.reflector_servers:
|
||||
log.debug("collecting streams to reflect")
|
||||
sd_hashes = await self.storage.get_streams_to_re_reflect()
|
||||
sd_hashes = [sd for sd in sd_hashes if sd in self._sources]
|
||||
batch = []
|
||||
@@ -171,10 +181,14 @@ class StreamManager(SourceManager):
|
||||
stream.fully_reflected.is_set():
|
||||
batch.append(self.reflect_stream(stream))
|
||||
if len(batch) >= self.config.concurrent_reflector_uploads:
|
||||
log.debug("waiting for batch of %s reflecting streams", len(batch))
|
||||
await asyncio.gather(*batch, loop=self.loop)
|
||||
log.debug("done processing %s streams", len(batch))
|
||||
batch = []
|
||||
if batch:
|
||||
log.debug("waiting for batch of %s reflecting streams", len(batch))
|
||||
await asyncio.gather(*batch, loop=self.loop)
|
||||
log.debug("done processing %s streams", len(batch))
|
||||
await asyncio.sleep(300, loop=self.loop)
|
||||
|
||||
async def start(self):
|
||||
|
||||
@@ -311,7 +311,7 @@ class Account:
|
||||
private_key=private_key,
|
||||
public_key=public_key,
|
||||
address_generator=d.get('address_generator', {}),
|
||||
modified_on=d.get('modified_on', time.time()),
|
||||
modified_on=int(d.get('modified_on', time.time())),
|
||||
channel_keys=d.get('certificates', {})
|
||||
)
|
||||
|
||||
@@ -343,7 +343,7 @@ class Account:
|
||||
def merge(self, d: dict):
|
||||
if d.get('modified_on', 0) > self.modified_on:
|
||||
self.name = d['name']
|
||||
self.modified_on = d.get('modified_on', time.time())
|
||||
self.modified_on = int(d.get('modified_on', time.time()))
|
||||
assert self.address_generator.name == d['address_generator']['name']
|
||||
for chain_name in ('change', 'receiving'):
|
||||
if chain_name in d['address_generator']:
|
||||
|
||||
@@ -529,13 +529,14 @@ def _get_spendable_utxos(transaction: sqlite3.Connection, accounts: List, decode
|
||||
|
||||
|
||||
def get_and_reserve_spendable_utxos(transaction: sqlite3.Connection, accounts: List, amount_to_reserve: int, floor: int,
|
||||
fee_per_byte: int, set_reserved: bool, return_insufficient_funds: bool):
|
||||
fee_per_byte: int, set_reserved: bool, return_insufficient_funds: bool,
|
||||
base_multiplier: int = 100):
|
||||
txs = defaultdict(list)
|
||||
decoded_transactions = {}
|
||||
reserved = []
|
||||
|
||||
reserved_dewies = 0
|
||||
multiplier = 10
|
||||
multiplier = base_multiplier
|
||||
gap_count = 0
|
||||
|
||||
while reserved_dewies < amount_to_reserve and gap_count < 5 and floor * multiplier < SQLITE_MAX_INTEGER:
|
||||
@@ -550,7 +551,7 @@ def get_and_reserve_spendable_utxos(transaction: sqlite3.Connection, accounts: L
|
||||
multiplier **= 2
|
||||
else:
|
||||
gap_count = 0
|
||||
multiplier = 10
|
||||
multiplier = base_multiplier
|
||||
|
||||
# reserve the accumulated txos if enough were found
|
||||
if reserved_dewies >= amount_to_reserve:
|
||||
@@ -762,7 +763,7 @@ class Database(SQLiteMixin):
|
||||
# 2. update address histories removing deleted TXs
|
||||
return True
|
||||
|
||||
async def get_spendable_utxos(self, ledger, reserve_amount, accounts: Optional[Iterable], min_amount: int = 100000,
|
||||
async def get_spendable_utxos(self, ledger, reserve_amount, accounts: Optional[Iterable], min_amount: int = 1,
|
||||
fee_per_byte: int = 50, set_reserved: bool = True,
|
||||
return_insufficient_funds: bool = False) -> List:
|
||||
to_spend = await self.db.run(
|
||||
|
||||
+227
-87
@@ -3,7 +3,6 @@ import copy
|
||||
import time
|
||||
import asyncio
|
||||
import logging
|
||||
from io import StringIO
|
||||
from datetime import datetime
|
||||
from functools import partial
|
||||
from operator import itemgetter
|
||||
@@ -133,7 +132,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
self._on_transaction_controller = StreamController()
|
||||
self.on_transaction = self._on_transaction_controller.stream
|
||||
self.on_transaction.listen(
|
||||
lambda e: log.debug(
|
||||
lambda e: log.info(
|
||||
'(%s) on_transaction: address=%s, height=%s, is_verified=%s, tx.id=%s',
|
||||
self.get_id(), e.address, e.tx.height, e.tx.is_verified, e.tx.id
|
||||
)
|
||||
@@ -164,6 +163,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
self._utxo_reservation_lock = asyncio.Lock()
|
||||
self._header_processing_lock = asyncio.Lock()
|
||||
self._address_update_locks: DefaultDict[str, asyncio.Lock] = defaultdict(asyncio.Lock)
|
||||
self._history_lock = asyncio.Lock()
|
||||
|
||||
self.coin_selection_strategy = None
|
||||
self._known_addresses_out_of_sync = set()
|
||||
@@ -244,8 +244,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
def get_address_count(self, **constraints):
|
||||
return self.db.get_address_count(**constraints)
|
||||
|
||||
async def get_spendable_utxos(self, amount: int, funding_accounts: Optional[Iterable['Account']],
|
||||
min_amount=100000):
|
||||
async def get_spendable_utxos(self, amount: int, funding_accounts: Optional[Iterable['Account']], min_amount=1):
|
||||
min_amount = min(amount // 10, min_amount)
|
||||
fee = Output.pay_pubkey_hash(COIN, NULL_HASH32).get_fee(self)
|
||||
selector = CoinSelector(amount, fee)
|
||||
@@ -490,10 +489,10 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
address, remote_status = update
|
||||
self._update_tasks.add(self.update_history(address, remote_status))
|
||||
|
||||
async def update_history(self, address, remote_status, address_manager: AddressManager = None):
|
||||
async def update_history(self, address, remote_status, address_manager: AddressManager = None,
|
||||
reattempt_update: bool = True):
|
||||
async with self._address_update_locks[address]:
|
||||
self._known_addresses_out_of_sync.discard(address)
|
||||
|
||||
local_status, local_history = await self.get_local_status_and_history(address)
|
||||
|
||||
if local_status == remote_status:
|
||||
@@ -503,60 +502,94 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
remote_history = list(map(itemgetter('tx_hash', 'height'), remote_history))
|
||||
we_need = set(remote_history) - set(local_history)
|
||||
if not we_need:
|
||||
remote_missing = set(local_history) - set(remote_history)
|
||||
if remote_missing:
|
||||
log.warning(
|
||||
"%i transactions we have for %s are not in the remote address history",
|
||||
len(remote_missing), address
|
||||
)
|
||||
return True
|
||||
|
||||
cache_tasks: List[asyncio.Task[Transaction]] = []
|
||||
synced_history = StringIO()
|
||||
loop = asyncio.get_running_loop()
|
||||
for i, (txid, remote_height) in enumerate(remote_history):
|
||||
if i < len(local_history) and local_history[i] == (txid, remote_height) and not cache_tasks:
|
||||
synced_history.write(f'{txid}:{remote_height}:')
|
||||
else:
|
||||
check_local = (txid, remote_height) not in we_need
|
||||
cache_tasks.append(loop.create_task(
|
||||
self.cache_transaction(txid, remote_height, check_local=check_local)
|
||||
))
|
||||
|
||||
acquire_lock_tasks = []
|
||||
synced_txs = []
|
||||
for task in cache_tasks:
|
||||
tx = await task
|
||||
to_request = {}
|
||||
pending_synced_history = {}
|
||||
updated_cached_items = {}
|
||||
already_synced = set()
|
||||
|
||||
check_db_for_txos = []
|
||||
for txi in tx.inputs:
|
||||
if txi.txo_ref.txo is not None:
|
||||
continue
|
||||
cache_item = self._tx_cache.get(txi.txo_ref.tx_ref.id)
|
||||
if cache_item is not None:
|
||||
if cache_item.tx is None:
|
||||
await cache_item.has_tx.wait()
|
||||
assert cache_item.tx is not None
|
||||
txi.txo_ref = cache_item.tx.outputs[txi.txo_ref.position].ref
|
||||
else:
|
||||
check_db_for_txos.append(txi.txo_ref.id)
|
||||
already_synced_offset = 0
|
||||
for i, (txid, remote_height) in enumerate(remote_history):
|
||||
if i == already_synced_offset and i < len(local_history) and local_history[i] == (txid, remote_height):
|
||||
pending_synced_history[i] = f'{txid}:{remote_height}:'
|
||||
already_synced.add((txid, remote_height))
|
||||
already_synced_offset += 1
|
||||
continue
|
||||
cache_item = self._tx_cache.get(txid)
|
||||
if cache_item is None:
|
||||
cache_item = TransactionCacheItem()
|
||||
self._tx_cache[txid] = cache_item
|
||||
|
||||
referenced_txos = {} if not check_db_for_txos else {
|
||||
txo.id: txo for txo in await self.db.get_txos(
|
||||
txoid__in=check_db_for_txos, order_by='txo.txoid', no_tx=True
|
||||
)
|
||||
}
|
||||
for txid, remote_height in remote_history[already_synced_offset:]:
|
||||
cache_item = self._tx_cache[txid]
|
||||
acquire_lock_tasks.append(asyncio.create_task(cache_item.lock.acquire()))
|
||||
|
||||
for txi in tx.inputs:
|
||||
if txi.txo_ref.txo is not None:
|
||||
continue
|
||||
referenced_txo = referenced_txos.get(txi.txo_ref.id)
|
||||
if referenced_txo is not None:
|
||||
txi.txo_ref = referenced_txo.ref
|
||||
if acquire_lock_tasks:
|
||||
await asyncio.wait(acquire_lock_tasks)
|
||||
|
||||
synced_history.write(f'{tx.id}:{tx.height}:')
|
||||
tx_indexes = {}
|
||||
|
||||
for i, (txid, remote_height) in enumerate(remote_history):
|
||||
tx_indexes[txid] = i
|
||||
if (txid, remote_height) in already_synced:
|
||||
continue
|
||||
cache_item = self._tx_cache.get(txid)
|
||||
cache_item.pending_verifications += 1
|
||||
updated_cached_items[txid] = cache_item
|
||||
|
||||
assert cache_item is not None, 'cache item is none'
|
||||
assert cache_item.lock.locked(), 'cache lock is not held?'
|
||||
# tx = cache_item.tx
|
||||
# if cache_item.tx is not None and \
|
||||
# cache_item.tx.height >= remote_height and \
|
||||
# (cache_item.tx.is_verified or remote_height < 1):
|
||||
# synced_txs.append(cache_item.tx) # cached tx is already up-to-date
|
||||
# pending_synced_history[i] = f'{tx.id}:{tx.height}:'
|
||||
# continue
|
||||
to_request[i] = (txid, remote_height)
|
||||
|
||||
log.debug(
|
||||
"request %i transactions, %i/%i for %s are already synced", len(to_request), len(synced_txs),
|
||||
len(remote_history), address
|
||||
)
|
||||
requested_txes = await self._request_transaction_batch(to_request, len(remote_history), address)
|
||||
for tx in requested_txes:
|
||||
pending_synced_history[tx_indexes[tx.id]] = f"{tx.id}:{tx.height}:"
|
||||
synced_txs.append(tx)
|
||||
|
||||
assert len(pending_synced_history) == len(remote_history), \
|
||||
f"{len(pending_synced_history)} vs {len(remote_history)}"
|
||||
synced_history = ""
|
||||
for remote_i, i in zip(range(len(remote_history)), sorted(pending_synced_history.keys())):
|
||||
assert i == remote_i, f"{i} vs {remote_i}"
|
||||
txid, height = remote_history[remote_i]
|
||||
if f"{txid}:{height}:" != pending_synced_history[i]:
|
||||
log.warning("history mismatch: %s vs %s", remote_history[remote_i], pending_synced_history[i])
|
||||
synced_history += pending_synced_history[i]
|
||||
|
||||
cache_size = self.config.get("tx_cache_size", 100_000)
|
||||
for txid, cache_item in updated_cached_items.items():
|
||||
cache_item.pending_verifications -= 1
|
||||
if cache_item.pending_verifications < 0:
|
||||
log.warning("config value tx cache size %i needs to be increased", cache_size)
|
||||
cache_item.pending_verifications = 0
|
||||
try:
|
||||
cache_item.lock.release()
|
||||
except RuntimeError:
|
||||
log.warning("lock was already released?")
|
||||
|
||||
await self.db.save_transaction_io_batch(
|
||||
synced_txs, address, self.address_to_hash160(address), synced_history.getvalue()
|
||||
[], address, self.address_to_hash160(address), synced_history
|
||||
)
|
||||
await asyncio.wait([
|
||||
self._on_transaction_controller.add(TransactionEvent(address, tx))
|
||||
for tx in synced_txs
|
||||
])
|
||||
|
||||
if address_manager is None:
|
||||
address_manager = await self.get_address_manager_for_address(address)
|
||||
@@ -564,8 +597,16 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
if address_manager is not None:
|
||||
await address_manager.ensure_address_gap()
|
||||
|
||||
for txid, cache_item in updated_cached_items.items():
|
||||
if self._tx_cache.get(txid) is not cache_item:
|
||||
log.warning("tx cache corrupted while syncing %s, reattempt sync=%s", address, reattempt_update)
|
||||
if reattempt_update:
|
||||
return await self.update_history(address, remote_status, address_manager, False)
|
||||
return False
|
||||
|
||||
local_status, local_history = \
|
||||
await self.get_local_status_and_history(address, synced_history.getvalue())
|
||||
await self.get_local_status_and_history(address, synced_history)
|
||||
|
||||
if local_status != remote_status:
|
||||
if local_history == remote_history:
|
||||
log.warning(
|
||||
@@ -591,6 +632,7 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
self._known_addresses_out_of_sync.add(address)
|
||||
return False
|
||||
else:
|
||||
log.debug("finished syncing transaction history for %s, %i known txs", address, len(local_history))
|
||||
return True
|
||||
|
||||
async def cache_transaction(self, txid, remote_height, check_local=True):
|
||||
@@ -602,41 +644,39 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
(cache_item.tx.is_verified or remote_height < 1):
|
||||
return cache_item.tx # cached tx is already up-to-date
|
||||
|
||||
cache_item.pending_verifications += 1
|
||||
try:
|
||||
cache_item.pending_verifications += 1
|
||||
return await self._update_cache_item(cache_item, txid, remote_height, check_local)
|
||||
async with cache_item.lock:
|
||||
tx = cache_item.tx
|
||||
if tx is None and check_local:
|
||||
# check local db
|
||||
tx = cache_item.tx = await self.db.get_transaction(txid=txid)
|
||||
merkle = None
|
||||
if tx is None:
|
||||
# fetch from network
|
||||
_raw, merkle = await self.network.retriable_call(
|
||||
self.network.get_transaction_and_merkle, txid, remote_height
|
||||
)
|
||||
tx = Transaction(unhexlify(_raw), height=merkle['block_height'])
|
||||
cache_item.tx = tx # make sure it's saved before caching it
|
||||
tx.height = remote_height
|
||||
if merkle and 0 < remote_height < len(self.headers):
|
||||
merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash)
|
||||
header = await self.headers.get(remote_height)
|
||||
tx.position = merkle['pos']
|
||||
tx.is_verified = merkle_root == header['merkle_root']
|
||||
return tx
|
||||
finally:
|
||||
cache_item.pending_verifications -= 1
|
||||
|
||||
async def _update_cache_item(self, cache_item, txid, remote_height, check_local=True):
|
||||
|
||||
async with cache_item.lock:
|
||||
|
||||
tx = cache_item.tx
|
||||
|
||||
if tx is None and check_local:
|
||||
# check local db
|
||||
tx = cache_item.tx = await self.db.get_transaction(txid=txid)
|
||||
|
||||
merkle = None
|
||||
if tx is None:
|
||||
# fetch from network
|
||||
_raw, merkle = await self.network.retriable_call(
|
||||
self.network.get_transaction_and_merkle, txid, remote_height
|
||||
)
|
||||
tx = Transaction(unhexlify(_raw), height=merkle.get('block_height'))
|
||||
cache_item.tx = tx # make sure it's saved before caching it
|
||||
await self.maybe_verify_transaction(tx, remote_height, merkle)
|
||||
return tx
|
||||
|
||||
async def maybe_verify_transaction(self, tx, remote_height, merkle=None):
|
||||
tx.height = remote_height
|
||||
cached = self._tx_cache.get(tx.id)
|
||||
if not cached:
|
||||
# cache txs looked up by transaction_show too
|
||||
cached = TransactionCacheItem()
|
||||
cached.tx = tx
|
||||
self._tx_cache[tx.id] = cached
|
||||
cached.tx = tx
|
||||
if 0 < remote_height < len(self.headers) and cached.pending_verifications <= 1:
|
||||
# can't be tx.pending_verifications == 1 because we have to handle the transaction_show case
|
||||
if not merkle:
|
||||
@@ -646,6 +686,100 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
tx.position = merkle['pos']
|
||||
tx.is_verified = merkle_root == header['merkle_root']
|
||||
|
||||
async def _request_transaction_batch(self, to_request, remote_history_size, address):
|
||||
header_cache = {}
|
||||
batches = [[]]
|
||||
remote_heights = {}
|
||||
synced_txs = []
|
||||
heights_in_batch = 0
|
||||
last_height = 0
|
||||
for idx in sorted(to_request):
|
||||
txid = to_request[idx][0]
|
||||
height = to_request[idx][1]
|
||||
remote_heights[txid] = height
|
||||
if height != last_height:
|
||||
heights_in_batch += 1
|
||||
last_height = height
|
||||
if len(batches[-1]) == 100 or heights_in_batch == 20:
|
||||
batches.append([])
|
||||
heights_in_batch = 1
|
||||
batches[-1].append(txid)
|
||||
if not batches[-1]:
|
||||
batches.pop()
|
||||
|
||||
last_showed_synced_count = 0
|
||||
|
||||
async def _single_batch(batch):
|
||||
this_batch_synced = []
|
||||
batch_result = await self.network.retriable_call(self.network.get_transaction_batch, batch)
|
||||
for txid, (raw, merkle) in batch_result.items():
|
||||
remote_height = remote_heights[txid]
|
||||
merkle_height = merkle['block_height']
|
||||
cache_item = self._tx_cache.get(txid)
|
||||
if cache_item is None:
|
||||
cache_item = TransactionCacheItem()
|
||||
self._tx_cache[txid] = cache_item
|
||||
tx = cache_item.tx or Transaction(unhexlify(raw), height=remote_height)
|
||||
tx.height = remote_height
|
||||
cache_item.tx = tx
|
||||
if 'merkle' in merkle and remote_heights[txid] > 0:
|
||||
merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash)
|
||||
try:
|
||||
header = header_cache.get(remote_heights[txid]) or (await self.headers.get(merkle_height))
|
||||
except IndexError:
|
||||
log.warning("failed to verify %s at height %i", tx.id, merkle_height)
|
||||
else:
|
||||
header_cache[remote_heights[txid]] = header
|
||||
tx.position = merkle['pos']
|
||||
tx.is_verified = merkle_root == header['merkle_root']
|
||||
check_db_for_txos = []
|
||||
|
||||
for txi in tx.inputs:
|
||||
if txi.txo_ref.txo is not None:
|
||||
continue
|
||||
cache_item = self._tx_cache.get(txi.txo_ref.tx_ref.id)
|
||||
if cache_item is not None:
|
||||
if cache_item.tx is not None:
|
||||
txi.txo_ref = cache_item.tx.outputs[txi.txo_ref.position].ref
|
||||
else:
|
||||
check_db_for_txos.append(txi.txo_ref.id)
|
||||
|
||||
referenced_txos = {} if not check_db_for_txos else {
|
||||
txo.id: txo for txo in await self.db.get_txos(
|
||||
txoid__in=check_db_for_txos, order_by='txo.txoid', no_tx=True
|
||||
)
|
||||
}
|
||||
|
||||
for txi in tx.inputs:
|
||||
if txi.txo_ref.txo is not None:
|
||||
continue
|
||||
referenced_txo = referenced_txos.get(txi.txo_ref.id)
|
||||
if referenced_txo is not None:
|
||||
txi.txo_ref = referenced_txo.ref
|
||||
continue
|
||||
cache_item = self._tx_cache.get(txi.txo_ref.id)
|
||||
if cache_item is None:
|
||||
cache_item = self._tx_cache[txi.txo_ref.id] = TransactionCacheItem()
|
||||
if cache_item.tx is not None:
|
||||
txi.txo_ref = cache_item.tx.ref
|
||||
|
||||
synced_txs.append(tx)
|
||||
this_batch_synced.append(tx)
|
||||
await self.db.save_transaction_io_batch(
|
||||
this_batch_synced, address, self.address_to_hash160(address), ""
|
||||
)
|
||||
await asyncio.wait([
|
||||
self._on_transaction_controller.add(TransactionEvent(address, tx))
|
||||
for tx in this_batch_synced
|
||||
])
|
||||
nonlocal last_showed_synced_count
|
||||
if last_showed_synced_count + 100 < len(synced_txs):
|
||||
log.info("synced %i/%i transactions for %s", len(synced_txs), remote_history_size, address)
|
||||
last_showed_synced_count = len(synced_txs)
|
||||
for batch in batches:
|
||||
await _single_batch(batch)
|
||||
return synced_txs
|
||||
|
||||
async def get_address_manager_for_address(self, address) -> Optional[AddressManager]:
|
||||
details = await self.db.get_address(address=address)
|
||||
for account in self.accounts:
|
||||
@@ -682,20 +816,26 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
address_record['address']
|
||||
)) for address_record in records
|
||||
], timeout=1)
|
||||
if pending:
|
||||
records = await self.db.get_addresses(address__in=addresses)
|
||||
for record in records:
|
||||
found = False
|
||||
local_history = (await self.get_local_status_and_history(
|
||||
record['address'], history=record['history']
|
||||
))[1] if record['history'] else []
|
||||
for txid, local_height in local_history:
|
||||
if txid == tx.id and local_height >= height:
|
||||
found = True
|
||||
if not found:
|
||||
log.debug("timeout: %s, %s, %s", record['history'], addresses, tx.id)
|
||||
if not pending:
|
||||
return True
|
||||
records = await self.db.get_addresses(address__in=addresses)
|
||||
for record in records:
|
||||
local_history = (await self.get_local_status_and_history(
|
||||
record['address'], history=record['history']
|
||||
))[1] if record['history'] else []
|
||||
for txid, local_height in local_history:
|
||||
if txid == tx.id:
|
||||
if local_height >= height:
|
||||
return True
|
||||
log.warning(
|
||||
"local history has higher height than remote for %s (%i vs %i)", txid,
|
||||
local_height, height
|
||||
)
|
||||
return False
|
||||
return True
|
||||
log.warning(
|
||||
"local history does not contain %s, requested height %i", tx.id, height
|
||||
)
|
||||
return False
|
||||
|
||||
async def _inflate_outputs(
|
||||
self, query, accounts,
|
||||
|
||||
@@ -255,6 +255,10 @@ class Network:
|
||||
restricted = known_height in (None, -1, 0) or 0 > known_height > self.remote_height - 10
|
||||
return self.rpc('blockchain.transaction.get', [tx_hash], restricted)
|
||||
|
||||
def get_transaction_batch(self, txids):
|
||||
# use any server if its old, otherwise restrict to who gave us the history
|
||||
return self.rpc('blockchain.transaction.get_batch', txids, True)
|
||||
|
||||
def get_transaction_and_merkle(self, tx_hash, known_height=None):
|
||||
# use any server if its old, otherwise restrict to who gave us the history
|
||||
restricted = known_height in (None, -1, 0) or 0 > known_height > self.remote_height - 10
|
||||
|
||||
@@ -251,7 +251,7 @@ class BlockProcessor:
|
||||
async def get_raw_blocks(last_height, hex_hashes):
|
||||
heights = range(last_height, last_height - len(hex_hashes), -1)
|
||||
try:
|
||||
blocks = [self.db.read_raw_block(height) for height in heights]
|
||||
blocks = [await self.db.read_raw_block(height) for height in heights]
|
||||
self.logger.info(f'read {len(blocks)} blocks from disk')
|
||||
return blocks
|
||||
except FileNotFoundError:
|
||||
@@ -351,11 +351,9 @@ class BlockProcessor:
|
||||
# performed on the DB.
|
||||
if self._caught_up_event.is_set():
|
||||
await self.flush(True)
|
||||
elif time.time() > self.next_cache_check:
|
||||
flush_arg = self.check_cache_size()
|
||||
if flush_arg is not None:
|
||||
await self.flush(flush_arg)
|
||||
self.next_cache_check = time.time() + 30
|
||||
elif time.perf_counter() > self.next_cache_check:
|
||||
await self.flush(True)
|
||||
self.next_cache_check = time.perf_counter() + 30
|
||||
|
||||
def check_cache_size(self):
|
||||
"""Flush a cache if it gets too big."""
|
||||
|
||||
+117
-90
@@ -32,6 +32,9 @@ from lbry.wallet.server.history import History
|
||||
|
||||
|
||||
UTXO = namedtuple("UTXO", "tx_num tx_pos tx_hash height value")
|
||||
HEADER_PREFIX = b'H'
|
||||
TX_COUNT_PREFIX = b'T'
|
||||
TX_HASH_PREFIX = b'X'
|
||||
|
||||
|
||||
@attr.s(slots=True)
|
||||
@@ -63,15 +66,7 @@ class LevelDB:
|
||||
self.logger = util.class_logger(__name__, self.__class__.__name__)
|
||||
self.env = env
|
||||
self.coin = env.coin
|
||||
self.executor = ThreadPoolExecutor(max(1, os.cpu_count() - 1))
|
||||
|
||||
# Setup block header size handlers
|
||||
if self.coin.STATIC_BLOCK_HEADERS:
|
||||
self.header_offset = self.coin.static_header_offset
|
||||
self.header_len = self.coin.static_header_len
|
||||
else:
|
||||
self.header_offset = self.dynamic_header_offset
|
||||
self.header_len = self.dynamic_header_len
|
||||
self.executor = None
|
||||
|
||||
self.logger.info(f'switching current directory to {env.db_dir}')
|
||||
|
||||
@@ -87,51 +82,71 @@ class LevelDB:
|
||||
self.merkle = Merkle()
|
||||
self.header_mc = MerkleCache(self.merkle, self.fs_block_hashes)
|
||||
|
||||
path = partial(os.path.join, self.env.db_dir)
|
||||
self.headers_file = util.LogicalFile(path('meta/headers'), 2, 16000000)
|
||||
self.tx_counts_file = util.LogicalFile(path('meta/txcounts'), 2, 2000000)
|
||||
self.hashes_file = util.LogicalFile(path('meta/hashes'), 4, 16000000)
|
||||
if not self.coin.STATIC_BLOCK_HEADERS:
|
||||
self.headers_offsets_file = util.LogicalFile(
|
||||
path('meta/headers_offsets'), 2, 16000000)
|
||||
self.hashes_db = None
|
||||
self.headers_db = None
|
||||
self.tx_count_db = None
|
||||
|
||||
async def _read_tx_counts(self):
|
||||
if self.tx_counts is not None:
|
||||
return
|
||||
# tx_counts[N] has the cumulative number of txs at the end of
|
||||
# height N. So tx_counts[0] is 1 - the genesis coinbase
|
||||
size = (self.db_height + 1) * 4
|
||||
tx_counts = self.tx_counts_file.read(0, size)
|
||||
assert len(tx_counts) == size
|
||||
|
||||
def get_counts():
|
||||
return tuple(
|
||||
util.unpack_be_uint64(tx_count)
|
||||
for tx_count in self.tx_count_db.iterator(prefix=TX_COUNT_PREFIX, include_key=False)
|
||||
)
|
||||
|
||||
tx_counts = await asyncio.get_event_loop().run_in_executor(self.executor, get_counts)
|
||||
assert len(tx_counts) == self.db_height + 1, f"{len(tx_counts)} vs {self.db_height + 1}"
|
||||
self.tx_counts = array.array('I', tx_counts)
|
||||
|
||||
if self.tx_counts:
|
||||
assert self.db_tx_count == self.tx_counts[-1]
|
||||
assert self.db_tx_count == self.tx_counts[-1], \
|
||||
f"{self.db_tx_count} vs {self.tx_counts[-1]} ({len(self.tx_counts)} counts)"
|
||||
else:
|
||||
assert self.db_tx_count == 0
|
||||
|
||||
async def _open_dbs(self, for_sync, compacting):
|
||||
assert self.utxo_db is None
|
||||
|
||||
# First UTXO DB
|
||||
self.utxo_db = self.db_class('utxo', for_sync)
|
||||
if self.utxo_db.is_new:
|
||||
self.logger.info('created new database')
|
||||
self.logger.info('creating metadata directory')
|
||||
os.mkdir(os.path.join(self.env.db_dir, 'meta'))
|
||||
coin_path = os.path.join(self.env.db_dir, 'meta', 'COIN')
|
||||
if self.executor is None:
|
||||
self.executor = ThreadPoolExecutor(max(1, os.cpu_count() - 1))
|
||||
coin_path = os.path.join(self.env.db_dir, 'COIN')
|
||||
if not os.path.isfile(coin_path):
|
||||
with util.open_file(coin_path, create=True) as f:
|
||||
f.write(f'ElectrumX databases and metadata for '
|
||||
f'{self.coin.NAME} {self.coin.NET}'.encode())
|
||||
if not self.coin.STATIC_BLOCK_HEADERS:
|
||||
self.headers_offsets_file.write(0, bytes(8))
|
||||
else:
|
||||
self.logger.info(f'opened UTXO DB (for sync: {for_sync})')
|
||||
|
||||
assert self.headers_db is None
|
||||
self.headers_db = self.db_class('headers', for_sync)
|
||||
if self.headers_db.is_new:
|
||||
self.logger.info('created new headers db')
|
||||
self.logger.info(f'opened headers DB (for sync: {for_sync})')
|
||||
|
||||
assert self.tx_count_db is None
|
||||
self.tx_count_db = self.db_class('tx_count', for_sync)
|
||||
if self.tx_count_db.is_new:
|
||||
self.logger.info('created new tx count db')
|
||||
self.logger.info(f'opened tx count DB (for sync: {for_sync})')
|
||||
|
||||
assert self.hashes_db is None
|
||||
self.hashes_db = self.db_class('hashes', for_sync)
|
||||
if self.hashes_db.is_new:
|
||||
self.logger.info('created new tx hashes db')
|
||||
self.logger.info(f'opened tx hashes DB (for sync: {for_sync})')
|
||||
|
||||
assert self.utxo_db is None
|
||||
# First UTXO DB
|
||||
self.utxo_db = self.db_class('utxo', for_sync)
|
||||
if self.utxo_db.is_new:
|
||||
self.logger.info('created new utxo db')
|
||||
self.logger.info(f'opened utxo db (for sync: {for_sync})')
|
||||
self.read_utxo_state()
|
||||
|
||||
# Then history DB
|
||||
self.utxo_flush_count = self.history.open_db(self.db_class, for_sync,
|
||||
self.utxo_flush_count,
|
||||
compacting)
|
||||
self.utxo_flush_count = self.history.open_db(
|
||||
self.db_class, for_sync, self.utxo_flush_count, compacting
|
||||
)
|
||||
self.clear_excess_undo_info()
|
||||
|
||||
# Read TX counts (requires meta directory)
|
||||
@@ -140,7 +155,11 @@ class LevelDB:
|
||||
def close(self):
|
||||
self.utxo_db.close()
|
||||
self.history.close_db()
|
||||
self.headers_db.close()
|
||||
self.tx_count_db.close()
|
||||
self.hashes_db.close()
|
||||
self.executor.shutdown(wait=True)
|
||||
self.executor = None
|
||||
|
||||
async def open_for_compacting(self):
|
||||
await self._open_dbs(True, True)
|
||||
@@ -152,18 +171,31 @@ class LevelDB:
|
||||
synchronization. When serving clients we want the open files for
|
||||
serving network connections.
|
||||
"""
|
||||
self.logger.info("opened for sync")
|
||||
await self._open_dbs(True, False)
|
||||
|
||||
async def open_for_serving(self):
|
||||
"""Open the databases for serving. If they are already open they are
|
||||
closed first.
|
||||
"""
|
||||
self.logger.info('closing DBs to re-open for serving')
|
||||
if self.utxo_db:
|
||||
self.logger.info('closing DBs to re-open for serving')
|
||||
self.utxo_db.close()
|
||||
self.history.close_db()
|
||||
self.utxo_db = None
|
||||
if self.headers_db:
|
||||
self.headers_db.close()
|
||||
self.headers_db = None
|
||||
if self.tx_count_db:
|
||||
self.tx_count_db.close()
|
||||
self.tx_count_db = None
|
||||
if self.hashes_db:
|
||||
self.hashes_db.close()
|
||||
self.hashes_db = None
|
||||
|
||||
await self._open_dbs(False, False)
|
||||
self.logger.info("opened for serving")
|
||||
|
||||
# Header merkle cache
|
||||
|
||||
@@ -248,31 +280,30 @@ class LevelDB:
|
||||
assert flush_data.tx_count == (self.tx_counts[-1] if self.tx_counts
|
||||
else 0)
|
||||
assert len(self.tx_counts) == flush_data.height + 1
|
||||
hashes = b''.join(flush_data.block_tx_hashes)
|
||||
flush_data.block_tx_hashes.clear()
|
||||
assert len(hashes) % 32 == 0
|
||||
assert len(hashes) // 32 == flush_data.tx_count - prior_tx_count
|
||||
assert len(b''.join(flush_data.block_tx_hashes)) // 32 == flush_data.tx_count - prior_tx_count
|
||||
|
||||
# Write the headers, tx counts, and tx hashes
|
||||
start_time = time.time()
|
||||
start_time = time.perf_counter()
|
||||
height_start = self.fs_height + 1
|
||||
offset = self.header_offset(height_start)
|
||||
self.headers_file.write(offset, b''.join(flush_data.headers))
|
||||
self.fs_update_header_offsets(offset, height_start, flush_data.headers)
|
||||
flush_data.headers.clear()
|
||||
tx_num = prior_tx_count
|
||||
|
||||
offset = height_start * self.tx_counts.itemsize
|
||||
self.tx_counts_file.write(offset,
|
||||
self.tx_counts[height_start:].tobytes())
|
||||
offset = prior_tx_count * 32
|
||||
self.hashes_file.write(offset, hashes)
|
||||
for header, tx_hashes in zip(flush_data.headers, flush_data.block_tx_hashes):
|
||||
tx_count = self.tx_counts[height_start]
|
||||
self.headers_db.put(HEADER_PREFIX + util.pack_be_uint64(height_start), header)
|
||||
self.tx_count_db.put(TX_COUNT_PREFIX + util.pack_be_uint64(height_start), util.pack_be_uint64(tx_count))
|
||||
height_start += 1
|
||||
offset = 0
|
||||
while offset < len(tx_hashes):
|
||||
self.hashes_db.put(TX_HASH_PREFIX + util.pack_be_uint64(tx_num), tx_hashes[offset:offset+32])
|
||||
tx_num += 1
|
||||
offset += 32
|
||||
|
||||
flush_data.block_tx_hashes.clear()
|
||||
self.fs_height = flush_data.height
|
||||
self.fs_tx_count = flush_data.tx_count
|
||||
|
||||
if self.utxo_db.for_sync:
|
||||
elapsed = time.time() - start_time
|
||||
self.logger.info(f'flushed filesystem data in {elapsed:.2f}s')
|
||||
flush_data.headers.clear()
|
||||
elapsed = time.perf_counter() - start_time
|
||||
self.logger.info(f'flushed filesystem data in {elapsed:.2f}s')
|
||||
|
||||
def flush_history(self):
|
||||
self.history.flush()
|
||||
@@ -350,28 +381,6 @@ class LevelDB:
|
||||
f'{elapsed:.1f}s. Height {flush_data.height:,d} '
|
||||
f'txs: {flush_data.tx_count:,d} ({tx_delta:+,d})')
|
||||
|
||||
def fs_update_header_offsets(self, offset_start, height_start, headers):
|
||||
if self.coin.STATIC_BLOCK_HEADERS:
|
||||
return
|
||||
offset = offset_start
|
||||
offsets = []
|
||||
for h in headers:
|
||||
offset += len(h)
|
||||
offsets.append(pack("<Q", offset))
|
||||
# For each header we get the offset of the next header, hence we
|
||||
# start writing from the next height
|
||||
pos = (height_start + 1) * 8
|
||||
self.headers_offsets_file.write(pos, b''.join(offsets))
|
||||
|
||||
def dynamic_header_offset(self, height):
|
||||
assert not self.coin.STATIC_BLOCK_HEADERS
|
||||
offset, = unpack('<Q', self.headers_offsets_file.read(height * 8, 8))
|
||||
return offset
|
||||
|
||||
def dynamic_header_len(self, height):
|
||||
return self.dynamic_header_offset(height + 1)\
|
||||
- self.dynamic_header_offset(height)
|
||||
|
||||
def backup_fs(self, height, tx_count):
|
||||
"""Back up during a reorg. This just updates our pointers."""
|
||||
self.fs_height = height
|
||||
@@ -403,9 +412,13 @@ class LevelDB:
|
||||
# Read some from disk
|
||||
disk_count = max(0, min(count, self.db_height + 1 - start_height))
|
||||
if disk_count:
|
||||
offset = self.header_offset(start_height)
|
||||
size = self.header_offset(start_height + disk_count) - offset
|
||||
return self.headers_file.read(offset, size), disk_count
|
||||
return b''.join(
|
||||
self.headers_db.iterator(
|
||||
start=HEADER_PREFIX + util.pack_be_uint64(start_height),
|
||||
stop=HEADER_PREFIX + util.pack_be_uint64(start_height + disk_count),
|
||||
include_key=False
|
||||
)
|
||||
), disk_count
|
||||
return b'', 0
|
||||
|
||||
return await asyncio.get_event_loop().run_in_executor(self.executor, read_headers)
|
||||
@@ -418,7 +431,7 @@ class LevelDB:
|
||||
if tx_height > self.db_height:
|
||||
tx_hash = None
|
||||
else:
|
||||
tx_hash = self.hashes_file.read(tx_num * 32, 32)
|
||||
tx_hash = self.hashes_db.get(TX_HASH_PREFIX + util.pack_be_uint64(tx_num))
|
||||
return tx_hash, tx_height
|
||||
|
||||
async def fs_block_hashes(self, height, count):
|
||||
@@ -428,9 +441,8 @@ class LevelDB:
|
||||
offset = 0
|
||||
headers = []
|
||||
for n in range(count):
|
||||
hlen = self.header_len(height + n)
|
||||
headers.append(headers_concat[offset:offset + hlen])
|
||||
offset += hlen
|
||||
headers.append(headers_concat[offset:offset + self.coin.BASIC_HEADER_SIZE])
|
||||
offset += self.coin.BASIC_HEADER_SIZE
|
||||
|
||||
return [self.coin.header_hash(header) for header in headers]
|
||||
|
||||
@@ -442,9 +454,20 @@ class LevelDB:
|
||||
limit to None to get them all.
|
||||
"""
|
||||
def read_history():
|
||||
tx_nums = list(self.history.get_txnums(hashX, limit))
|
||||
fs_tx_hash = self.fs_tx_hash
|
||||
return [fs_tx_hash(tx_num) for tx_num in tx_nums]
|
||||
hashx_history = []
|
||||
for key, hist in self.history.db.iterator(prefix=hashX):
|
||||
a = array.array('I')
|
||||
a.frombytes(hist)
|
||||
for tx_num in a:
|
||||
tx_height = bisect_right(self.tx_counts, tx_num)
|
||||
if tx_height > self.db_height:
|
||||
tx_hash = None
|
||||
else:
|
||||
tx_hash = self.hashes_db.get(TX_HASH_PREFIX + util.pack_be_uint64(tx_num))
|
||||
hashx_history.append((tx_hash, tx_height))
|
||||
if limit and len(hashx_history) >= limit:
|
||||
return hashx_history
|
||||
return hashx_history
|
||||
|
||||
while True:
|
||||
history = await asyncio.get_event_loop().run_in_executor(self.executor, read_history)
|
||||
@@ -474,16 +497,20 @@ class LevelDB:
|
||||
batch_put(self.undo_key(height), b''.join(undo_info))
|
||||
|
||||
def raw_block_prefix(self):
|
||||
return 'meta/block'
|
||||
return 'block'
|
||||
|
||||
def raw_block_path(self, height):
|
||||
return os.path.join(self.env.db_dir, f'{self.raw_block_prefix()}{height:d}')
|
||||
|
||||
def read_raw_block(self, height):
|
||||
async def read_raw_block(self, height):
|
||||
"""Returns a raw block read from disk. Raises FileNotFoundError
|
||||
if the block isn't on-disk."""
|
||||
with util.open_file(self.raw_block_path(height)) as f:
|
||||
return f.read(-1)
|
||||
|
||||
def read():
|
||||
with util.open_file(self.raw_block_path(height)) as f:
|
||||
return f.read(-1)
|
||||
|
||||
return await asyncio.get_event_loop().run_in_executor(self.executor, read)
|
||||
|
||||
def write_raw_block(self, block, height):
|
||||
"""Write a raw block to disk."""
|
||||
|
||||
@@ -13,6 +13,7 @@ import time
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import defaultdict
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
from prometheus_client import Histogram
|
||||
|
||||
import attr
|
||||
|
||||
@@ -79,6 +80,16 @@ class MemPoolAPI(ABC):
|
||||
daemon's height at the time the mempool was obtained."""
|
||||
|
||||
|
||||
NAMESPACE = "wallet_server"
|
||||
HISTOGRAM_BUCKETS = (
|
||||
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
|
||||
)
|
||||
mempool_process_time_metric = Histogram(
|
||||
"processed_mempool", "Time to process mempool and notify touched addresses",
|
||||
namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS
|
||||
)
|
||||
|
||||
|
||||
class MemPool:
|
||||
"""Representation of the daemon's mempool.
|
||||
|
||||
@@ -93,7 +104,7 @@ class MemPool:
|
||||
hashXs: hashX -> set of all hashes of txs touching the hashX
|
||||
"""
|
||||
|
||||
def __init__(self, coin, api, refresh_secs=5.0, log_status_secs=120.0):
|
||||
def __init__(self, coin, api, refresh_secs=1.0, log_status_secs=120.0):
|
||||
assert isinstance(api, MemPoolAPI)
|
||||
self.coin = coin
|
||||
self.api = api
|
||||
@@ -107,6 +118,7 @@ class MemPool:
|
||||
self.lock = asyncio.Lock()
|
||||
self.wakeup = asyncio.Event()
|
||||
self.executor = ThreadPoolExecutor(max(os.cpu_count() - 1, 1))
|
||||
self.mempool_process_time_metric = mempool_process_time_metric
|
||||
|
||||
async def _logging(self, synchronized_event):
|
||||
"""Print regular logs of mempool stats."""
|
||||
@@ -207,6 +219,7 @@ class MemPool:
|
||||
async def _refresh_hashes(self, synchronized_event):
|
||||
"""Refresh our view of the daemon's mempool."""
|
||||
while True:
|
||||
start = time.perf_counter()
|
||||
height = self.api.cached_height()
|
||||
hex_hashes = await self.api.mempool_hashes()
|
||||
if height != await self.api.height():
|
||||
@@ -217,6 +230,8 @@ class MemPool:
|
||||
synchronized_event.set()
|
||||
synchronized_event.clear()
|
||||
await self.api.on_mempool(touched, height)
|
||||
duration = time.perf_counter() - start
|
||||
self.mempool_process_time_metric.observe(duration)
|
||||
try:
|
||||
# we wait up to `refresh_secs` but go early if a broadcast happens (which triggers wakeup event)
|
||||
await asyncio.wait_for(self.wakeup.wait(), timeout=self.refresh_secs)
|
||||
@@ -374,3 +389,17 @@ class MemPool:
|
||||
if hX == hashX:
|
||||
utxos.append(UTXO(-1, pos, tx_hash, 0, value))
|
||||
return utxos
|
||||
|
||||
def get_mempool_height(self, tx_hash):
|
||||
# Height Progression
|
||||
# -2: not broadcast
|
||||
# -1: in mempool but has unconfirmed inputs
|
||||
# 0: in mempool and all inputs confirmed
|
||||
# +num: confirmed in a specific block (height)
|
||||
if tx_hash not in self.txs:
|
||||
return -2
|
||||
tx = self.txs[tx_hash]
|
||||
unspent_inputs = sum(1 if hash in self.txs else 0 for hash, idx in tx.prevouts)
|
||||
if unspent_inputs:
|
||||
return -1
|
||||
return 0
|
||||
@@ -123,6 +123,7 @@ HISTOGRAM_BUCKETS = (
|
||||
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
|
||||
)
|
||||
|
||||
|
||||
class SessionManager:
|
||||
"""Holds global state about all sessions."""
|
||||
|
||||
@@ -160,6 +161,18 @@ class SessionManager:
|
||||
"clients", "Number of connections received per client version",
|
||||
namespace=NAMESPACE, labelnames=("version",)
|
||||
)
|
||||
address_history_metric = Histogram(
|
||||
"address_history", "Time to fetch an address history",
|
||||
namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS
|
||||
)
|
||||
notifications_in_flight_metric = Gauge(
|
||||
"notifications_in_flight", "Count of notifications in flight",
|
||||
namespace=NAMESPACE
|
||||
)
|
||||
notifications_sent_metric = Histogram(
|
||||
"notifications_sent", "Time to send an address notification",
|
||||
namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS
|
||||
)
|
||||
|
||||
def __init__(self, env: 'Env', db: LBRYLevelDB, bp: LBRYBlockProcessor, daemon: 'Daemon', mempool: 'MemPool',
|
||||
shutdown_event: asyncio.Event):
|
||||
@@ -602,7 +615,6 @@ class SessionManager:
|
||||
|
||||
async def broadcast_transaction(self, raw_tx):
|
||||
hex_hash = await self.daemon.broadcast_transaction(raw_tx)
|
||||
self.mempool.wakeup.set()
|
||||
self.txs_sent += 1
|
||||
return hex_hash
|
||||
|
||||
@@ -923,36 +935,40 @@ class LBRYElectrumX(SessionBase):
|
||||
args = (await self.subscribe_headers_result(), )
|
||||
if not (await self.send_notification('blockchain.headers.subscribe', args)):
|
||||
return
|
||||
|
||||
async def send_history_notification(alias, hashX):
|
||||
start = time.perf_counter()
|
||||
if len(alias) == 64:
|
||||
method = 'blockchain.scripthash.subscribe'
|
||||
else:
|
||||
method = 'blockchain.address.subscribe'
|
||||
try:
|
||||
self.session_mgr.notifications_in_flight_metric.inc()
|
||||
status = await self.address_status(hashX)
|
||||
self.session_mgr.address_history_metric.observe(time.perf_counter() - start)
|
||||
start = time.perf_counter()
|
||||
await self.send_notification(method, (alias, status))
|
||||
self.session_mgr.notifications_sent_metric.observe(time.perf_counter() - start)
|
||||
finally:
|
||||
self.session_mgr.notifications_in_flight_metric.dec()
|
||||
|
||||
touched = touched.intersection(self.hashX_subs)
|
||||
if touched or (height_changed and self.mempool_statuses):
|
||||
changed = {}
|
||||
|
||||
notified = set()
|
||||
mempool_addrs = tuple(self.mempool_statuses.keys())
|
||||
for hashX in touched:
|
||||
alias = self.hashX_subs[hashX]
|
||||
status = await self.address_status(hashX)
|
||||
changed[alias] = status
|
||||
|
||||
# Check mempool hashXs - the status is a function of the
|
||||
# confirmed state of other transactions. Note: we cannot
|
||||
# iterate over mempool_statuses as it changes size.
|
||||
for hashX in tuple(self.mempool_statuses):
|
||||
# Items can be evicted whilst await-ing status; False
|
||||
# ensures such hashXs are notified
|
||||
old_status = self.mempool_statuses.get(hashX, False)
|
||||
status = await self.address_status(hashX)
|
||||
if status != old_status:
|
||||
asyncio.create_task(send_history_notification(alias, hashX))
|
||||
notified.add(hashX)
|
||||
for hashX in mempool_addrs:
|
||||
if hashX not in notified:
|
||||
alias = self.hashX_subs[hashX]
|
||||
changed[alias] = status
|
||||
asyncio.create_task(send_history_notification(alias, hashX))
|
||||
notified.add(hashX)
|
||||
|
||||
for alias, status in changed.items():
|
||||
if len(alias) == 64:
|
||||
method = 'blockchain.scripthash.subscribe'
|
||||
else:
|
||||
method = 'blockchain.address.subscribe'
|
||||
asyncio.create_task(self.send_notification(method, (alias, status)))
|
||||
if changed:
|
||||
es = '' if len(changed) == 1 else 'es'
|
||||
self.logger.info(f'notified of {len(changed):,d} address{es}')
|
||||
if touched:
|
||||
es = '' if len(touched) == 1 else 'es'
|
||||
self.logger.info(f'notified {len(notified)} mempool/{len(touched):,d} touched address{es}')
|
||||
|
||||
def get_metrics_or_placeholder_for_api(self, query_name):
|
||||
""" Do not hold on to a reference to the metrics
|
||||
@@ -1181,7 +1197,6 @@ class LBRYElectrumX(SessionBase):
|
||||
self.mempool_statuses[hashX] = status
|
||||
else:
|
||||
self.mempool_statuses.pop(hashX, None)
|
||||
|
||||
return status
|
||||
|
||||
async def hashX_listunspent(self, hashX):
|
||||
@@ -1497,6 +1512,7 @@ class LBRYElectrumX(SessionBase):
|
||||
try:
|
||||
hex_hash = await self.session_mgr.broadcast_transaction(raw_tx)
|
||||
self.txs_sent += 1
|
||||
self.mempool.wakeup.set()
|
||||
self.logger.info(f'sent tx: {hex_hash}')
|
||||
return hex_hash
|
||||
except DaemonError as e:
|
||||
@@ -1513,7 +1529,7 @@ class LBRYElectrumX(SessionBase):
|
||||
block_hash = tx_info.get('blockhash')
|
||||
if not block_hash:
|
||||
return raw_tx, {'block_height': -1}
|
||||
merkle_height = (await self.daemon_request('deserialised_block', block_hash))['height']
|
||||
merkle_height = (await self.daemon.deserialised_block(block_hash))['height']
|
||||
merkle = await self.transaction_merkle(tx_hash, merkle_height)
|
||||
return raw_tx, merkle
|
||||
|
||||
@@ -1523,35 +1539,25 @@ class LBRYElectrumX(SessionBase):
|
||||
for tx_hash in tx_hashes:
|
||||
assert_tx_hash(tx_hash)
|
||||
batch_result = {}
|
||||
height = None
|
||||
block_hash = None
|
||||
block = None
|
||||
for tx_hash in tx_hashes:
|
||||
tx_info = await self.daemon_request('getrawtransaction', tx_hash, True)
|
||||
raw_tx = tx_info['hex']
|
||||
if height is None:
|
||||
if 'blockhash' in tx_info:
|
||||
block_hash = tx_info['blockhash']
|
||||
block = await self.daemon_request('deserialised_block', block_hash)
|
||||
height = block['height']
|
||||
else:
|
||||
height = -1
|
||||
if block_hash != tx_info.get('blockhash'):
|
||||
raise RPCError(BAD_REQUEST, f'request contains a mix of transaction heights')
|
||||
block_hash = tx_info.get('blockhash')
|
||||
merkle = {}
|
||||
if block_hash:
|
||||
block = await self.daemon.deserialised_block(block_hash)
|
||||
height = block['height']
|
||||
try:
|
||||
pos = block['tx'].index(tx_hash)
|
||||
except ValueError:
|
||||
raise RPCError(BAD_REQUEST, f'tx hash {tx_hash} not in '
|
||||
f'block {block_hash} at height {height:,d}')
|
||||
merkle["merkle"] = self._get_merkle_branch(block['tx'], pos)
|
||||
merkle["pos"] = pos
|
||||
else:
|
||||
if not block_hash:
|
||||
merkle = {'block_height': -1}
|
||||
else:
|
||||
try:
|
||||
pos = block['tx'].index(tx_hash)
|
||||
except ValueError:
|
||||
raise RPCError(BAD_REQUEST, f'tx hash {tx_hash} not in '
|
||||
f'block {block_hash} at height {height:,d}')
|
||||
merkle = {
|
||||
"merkle": self._get_merkle_branch(block['tx'], pos),
|
||||
"pos": pos
|
||||
}
|
||||
batch_result[tx_hash] = [raw_tx, merkle]
|
||||
height = -1
|
||||
merkle['block_height'] = height
|
||||
batch_result[tx_hash] = [raw_tx, merkle]
|
||||
return batch_result
|
||||
|
||||
async def transaction_get(self, tx_hash, verbose=False):
|
||||
@@ -1576,7 +1582,7 @@ class LBRYElectrumX(SessionBase):
|
||||
height = non_negative_integer(height)
|
||||
hex_hashes = await self.daemon_request('block_hex_hashes', height, 1)
|
||||
block_hash = hex_hashes[0]
|
||||
block = await self.daemon_request('deserialised_block', block_hash)
|
||||
block = await self.daemon.deserialised_block(block_hash)
|
||||
return block_hash, block['tx']
|
||||
|
||||
def _get_merkle_branch(self, tx_hashes, tx_pos):
|
||||
|
||||
@@ -333,11 +333,13 @@ unpack_le_uint64_from = struct_le_Q.unpack_from
|
||||
unpack_be_uint16_from = struct_be_H.unpack_from
|
||||
unpack_be_uint32_from = struct_be_I.unpack_from
|
||||
|
||||
unpack_be_uint64 = lambda x: int.from_bytes(x, byteorder='big')
|
||||
|
||||
pack_le_int32 = struct_le_i.pack
|
||||
pack_le_int64 = struct_le_q.pack
|
||||
pack_le_uint16 = struct_le_H.pack
|
||||
pack_le_uint32 = struct_le_I.pack
|
||||
pack_le_uint64 = struct_le_Q.pack
|
||||
pack_be_uint64 = lambda x: x.to_bytes(8, byteorder='big')
|
||||
pack_be_uint16 = struct_be_H.pack
|
||||
pack_be_uint32 = struct_be_I.pack
|
||||
pack_byte = structB.pack
|
||||
|
||||
@@ -35,7 +35,7 @@ class TimestampedPreferences(UserDict):
|
||||
def __setitem__(self, key, value):
|
||||
self.data[key] = {
|
||||
'value': value,
|
||||
'ts': time.time()
|
||||
'ts': int(time.time())
|
||||
}
|
||||
|
||||
def __repr__(self):
|
||||
|
||||
@@ -35,6 +35,7 @@ disable=
|
||||
too-many-statements,
|
||||
too-many-nested-blocks,
|
||||
too-many-public-methods,
|
||||
too-many-return-statements,
|
||||
too-many-instance-attributes,
|
||||
protected-access,
|
||||
unused-argument
|
||||
@@ -46,7 +46,7 @@ setup(
|
||||
'msgpack==0.6.1',
|
||||
'prometheus_client==0.7.1',
|
||||
'ecdsa==0.13.3',
|
||||
'pyyaml==4.2b1',
|
||||
'pyyaml==5.3.1',
|
||||
'docopt==0.6.2',
|
||||
'hachoir',
|
||||
'multidict==4.6.1',
|
||||
|
||||
@@ -4,6 +4,7 @@ from lbry.testcase import CommandTestCase
|
||||
class EpicAdventuresOfChris45(CommandTestCase):
|
||||
|
||||
async def test_no_this_is_not_a_test_its_an_adventure(self):
|
||||
|
||||
# Chris45 is an avid user of LBRY and this is his story. It's fact and fiction
|
||||
# and everything in between; it's also the setting of some record setting
|
||||
# integration tests.
|
||||
@@ -194,3 +195,9 @@ class EpicAdventuresOfChris45(CommandTestCase):
|
||||
}},
|
||||
await self.resolve(uri)
|
||||
)
|
||||
|
||||
# He closes and opens the wallet server databases to see how horribly they break
|
||||
db = self.conductor.spv_node.server.db
|
||||
db.close()
|
||||
await db.open_for_serving()
|
||||
# They didn't! (error would be AssertionError: 276 vs 266 (264 counts) on startup)
|
||||
@@ -10,7 +10,7 @@ from lbry.stream.stream_manager import StreamManager
|
||||
from lbry.stream.reflector.server import ReflectorServer
|
||||
|
||||
|
||||
class TestStreamAssembler(AsyncioTestCase):
|
||||
class TestReflector(AsyncioTestCase):
|
||||
async def asyncSetUp(self):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.key = b'deadbeef' * 4
|
||||
@@ -22,6 +22,7 @@ class TestStreamAssembler(AsyncioTestCase):
|
||||
self.storage = SQLiteStorage(self.conf, os.path.join(tmp_dir, "lbrynet.sqlite"))
|
||||
await self.storage.open()
|
||||
self.blob_manager = BlobManager(self.loop, tmp_dir, self.storage, self.conf)
|
||||
self.addCleanup(self.blob_manager.stop)
|
||||
self.stream_manager = StreamManager(self.loop, Config(), self.blob_manager, None, self.storage, None)
|
||||
|
||||
server_tmp_dir = tempfile.mkdtemp()
|
||||
@@ -30,6 +31,7 @@ class TestStreamAssembler(AsyncioTestCase):
|
||||
self.server_storage = SQLiteStorage(self.server_conf, os.path.join(server_tmp_dir, "lbrynet.sqlite"))
|
||||
await self.server_storage.open()
|
||||
self.server_blob_manager = BlobManager(self.loop, server_tmp_dir, self.server_storage, self.server_conf)
|
||||
self.addCleanup(self.server_blob_manager.stop)
|
||||
|
||||
download_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(lambda: shutil.rmtree(download_dir))
|
||||
@@ -54,6 +56,7 @@ class TestStreamAssembler(AsyncioTestCase):
|
||||
set(map(lambda b: b.blob_hash,
|
||||
self.stream.descriptor.blobs[:-1] + [self.blob_manager.get_blob(self.stream.sd_hash)]))
|
||||
)
|
||||
self.assertTrue(self.stream.is_fully_reflected)
|
||||
server_sd_blob = self.server_blob_manager.get_blob(self.stream.sd_hash)
|
||||
self.assertTrue(server_sd_blob.get_is_verified())
|
||||
self.assertEqual(server_sd_blob.length, server_sd_blob.length)
|
||||
@@ -75,3 +78,112 @@ class TestStreamAssembler(AsyncioTestCase):
|
||||
to_announce = await self.storage.get_blobs_to_announce()
|
||||
self.assertIn(self.stream.sd_hash, to_announce, "sd blob not set to announce")
|
||||
self.assertIn(self.stream.descriptor.blobs[0].blob_hash, to_announce, "head blob not set to announce")
|
||||
|
||||
async def test_result_from_disconnect_mid_sd_transfer(self):
|
||||
stop = asyncio.Event()
|
||||
incoming = asyncio.Event()
|
||||
reflector = ReflectorServer(
|
||||
self.server_blob_manager, response_chunk_size=50, stop_event=stop, incoming_event=incoming
|
||||
)
|
||||
reflector.start_server(5566, '127.0.0.1')
|
||||
await reflector.started_listening.wait()
|
||||
self.addCleanup(reflector.stop_server)
|
||||
self.assertEqual(0, self.stream.reflector_progress)
|
||||
reflect_task = asyncio.create_task(self.stream.upload_to_reflector('127.0.0.1', 5566))
|
||||
await incoming.wait()
|
||||
stop.set()
|
||||
# this used to raise (and then propagate) a CancelledError
|
||||
self.assertListEqual(await reflect_task, [])
|
||||
self.assertFalse(self.stream.is_fully_reflected)
|
||||
|
||||
async def test_result_from_disconnect_after_sd_transfer(self):
|
||||
stop = asyncio.Event()
|
||||
incoming = asyncio.Event()
|
||||
not_incoming = asyncio.Event()
|
||||
reflector = ReflectorServer(
|
||||
self.server_blob_manager, response_chunk_size=50, stop_event=stop, incoming_event=incoming,
|
||||
not_incoming_event=not_incoming
|
||||
)
|
||||
reflector.start_server(5566, '127.0.0.1')
|
||||
await reflector.started_listening.wait()
|
||||
self.addCleanup(reflector.stop_server)
|
||||
self.assertEqual(0, self.stream.reflector_progress)
|
||||
reflect_task = asyncio.create_task(self.stream.upload_to_reflector('127.0.0.1', 5566))
|
||||
await incoming.wait()
|
||||
await not_incoming.wait()
|
||||
stop.set()
|
||||
self.assertListEqual(await reflect_task, [self.stream.sd_hash])
|
||||
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
|
||||
self.assertFalse(self.stream.is_fully_reflected)
|
||||
|
||||
async def test_result_from_disconnect_after_data_transfer(self):
|
||||
stop = asyncio.Event()
|
||||
incoming = asyncio.Event()
|
||||
not_incoming = asyncio.Event()
|
||||
reflector = ReflectorServer(
|
||||
self.server_blob_manager, response_chunk_size=50, stop_event=stop, incoming_event=incoming,
|
||||
not_incoming_event=not_incoming
|
||||
)
|
||||
reflector.start_server(5566, '127.0.0.1')
|
||||
await reflector.started_listening.wait()
|
||||
self.addCleanup(reflector.stop_server)
|
||||
self.assertEqual(0, self.stream.reflector_progress)
|
||||
reflect_task = asyncio.create_task(self.stream.upload_to_reflector('127.0.0.1', 5566))
|
||||
await incoming.wait()
|
||||
await not_incoming.wait()
|
||||
await incoming.wait()
|
||||
await not_incoming.wait()
|
||||
stop.set()
|
||||
self.assertListEqual(await reflect_task, [self.stream.sd_hash, self.stream.descriptor.blobs[0].blob_hash])
|
||||
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
|
||||
self.assertTrue(self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified())
|
||||
self.assertFalse(self.stream.is_fully_reflected)
|
||||
|
||||
async def test_result_from_disconnect_mid_data_transfer(self):
|
||||
stop = asyncio.Event()
|
||||
incoming = asyncio.Event()
|
||||
not_incoming = asyncio.Event()
|
||||
reflector = ReflectorServer(
|
||||
self.server_blob_manager, response_chunk_size=50, stop_event=stop, incoming_event=incoming,
|
||||
not_incoming_event=not_incoming
|
||||
)
|
||||
reflector.start_server(5566, '127.0.0.1')
|
||||
await reflector.started_listening.wait()
|
||||
self.addCleanup(reflector.stop_server)
|
||||
self.assertEqual(0, self.stream.reflector_progress)
|
||||
reflect_task = asyncio.create_task(self.stream.upload_to_reflector('127.0.0.1', 5566))
|
||||
await incoming.wait()
|
||||
await not_incoming.wait()
|
||||
await incoming.wait()
|
||||
stop.set()
|
||||
self.assertListEqual(await reflect_task, [self.stream.sd_hash])
|
||||
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
|
||||
self.assertFalse(
|
||||
self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified()
|
||||
)
|
||||
self.assertFalse(self.stream.is_fully_reflected)
|
||||
|
||||
async def test_delete_file_during_reflector_upload(self):
|
||||
stop = asyncio.Event()
|
||||
incoming = asyncio.Event()
|
||||
not_incoming = asyncio.Event()
|
||||
reflector = ReflectorServer(
|
||||
self.server_blob_manager, response_chunk_size=50, stop_event=stop, incoming_event=incoming,
|
||||
not_incoming_event=not_incoming
|
||||
)
|
||||
reflector.start_server(5566, '127.0.0.1')
|
||||
await reflector.started_listening.wait()
|
||||
self.addCleanup(reflector.stop_server)
|
||||
self.assertEqual(0, self.stream.reflector_progress)
|
||||
reflect_task = asyncio.create_task(self.stream.upload_to_reflector('127.0.0.1', 5566))
|
||||
await incoming.wait()
|
||||
await not_incoming.wait()
|
||||
await incoming.wait()
|
||||
await self.stream_manager.delete(self.stream, delete_file=True)
|
||||
# this used to raise OSError when it can't read the deleted blob for the upload
|
||||
self.assertListEqual(await reflect_task, [self.stream.sd_hash])
|
||||
self.assertTrue(self.server_blob_manager.get_blob(self.stream.sd_hash).get_is_verified())
|
||||
self.assertFalse(
|
||||
self.server_blob_manager.get_blob(self.stream.descriptor.blobs[0].blob_hash).get_is_verified()
|
||||
)
|
||||
self.assertFalse(self.stream.is_fully_reflected)
|
||||
@@ -132,7 +132,7 @@ class TestAccount(AsyncioTestCase):
|
||||
async def test_load_and_save_account(self):
|
||||
account_data = {
|
||||
'name': 'Main Account',
|
||||
'modified_on': 123.456,
|
||||
'modified_on': 123,
|
||||
'seed':
|
||||
"carbon smart garage balance margin twelve chest sword toast envelope bottom stomac"
|
||||
"h absent",
|
||||
@@ -183,7 +183,7 @@ class TestAccount(AsyncioTestCase):
|
||||
def test_merge_diff(self):
|
||||
account_data = {
|
||||
'name': 'My Account',
|
||||
'modified_on': 123.456,
|
||||
'modified_on': 123,
|
||||
'seed':
|
||||
"carbon smart garage balance margin twelve chest sword toast envelope bottom stomac"
|
||||
"h absent",
|
||||
@@ -203,7 +203,7 @@ class TestAccount(AsyncioTestCase):
|
||||
account = Account.from_dict(self.ledger, Wallet(), account_data)
|
||||
|
||||
self.assertEqual(account.name, 'My Account')
|
||||
self.assertEqual(account.modified_on, 123.456)
|
||||
self.assertEqual(account.modified_on, 123)
|
||||
self.assertEqual(account.change.gap, 5)
|
||||
self.assertEqual(account.change.maximum_uses_per_address, 2)
|
||||
self.assertEqual(account.receiving.gap, 5)
|
||||
@@ -366,7 +366,7 @@ class TestSingleKeyAccount(AsyncioTestCase):
|
||||
async def test_load_and_save_account(self):
|
||||
account_data = {
|
||||
'name': 'My Account',
|
||||
'modified_on': 123.456,
|
||||
'modified_on': 123,
|
||||
'seed':
|
||||
"carbon smart garage balance margin twelve chest sword toast envelope bottom stomac"
|
||||
"h absent",
|
||||
|
||||
@@ -35,11 +35,17 @@ class MockNetwork:
|
||||
|
||||
async def get_transaction_and_merkle(self, tx_hash, known_height=None):
|
||||
tx = await self.get_transaction(tx_hash)
|
||||
merkle = {}
|
||||
merkle = {'block_height': -1}
|
||||
if known_height:
|
||||
merkle = await self.get_merkle(tx_hash, known_height)
|
||||
return tx, merkle
|
||||
|
||||
async def get_transaction_batch(self, txids):
|
||||
return {
|
||||
txid: await self.get_transaction_and_merkle(txid)
|
||||
for txid in txids
|
||||
}
|
||||
|
||||
|
||||
class LedgerTestCase(AsyncioTestCase):
|
||||
|
||||
@@ -120,8 +126,9 @@ class TestSynchronization(LedgerTestCase):
|
||||
|
||||
self.ledger.network.get_history_called = []
|
||||
self.ledger.network.get_transaction_called = []
|
||||
for cache_item in self.ledger._tx_cache.values():
|
||||
cache_item.tx.is_verified = True
|
||||
self.assertFalse(self.ledger._tx_cache[txid1].tx.is_verified)
|
||||
self.assertFalse(self.ledger._tx_cache[txid2].tx.is_verified)
|
||||
self.assertFalse(self.ledger._tx_cache[txid3].tx.is_verified)
|
||||
await self.ledger.update_history(address, '')
|
||||
self.assertListEqual(self.ledger.network.get_history_called, [address])
|
||||
self.assertListEqual(self.ledger.network.get_transaction_called, [])
|
||||
|
||||
@@ -38,7 +38,7 @@ class TestWalletCreation(AsyncioTestCase):
|
||||
'certificates': {},
|
||||
'name': 'An Account',
|
||||
'ledger': 'lbc_mainnet',
|
||||
'modified_on': 123.456,
|
||||
'modified_on': 123,
|
||||
'seed':
|
||||
"carbon smart garage balance margin twelve chest sword toast envelope bottom stomac"
|
||||
"h absent",
|
||||
@@ -62,7 +62,7 @@ class TestWalletCreation(AsyncioTestCase):
|
||||
wallet = Wallet.from_storage(storage, self.manager)
|
||||
self.assertEqual(wallet.name, 'Main Wallet')
|
||||
self.assertEqual(
|
||||
hexlify(wallet.hash), b'a75913d2e7339c1a9ac0c89d621a4e10fd3a40dc3560dc01f4cf4ada0a0b05b8'
|
||||
hexlify(wallet.hash), b'869acc4660dde0f13784ed743796adf89562cdf79fdfc9e5c6dbea98d62ccf90'
|
||||
)
|
||||
self.assertEqual(len(wallet.accounts), 1)
|
||||
account = wallet.default_account
|
||||
|
||||
Reference in new issue
Block a user